def get_modules(modulename=None):
"""Return a list of modules and packages under modulename.
If modulename is not given, return a list of all top level modules
and packages.
"""
modulename = compat.ensure_not_unicode(modulename)
if not modulename:
return ([
modname for (importer, modname, ispkg) in iter_modules()
if not modname.startswith("_")
] + list(sys.builtin_module_names))
try:
module = safeimport(modulename)
except ErrorDuringImport:
return []
if module is None:
return []
if hasattr(module, "__path__"):
return [
modname
for (importer, modname, ispkg) in iter_modules(module.__path__)
if not modname.startswith("_")
]
return []
def get_modules(modulename=None):
"""Return a list of modules and packages under modulename.
If modulename is not given, return a list of all top level modules
and packages.
"""
modulename = compat.ensure_not_unicode(modulename)
if not modulename:
try:
return ([modname for (importer, modname, ispkg)
in iter_modules()
if not modname.startswith("_")] +
list(sys.builtin_module_names))
except OSError:
# Bug in Python 2.6, see #275
return list(sys.builtin_module_names)
try:
module = safeimport(modulename)
except ErrorDuringImport:
return []
if module is None:
return []
if hasattr(module, "__path__"):
return [modname for (importer, modname, ispkg)
in iter_modules(module.__path__)
if not modname.startswith("_")]
return []
def _extract_folder_module(self):
rel_path = os.path.relpath(self.target_dir,
self.fe_dir).replace(sep, ".")
dir_module = pydoc.safeimport(rel_path)
self.func_and_classes = []
self.file_modules = []
# pdb.set_trace()
for name, member in inspect.getmembers(dir_module):
if name.startswith("_"):
continue
if not member: # ex: EarlyStop
continue
source_path = inspect.getfile(member)
if not source_path.startswith(
self.fe_dir): # skip module not from fe ex: cv2
continue
if os.path.basename(
source_path) == "__init__.py": # skip folder module
continue
if inspect.isfunction(member) or inspect.isclass(member):
self.func_and_classes.append([name, member])
elif inspect.ismodule(member):
self.file_modules.append([name, member])
# remove file_modules in which any member of func_and_classes is defined
# ex: >>> from fastestimator.estimator import Estimator
# this will include both estimator.py and Estimator but we only want Estimator
for idx, (name,
module) in reversed(list(enumerate(self.file_modules))):
for _, member in self.func_and_classes:
if inspect.getmodule(member) == module:
self.file_modules.pop(idx)
break
def extractmarkdown(module, save_path):
output = list()
mod = pydoc.safeimport(inspect.getmodulename(module))
output.append('## ' + str(module))
output.append("***\n")
getclasses(mod, save_path)
getfunctions(mod, save_path)
return "".join(output)
def get_module(module):
if isinstance(module, str):
sys.path.append(os.getcwd())
# Attempt import
module = pydoc.safeimport(module)
if module is None:
print("Module not found")
return module
def load_module(module_path, up=None):
""" Get python object from name. """
obj = pydoc.safeimport(module_path)
if obj is None:
raise ImportError("%r not found" % module_path)
node = build_node(obj, up=up)
D("%r -> %r", module_path, node)
return node
def module_section(self, obj, package_context ):
"""Create a module-links section for the given object (module)"""
modules = inspect.getmembers(obj, inspect.ismodule)
package_context.clean(modules, obj)
package_context.recurse_scan(modules)
if hasattr(obj, '__path__'):
modpkgs = []
modnames = []
for file in os.listdir(obj.__path__[0]):
path = os.path.join(obj.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, obj.__name__, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, obj.__name__, 1, 0))
modpkgs.sort()
# do more recursion here...
for (modname, name, ya, yo) in modpkgs:
package_context.add_interesting('.'.join((obj.__name__, modname)))
items = []
for (modname, name, ispackage, is_shadowed) in modpkgs:
try:
# get the actual module obj...
#if modname == "events":
# import pdb
# pdb.set_trace()
module = pydoc.safeimport('{0}.{1}'.format(name, modname))
description, documentation = pydoc.splitdoc(inspect.getdoc(module))
if description:
items.append(
'{0} -- {1}'.format(
self.modpkglink((modname, name, ispackage, is_shadowed)),
description,
)
)
else:
items.append(
self.modpkglink((modname, name, ispackage, is_shadowed))
)
except:
items.append(
self.modpkglink((modname, name, ispackage, is_shadowed))
)
contents = '<br>'.join(items)
result = self.bigsection(
'Package Contents', '#ffffff', '#aa55cc', contents)
elif modules:
contents = self.multicolumn(
modules,
lambda a: self.modulelink(a[1])
)
result = self.bigsection(
'Modules', '#fffff', '#aa55cc', contents)
else:
result = ""
return result
def moduleSection(self, object, packageContext):
"""Create a module-links section for the given object (module)"""
modules = inspect.getmembers(object, inspect.ismodule)
packageContext.clean(modules, object)
packageContext.recurseScan(modules)
if hasattr(object, '__path__'):
modpkgs = []
modnames = []
for file in os.listdir(object.__path__[0]):
path = os.path.join(object.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, object.__name__, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, object.__name__, 1, 0))
modpkgs.sort()
# do more recursion here...
for (modname, name, ya, yo) in modpkgs:
packageContext.addInteresting(
join((object.__name__, modname), '.'))
items = []
for (modname, name, ispackage, isshadowed) in modpkgs:
try:
# get the actual module object...
## if modname == "events":
## import pdb
## pdb.set_trace()
module = pydoc.safeimport("%s.%s" % (name, modname))
description, documentation = pydoc.splitdoc(
inspect.getdoc(module))
if description:
items.append("""%s -- %s""" % (
self.modpkglink(
(modname, name, ispackage, isshadowed)),
description,
))
else:
items.append(
self.modpkglink(
(modname, name, ispackage, isshadowed)))
except:
items.append(
self.modpkglink(
(modname, name, ispackage, isshadowed)))
contents = string.join(items, '<br>')
result = self.bigsection('Package Contents', '#ffffff', '#aa55cc',
contents)
elif modules:
contents = self.multicolumn(
modules, lambda (key, value), s=self: s.modulelink(value))
result = self.bigsection('Modules', '#fffff', '#aa55cc', contents)
else:
result = ""
return result
def module_section(self, obj, package_context):
"""Create a module-links section for the given object (module)"""
modules = inspect.getmembers(obj, inspect.ismodule)
package_context.clean(modules, obj)
package_context.recurse_scan(modules)
if hasattr(obj, '__path__'):
modpkgs = []
modnames = []
for file in os.listdir(obj.__path__[0]):
path = os.path.join(obj.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, obj.__name__, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, obj.__name__, 1, 0))
modpkgs.sort()
# do more recursion here...
for (modname, name, ya, yo) in modpkgs:
package_context.add_interesting('.'.join(
(obj.__name__, modname)))
items = []
for (modname, name, ispackage, is_shadowed) in modpkgs:
try:
# get the actual module obj...
#if modname == "events":
# import pdb
# pdb.set_trace()
module = pydoc.safeimport('{0}.{1}'.format(name, modname))
description, documentation = pydoc.splitdoc(
inspect.getdoc(module))
if description:
items.append('{0} -- {1}'.format(
self.modpkglink(
(modname, name, ispackage, is_shadowed)),
description,
))
else:
items.append(
self.modpkglink(
(modname, name, ispackage, is_shadowed)))
except:
items.append(
self.modpkglink(
(modname, name, ispackage, is_shadowed)))
contents = '<br>'.join(items)
result = self.bigsection('Package Contents', '#ffffff', '#aa55cc',
contents)
elif modules:
contents = self.multicolumn(modules,
lambda a: self.modulelink(a[1]))
result = self.bigsection('Modules', '#fffff', '#aa55cc', contents)
else:
result = ""
return result
def moduleSection( self, object, packageContext ): """Create a module-links section for the given object (module)""" modules = inspect.getmembers(object, inspect.ismodule) packageContext.clean ( modules, object ) packageContext.recurseScan( modules ) if hasattr(object, '__path__'): modpkgs = [] modnames = [] for file in os.listdir(object.__path__[0]): path = os.path.join(object.__path__[0], file) modname = inspect.getmodulename(file) if modname and modname not in modnames: modpkgs.append((modname, object.__name__, 0, 0)) modnames.append(modname) elif pydoc.ispackage(path): modpkgs.append((file, object.__name__, 1, 0)) modpkgs.sort() # do more recursion here... for (modname, name, ya,yo) in modpkgs: packageContext.addInteresting( join( (object.__name__, modname), '.')) items = [] for (modname, name, ispackage,isshadowed) in modpkgs: try: # get the actual module object... ## if modname == "events": ## import pdb ## pdb.set_trace() module = pydoc.safeimport( "%s.%s"%(name,modname) ) description, documentation = pydoc.splitdoc( inspect.getdoc( module )) if description: items.append( """%s -- %s"""% ( self.modpkglink( (modname, name, ispackage, isshadowed) ), description, ) ) else: items.append( self.modpkglink( (modname, name, ispackage, isshadowed) ) ) except: items.append( self.modpkglink( (modname, name, ispackage, isshadowed) ) ) contents = string.join( items, '<br>') result = self.bigsection( 'Package Contents', '#ffffff', '#aa55cc', contents) elif modules: contents = self.multicolumn( modules, lambda (key, value), s=self: s.modulelink(value)) result = self.bigsection( 'Modules', '#fffff', '#aa55cc', contents) else: result = "" return result
def pydoc2markdown(module_):
try:
path.append(getcwd())
module_ = safeimport(module_)
if module_ is None:
print("%s module not found" % module_)
print(module2markdown(module_))
except ErrorDuringImport:
print("Error while importing %s" % module_)
def generatedocs(module):
try:
sys.path.append(os.getcwd())
# Attempt import
mod = pydoc.safeimport(module)
if mod is None:
print("Module not found")
# Module imported correctly, let's create the docs
return getmarkdown(mod)
except pydoc.ErrorDuringImport as e:
print("Error while trying to import " + module)
def import_module_safely(module_name):
try:
sys.path.append(os.getcwd())
mod = pydoc.safeimport(module_name)
if mod is None:
print "Module not found ..."
return mod
except pydoc.ErrorDuringImport as e:
print "Error while trying to import {} ... The docs could not be generated".format(module_name)
sys.exit(1)
def generatedocs(module, filename):
try:
sys.path.insert(0, os.getcwd() + "/..")
# Attempt import
mod = pydoc.safeimport(module)
if mod is None:
print("Module not found")
# Module imported correctly, let's create the docs
with open(filename, "w") as f:
f.write(getmarkdown(mod))
except pydoc.ErrorDuringImport as e:
print("Error while trying to import " + module)
def locate(path, forceload=0):
"""Locate an object by name or dotted path, importing as necessary."""
parts = [part for part in path.split('.') if part]
module, n = None, 0
while n < len(parts):
nextmodule = safeimport('.'.join(parts[:n + 1]), forceload)
if nextmodule:
module, n = nextmodule, n + 1
else:
break
for part in parts[n:]:
try:
module = getattr(module, part)
except AttributeError:
raise RuntimeError('Object: {} not found'.format(path))
return module
def generate_doc(self, module_import_path: str, output_path: str) -> None:
try:
module = pydoc.safeimport(module_import_path)
if module is None:
logging.error("Module not found")
raise FileExistsError("Module not found")
except pydoc.ErrorDuringImport as e:
logging.error(
f"Error while trying to import {module_import_path}: {e}")
raise e
else:
relpath = self._get_relpath(module.__file__, output_path)
docs = self._get_full_markdown(module,
relpath.replace(os.path.sep, "/"))
self._write_lines(output_path, docs)
def resolve_name(path, forceload=0):
"""Locate an object by name or dotted path, importing as necessary."""
parts = [part for part in path.split('.') if part]
module, n = None, 0
while n < len(parts):
nextmodule = safeimport('.'.join(parts[:n + 1]), forceload)
if nextmodule:
module, n = nextmodule, n + 1
else:
break
if module:
object = module
else:
object = __builtin__
for part in parts[n:]:
try:
object = getattr(object, part)
except AttributeError:
return None
return object
def get_modules(modulename=None):
"""Return a list of modules and packages under modulename.
If modulename is not given, return a list of all top level modules
and packages.
"""
if not modulename:
return ([modulename for (importer, modulename, ispkg)
in iter_modules()
if not modulename.startswith("_")] +
list(sys.builtin_module_names))
try:
module = safeimport(modulename)
except ErrorDuringImport:
return None
if module is None:
return None
if hasattr(module, "__path__"):
return [modulename for (importer, modulename, ispkg)
in iter_modules(module.__path__)
if not modulename.startswith("_")]
return None
def _extract_api_module(self):
self.module_infos = []
for f in os.listdir(self.target_dir):
if not f.endswith(".py"):
continue
f_rel_path = os.path.relpath(
os.path.join(self.target_dir, f),
self.fe_dir,
)
f_module_path = f_rel_path.replace(sep, ".")
f_module = pydoc.safeimport(inspect.getmodulename(f_module_path))
for name, api_module in inspect.getmembers(f_module):
if inspect.getmodule(api_module) != f_module:
continue
if name.startswith('_'):
continue
# load -> cifar10.load
substitue_name = f"{f.split('.')[-2]}.{name}"
self.module_infos.append(
[name, api_module, substitue_name, f_rel_path]
)
datefmt='%m/%d/%Y %I:%M:%S',
level=args.loglevel)
logger = logging.getLogger(__name__)
logger.debug("Arguments = %s" % args)
loopcount = args.daemon
if args.daemon == None:
loopcount = -1
logger.debug("Loopcount = %d", loopcount)
# load referenced packages (loading here since below is in while loop)
modfuncs = {}
for p in args.stype:
try:
logger.debug("Loading module %s" % p)
statmod = safeimport(p)
statfn = getattr(statmod, "status")
modfuncs[p] = statfn
except:
logger.exception(
"Package %s in configfile %s not found or doesn't have status() function"
% (p, config["sourcefile"]))
sys.exit(1)
sleepytime = -1
packages = config["packages"]
while 1:
data = ""
for p in args.stype:
try:
def spyder_safeimport(path, forceload=0, cache={}):
if path in DIRECT_PYDOC_IMPORT_MODULES:
forceload = 0
return safeimport(path, forceload=forceload, cache=cache)
def document(self, module):
"""
Produce documentation for a given module object.
"""
module_name = module.__name__
attributes = {'name': module_name}
self.start_element('module', attributes)
absfile, module_type = self.module_info[module_name]
mtime = os.stat(absfile)[stat.ST_MTIME]
mtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(mtime))
self.write_element('modification-date', content=mtime)
self.write_description(module)
# standard module metadata
for attr in ['author', 'credits', 'date', 'version']:
if hasattr(module, '__%s__' % attr):
content = self.escape(str(getattr(module, '__%s__' % attr)))
self.write_element(attr, content=content)
submodules = []
if module_type == imp.PKG_DIRECTORY:
name = re.escape(module_name)
submodule_match = re.compile(r'^%s\.([^.]+)$' % name).match
for fullname in self.module_info:
match = submodule_match(fullname)
if match:
name = match.group(1)
try:
submod = pydoc.safeimport(fullname)
except:
submod = None
if submod is None:
# Could not import for some reason, create a stub
submod = imp.new_module(fullname)
submodules.append((name, submod))
all = getattr(module, '__all__', [])
# Only document the modules listed in __all__.
for name, member in inspect.getmembers(module, inspect.ismodule):
if name in all:
submodules.append((name, member))
if submodules:
submodules.sort()
self.section('modules', submodules, self.doc_submodule)
# Only document the classes defined in this module or
# are listed in __all__.
def isclass(object):
"""
Replacement for inspect's broken isclass() which fails for
instances of classes which define a custom __getattr__.
"""
return isinstance(object, (types.ClassType, type))
classes = [
t for t in inspect.getmembers(module, isclass)
if ((inspect.getmodule(t[1]) or module) is module or t[0] in all)
]
if classes:
self.section('classes', classes, self.doc_class)
# Only document the functions defined in this module or
# are listed in __all__.
funcs = [
t for t in inspect.getmembers(module, inspect.isroutine)
if ((inspect.getmodule(t[1]) or module) is module or t[0] in all)
]
if funcs:
self.section('functions', funcs, self.doc_function)
# Only document the "visible" objects or those listed in __all__.
globals = [
t for t in inspect.getmembers(module, pydoc.isdata)
if t[0] in all or _visiblename(t[0])
]
if globals:
self.section('globals', globals, self.doc_global)
self.end_element('module')
return
def html(self, level=1):
ret = "<h%(level)d class='title'>%(name)s%(sign)s</h%(level)s>\n" % {
"level": level,
"name": self.name,
"sign": self.signature
}
#ret="<h1 class='title'>"+self.name+"</h1>\n"
ret += "<section>\n"
ret += "%s</section>\n" % self.doc_html
return ret
sys.path.append(os.getcwd())
microdaemon = pydoc.safeimport("microdaemon")
if microdaemon is None:
print("Module not found")
sys.exit(1)
modlist = [] #getfunctions(microdaemon) + getclasses(microdaemon)
for entry in os.scandir("microdaemon"):
fname = entry.name
if fname == "__init__.py": continue
if not fname.endswith(".py"):
continue
modname = "microdaemon.%s" % fname.replace(".py", "")
module = pydoc.safeimport(modname)
def document(self, module):
"""
Produce documentation for a given module object.
"""
module_name = module.__name__
attributes = {"name": module_name}
self.start_element("module", attributes)
absfile, module_type = self.module_info[module_name]
mtime = os.stat(absfile)[stat.ST_MTIME]
mtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(mtime))
self.write_element("modification-date", content=mtime)
self.write_description(module)
# standard module metadata
for attr in ["author", "credits", "date", "version"]:
if hasattr(module, "__%s__" % attr):
content = self.escape(str(getattr(module, "__%s__" % attr)))
self.write_element(attr, content=content)
submodules = []
if module_type == imp.PKG_DIRECTORY:
name = re.escape(module_name)
submodule_match = re.compile(r"^%s\.([^.]+)$" % name).match
for fullname in self.module_info:
match = submodule_match(fullname)
if match:
name = match.group(1)
try:
submod = pydoc.safeimport(fullname)
except:
submod = None
if submod is None:
# Could not import for some reason, create a stub
submod = imp.new_module(fullname)
submodules.append((name, submod))
all = getattr(module, "__all__", [])
# Only document the modules listed in __all__.
for name, member in inspect.getmembers(module, inspect.ismodule):
if name in all:
submodules.append((name, member))
if submodules:
submodules.sort()
self.section("modules", submodules, self.doc_submodule)
# Only document the classes defined in this module or
# are listed in __all__.
def isclass(object):
"""
Replacement for inspect's broken isclass() which fails for
instances of classes which define a custom __getattr__.
"""
return isinstance(object, (types.ClassType, type))
classes = [
t
for t in inspect.getmembers(module, isclass)
if ((inspect.getmodule(t[1]) or module) is module or t[0] in all)
]
if classes:
self.section("classes", classes, self.doc_class)
# Only document the functions defined in this module or
# are listed in __all__.
funcs = [
t
for t in inspect.getmembers(module, inspect.isroutine)
if ((inspect.getmodule(t[1]) or module) is module or t[0] in all)
]
if funcs:
self.section("functions", funcs, self.doc_function)
# Only document the "visible" objects or those listed in __all__.
globals = [t for t in inspect.getmembers(module, pydoc.isdata) if t[0] in all or _visiblename(t[0])]
if globals:
self.section("globals", globals, self.doc_global)
self.end_element("module")
return
def run_lucidoc(pkg,
parse_style,
outfile=None,
outfolder=None,
no_mod_docstr=False,
include_inherited=False,
whitelist=None,
blacklist=None,
groups=None,
omit_meta=False,
**log_kwargs):
"""
Discover docstrings and create package API documentation in Markdown.
:param str pkg: name of the package to document
:param str parse_style: key/name of parsing strategy to use
:param str outfile: path to documentation output file
:param str outfolder: path to folder in which to place docs output
:param bool no_mod_docstr: whether to exclude the module-level docstring,
if present
:param bool include_inherited: whether to document inherited members
:param Iterable[str] whitelist: names of doc targets to include
:param Iterable[str] blacklist: names of doc targets to exclude
:param Mapping[str, str | Iterable[str]] | Iterable[(str, str | Iterable[str])] groups:
pairing of group name with either single target name or collection of target names
:param bool omit_meta: whether the version metadata for documentation
target and for this package should be omitted from the documentation
that's created
:raise TypeError: if passing an argument to whitelist or blacklist that's
not a non-string collection, or if passing an argument to groups in
which a group's names isn't a non-string collection
:raise LucidocError: if passing both output file and output folder, or if
passing output file and using groups; or if using more than one of
whitelist, blacklist, and groups
:raise pydoc.ErrorDuringImport: if the argument to the package parameter
(pkg) cannot be imported
"""
global _LOGGER
_LOGGER = setup_logger(**log_kwargs)
if outfile and outfolder:
raise LucidocError("Cannot specify both output file and output folder")
groups = _standardize_groups_type(groups)
retain = _determine_retention_strategy(whitelist, blacklist, groups)
if groups and outfile:
raise LucidocError(
"Cannot use output file with groups; to control output destination, "
"consider output folder and each group names as filename.")
try:
sys.path.append(os.getcwd())
# Attempt import
pkg_obj = pydoc.safeimport(pkg)
if pkg_obj is None:
raise LucidocError(
"ERROR -- Documentation target not found: {}".format(pkg))
except pydoc.ErrorDuringImport:
_LOGGER.error("Failed to import '{}'".format(pkg))
raise
else:
show_tag = MdTagRenderer()
parse = get_parser(parse_style)
doc_res = doc_module(pkg_obj,
parse,
show_tag,
no_mod_docstr=no_mod_docstr,
include_inherited=include_inherited,
retain=retain,
groups=groups,
omit_meta=omit_meta)
if groups:
outfolder = expandpath(outfolder or os.getcwd())
_LOGGER.debug("Base output folder: {}".format(outfolder))
missing, invalid = [], []
for g, _ in groups:
try:
doc = doc_res[g]
except KeyError:
missing.append(g)
continue
base, ext = os.path.splitext(g)
if ext:
if ext != ".md":
invalid.append(g)
continue
fn = g
else:
fn = base + ".md"
_write_docs(os.path.join(outfolder, fn), doc)
if missing:
_LOGGER.warning("Missing output for {} group(s): {}".format(
len(missing), ", ".join(missing)))
if invalid:
_LOGGER.warning(
"Skipped writing {} group(s) on account of illegal output "
"file extension: {}".format(len(invalid),
", ".join(invalid)))
_LOGGER.info("Done.")
elif outfile:
_write_docs(outfile, doc_res)
_LOGGER.info("Done.")
else:
print(doc_res)
def update_event(self, inp=-1):
self.set_output_val(
0, pydoc.safeimport(self.input(0), self.input(1), self.input(2)))