def collect_docstrings(self):
"""Collect list of paths + classes + methods/functions, and docstrings.
This function assumes that there are no classes or methods/functions
nested within others.
"""
self.node_sequence = []
# Module-level docstrings.
self.node_sequence.append([self.path, ast.get_docstring(self.module)])
# Class-level doc-strings
# Function-level doc-strings
for class_def in self.class_defs:
for node in class_def.body:
if isinstance(node, ast.ClassDef):
self.node_sequence.append(
[self.path + '.' + node.name,
ast.get_docstring(node)]
)
elif isinstance(node, ast.FunctionDef):
if self.tests_only and node.name[:5] != 'test_':
continue
self.node_sequence.append(
[self.path + '.' + class_def.name + '.' + node.name,
ast.get_docstring(node)]
)
for func_def in self.func_defs:
if isinstance(func_def, ast.FunctionDef):
if self.tests_only and func_def.name[:5] != 'test_':
continue
self.node_sequence.append(
[self.path + '.' + func_def.name,
ast.get_docstring(func_def)]
)
def test_get_docstring(self):
node = ast.parse('def foo():\n """line one\n line two"""')
self.assertEqual(ast.get_docstring(node.body[0]),
'line one\nline two')
node = ast.parse('async def foo():\n """spam\n ham"""')
self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham')
def _parse_docstring(self):
"""Parses the test docstring extracting expected values.
If the expected tags is not spelled right they will not be parsed.
"""
if self.docstring is None:
return
# Create the contexts
tags, unexpected_tags, _ = self._parse_tags(
ast.get_docstring(self.module_def))
class_tags, class_unexpected_tags, _ = self._parse_tags(
ast.get_docstring(self.parent_class_def))
function_tags, function_unexpected_tags, self.skipped_lines = (
self._parse_tags(self.docstring))
# Update context dictionaries
tags.update(class_tags)
tags.update(function_tags)
unexpected_tags.update(class_unexpected_tags)
unexpected_tags.update(function_unexpected_tags)
for tag, value in tags.items():
if tag == 'bz':
tag = 'bugs'
if tag == 'assert':
tag = 'assertion'
if tag == 'type':
tag = 'test_type'
setattr(self, tag, value)
self.unexpected_tags = unexpected_tags
# Always use the first line of docstring as test case name
if self.test is None:
self.test = self.docstring.strip().split('\n')[0]
def parse_module(file_path):
module = ast.parse(file_path.read_text())
tmpl_str = HEADER.format(file_path.name.rsplit('.',1)[0])
cls_defs = [node for node in module.body if isinstance(node, ast.ClassDef)]
mod_func_defs = [node for node in module.body if isinstance(node, ast.FunctionDef)]
for cls_def in cls_defs:
cls_name = cls_def.name
cls_bases = ','.join([parse(each) for each in cls_def.bases])
tmpl_str += f'== {{{{class {cls_name}{":" + cls_bases if cls_bases else ""}}}}}\n\n'
method_str = None
for fn_def in (fn_def for fn_def in cls_def.body if isinstance(fn_def, ast.FunctionDef)):
if fn_def.name == '__init__':
tmpl_str += "=== Arguments\n" + parse_args(fn_def.args) + "\n\n"
else:
if not method_str:
method_str = '=== Methods\n\n'
doc_str = ast.get_docstring(fn_def)
method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n'
tmpl_str += method_str if method_str else ''
method_str = None
for fn_def in mod_func_defs:
if not method_str:
method_str = '== Module Functions\n\n'
doc_str = ast.get_docstring(fn_def)
method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n'
tmpl_str += method_str if method_str else ''
return tmpl_str
def extract_info(self):
for node in self.module.body:
if isinstance(node, ast.ClassDef):
yield {
"name": node.name,
"lineno": node.lineno,
"docstring": ast.get_docstring(node),
"type": 'class',
}
for sub_node in node.body:
if isinstance(sub_node, ast.FunctionDef):
yield {
"name": sub_node.name,
"lineno": sub_node.lineno,
"docstring": ast.get_docstring(sub_node),
"type": 'attribute',
"args": [arg.id for arg in sub_node.args.args],
"header": ''
}
elif isinstance(node, ast.FunctionDef):
yield {
"name": node.name,
"lineno": node.lineno,
"docstring": ast.get_docstring(node),
"type": 'function',
"args": [arg.id for arg in node.args.args],
}
def get_doc(self, module):
for node in module.body:
if isinstance(node, ast.ClassDef):
yield ast.get_docstring(node)
for sub_node in node.body:
if isinstance(sub_node, ast.FunctionDef):
yield ast.get_docstring(sub_node)
def __init__(self, function_def, parent_class=None, testmodule=None):
"""Wrap a ``ast.FunctionDef`` instance used to extract information."""
self.docstring = ast.get_docstring(function_def)
self.function_def = function_def
self.name = function_def.name
if parent_class:
self.parent_class = parent_class.name
self.parent_class_def = parent_class
self.class_docstring = ast.get_docstring(self.parent_class_def)
else:
self.parent_class = None
self.parent_class_def = None
self.class_docstring = None
self.testmodule = testmodule.path
self.module_def = testmodule
self.module_docstring = ast.get_docstring(self.module_def)
self.pkginit = os.path.join(
os.path.dirname(self.testmodule), '__init__.py')
if os.path.exists(self.pkginit):
self.pkginit_def = ast.parse(''.join(open(self.pkginit)))
self.pkginit_docstring = ast.get_docstring(self.pkginit_def)
else:
self.pkginit_def = None
self.pkginit_docstring = None
self.tokens = {}
self.invalid_tokens = {}
self._rst_parser_messages = []
self.parser = DocstringParser(
SETTINGS.get('tokens'),
SETTINGS.get('minimum_tokens'),
)
self._parse_docstring()
def _getMarvinTestDocStrings(classname, testnames, marvinCodePath):
pathToClass = os.path.join(marvinCodePath, *classname.split('.')[1:-1])+'.py'
astData = ast.parse(open(pathToClass).read())
classElement = filter(lambda x:isinstance(x, ast.ClassDef) and x.name == classname.split('.')[-1], astData.body)[0]
classDocString = ast.get_docstring(classElement)
classDocString = classDocString and classDocString.rstrip() or ''
testMethodElements = filter(lambda x:isinstance(x, ast.FunctionDef) and x.name in testnames, classElement.body)
testMethodDocStrings = []
for testMethod in testMethodElements:
docStr = ast.get_docstring(testMethod)
docStr = docStr and docStr.rstrip() or ''
testMethodDocStrings.append((testMethod.name, docStr))
return (classDocString, testMethodDocStrings)
def _extract_default_fits_file(self, path):
with open(path, 'r') as rfile:
m = ast.parse(rfile.read())
docstr = ast.get_docstring(m)
yd = yaml.load(docstr)
if yd:
return yd.get('default_fits', None)
def _parse_function(symbol, with_docstrings):
docstring = {}
attrs = {}
func = {'functions': {}}
func_name = symbol.name + '('
#We store the arguments to compare with default backwards
defaults = []
for value in symbol.args.defaults:
#TODO: In some cases we can have something like: a=os.path
defaults.append(value)
arguments = []
for arg in reversed(symbol.args.args):
if arg.__class__ is not _ast.Name or arg.id == 'self':
continue
argument = arg.id
if defaults:
value = defaults.pop()
arg_default = _map_type.get(value.__class__, None)
if arg_default is None:
if value.__class__ is _ast.Attribute:
arg_default = analyzer.expand_attribute(value)
elif value.__class__ is _ast.Name:
arg_default = value.id
else:
arg_default = 'object'
argument += '=' + arg_default
arguments.append(argument)
func_name += ', '.join(reversed(arguments))
if symbol.args.vararg is not None:
if not func_name.endswith('('):
func_name += ', '
func_name += '*' + symbol.args.vararg
if symbol.args.kwarg is not None:
if not func_name.endswith('('):
func_name += ', '
func_name += '**' + symbol.args.kwarg
func_name += ')'
for sym in symbol.body:
if sym.__class__ is ast.Assign:
result = _parse_assign(sym)
attrs.update(result[1])
elif sym.__class__ is ast.FunctionDef:
result = _parse_function(sym, with_docstrings)
if with_docstrings:
docstring.update(result['docstring'])
func['functions'][result['name']] = {'lineno': result['lineno'],
'functions':
result['functions']}
if with_docstrings:
docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)
lineno = symbol.lineno
for decorator in symbol.decorator_list:
lineno += 1
return {'name': func_name, 'lineno': lineno,
'attrs': attrs, 'docstring': docstring, 'functions': func}
def parse_tree (self, name, node):
"""
Recursive function that explores the python parse tree and
constructs our TreeNode datastructure to represent a document.
Takes the root node of the tree provided by the std ast module and
the file name.
"""
logging.debug('Exploring node %s of %s' % (node, name))
tree_node = ParserNode(
type = type_lookup[type(node)],
name = node.name if 'name' in node.__dict__ else \
name.split('.')[0],
docstring = ast.get_docstring(node),
children = [],
terms = []
)
for child in node.body:
# We only want to look for modules, functions and classes as
# these are the only items that have doc strings and therefore
# terms at the moment. Later on, we'll want to use imports and
# function names and other elements of the parse tree.
if type(child) not in [ast.Module, ast.FunctionDef, ast.ClassDef]:
continue
tree_node.children.append(self.parse_tree(name, child))
return tree_node
def visit_FunctionDef(node):
""" https://docs.python.org/2/library/ast.html#ast.NodeVisitor.visit """
if node.name != self.controller_method_name:
return
doc = ast.get_docstring(node)
raise StopIteration(doc if doc else u"")
def crop_to_docstring(strlines, outputLineNums=False, includeMarks=False):
'''Gets the first found docstring from the input code.
.. NOTE::
If the input contains the outer quotes ""/'', then the output will, too
:TODO: Make an optional output for line nums, which will be necessary for
Style Guide classes later on
:TODO: Force the output of the cropped docstring to include its original ''
Args:
strlines (str or iter): The snippet of code that contains the docstring
Returns:
str : The full docstring
'''
try:
iter(strlines)
if not isinstance(strlines, six.string_types):
raise TypeError
except TypeError:
strlines = '\n'.join(strlines)
module = ast.parse(strlines)
defs = [node for node in module.body
if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Module))]
for funcDef in defs:
return ast.get_docstring(funcDef)
def get_init_code(tree, table):
"""Get one-lined code from `tree` and `table.
Calculate the helper variables that we will need, and wrap the output
code (computed from `tree`) with definitions of those variables.
TODO: Short-circuit to something far simpler if the program has only one
print statement.
Arguments:
tree: Python AST node
table: symtable.symtable
Returns:
string - valid one-line code
"""
output = Namespace(table).many_to_one(tree.body)
doc = ast.get_docstring(tree, clean=False)
if doc is not None:
output = assignment_component(output, T("{__g}['__doc__']"), repr(doc))
output = provide(
output.format(__l=T('{__g}')),
__print=T("__import__('__builtin__').__dict__['print']"),
__exec="__import__('trace').Trace(count=False,"
" trace=False).runctx",
__y="(lambda f: (lambda x: x(x))(lambda y:"
" f(lambda: y(y)())))",
__g=T("globals()"),
__contextlib="__import__('contextlib')",
__sys="__import__('sys')",
__types="__import__('types')")
return output.close()
def find_info(*path_parts):
finder = VersionFinder()
node = ast.parse(read(*path_parts))
finder.visit(node)
info = finder.data
info['docstring'] = ast.get_docstring(node)
return info
def scripts2rst(path, f):
""" creates rst summary documentation for files in scripts folder
which is not a package so cannot be imported """
# get list of script files
try:
files = [name for name in os.listdir(path)
if not name.startswith("_")
and name.endswith(".py")]
except:
return
f.write("**Scripts**\n\n")
# loop over script files
for name in files:
sfile = os.path.join(path, name)
try:
try:
source = ast.parse(open(sfile, "r", encoding="utf8").read())
except:
#py2 fails if encoding in string
source = ast.parse(open(sfile, "r").read())
except Exception as e:
log.warning("Problem parsing %s\n%s"%(name, e))
f.write(name+"\n")
doc = i.cleandoc(ast.get_docstring(source)) or "."
doc = py2decode(doc) #py2
f.write(" "+doc.splitlines()[0]+"\n")
f.write("\n")
def main():
with open(FILENAME) as fd:
file_contents = fd.read()
module = ast.parse(file_contents)
print('loaded file')
function_definitions = [node for node in module.body if isinstance(node, ast.FunctionDef)]
for f in function_definitions:
docstring = ast.get_docstring(f)
params = {}
returntype = ""
for line in docstring.splitlines():
# if ":param" in line:
# param = line.split()[2]
# params[param] = "Any"
if ":type" in line:
# this line is a type
tokens = line.split()
param = tokens[1][:-1]
ty = tokens[2]
# print ("param is " + param + " with type " + ty)
params[param] = ty
string = "def {0}({1})".format(
f.name, ", ".join(
["%s: %s" % (param, params[param]) for param in params]))
'''
string = "def " + f.name + "("
for param in params:
string += "" + param + ": " + params[param] + ", "
string += "):"
'''
print string
def visit_FunctionDef(self, tree):
# self.visit() returns something of the form
# ('lambda x, y, z=5, *args: ', ['x', 'y', 'z', 'args'])
args, arg_names = self.visit(tree.args)
decoration = T('{}')
for decorator in tree.decorator_list:
decoration = decoration.format(T('{}({})').format(self.visit(decorator), T('{}')))
ns = self.next_child()
body = ns.many_to_one(tree.body).format(pre_return='', post_return='')
if arg_names:
body = assignment_component(body,
T(', ').join(ns.var(name) for name in arg_names),
T(', ').join(arg_names))
body = self.close(ns, '{}', body)
function_code = args + body
doc = ast.get_docstring(tree, clean=False)
if tree.decorator_list:
return assignment_component(
T('{after}'),
self.store_var(tree.name),
decoration.format(assignment_component(
'__func',
'__func, __func.__name__' + ('' if doc is None else ', __func.__doc__'),
T('{}, {!r}' + ('' if doc is None else ', {!r}')).format(
function_code, tree.name, doc))))
else:
return assignment_component(
T('{after}'),
T('{}, {}.__name__' + ('' if doc is None else ', {}.__doc__')).format(
self.store_var(tree.name), self.var(tree.name), self.var(tree.name)),
T('{}, {!r}' + ('' if doc is None else ', {!r}')).format(
function_code, tree.name, doc))
def ast(self, value): #@DuplicatedSignature
if not isinstance(value, _ast.FunctionDef):
raise Error("Root of ast must be a FunctionDef, but got a %s." %
value.__class__.__name__)
self._ast = value
self.__name__ = value.name
self.__doc__ = _ast.get_docstring(value, clean=False)
def visit_FunctionDef(self, node, **kwargs):
"""
Handles function definitions within code.
Process a function's docstring, keeping well aware of the function's
context and whether or not it's part of an interface definition.
"""
if self.options.debug:
stderr.write("# Function {0.name}{1}".format(node, linesep))
# Push either 'interface' or 'class' onto our containing nodes
# hierarchy so we can keep track of context. This will let us tell
# if a function is nested within another function or even if a class
# is nested within a function.
containingNodes = kwargs.get('containingNodes', []) or []
containingNodes.append((node.name, 'function'))
if self.options.topLevelNamespace:
fullPathNamespace = self._getFullPathName(containingNodes)
contextTag = '.'.join(pathTuple[0] for pathTuple in fullPathNamespace)
modifiedContextTag = self._processMembers(node, contextTag)
tail = '@namespace {0}'.format(modifiedContextTag)
else:
tail = self._processMembers(node, '')
if get_docstring(node):
self._processDocstring(node, tail,
containingNodes=containingNodes)
# Visit any contained nodes.
self.generic_visit(node, containingNodes=containingNodes)
# Remove the item we pushed onto the containing nodes hierarchy.
containingNodes.pop()
def _get_service_classes(self, node, path):
ret = []
classes = [c for c in node.body if isinstance(c, ast.ClassDef)]
for cls in classes:
exposed = False
bases = []
for cls_base in cls.bases:
if cls_base.id in ['StorageService', 'ComputationalService']:
exposed = True
bases.append(cls_base.id)
if not exposed:
continue
ret.append({
'defs': self._get_exposed_funcs(cls, path, method=True),
'name': cls.name,
'doc': ast.get_docstring(cls, clean=True),
'bases': bases,
'path': path
})
return ret
def _parse_py_file(py_file):
tree = ast.parse("".join(open(py_file)))
# noinspection PyArgumentEqualDefault
docstring = (ast.get_docstring(tree, clean=True) or "").strip()
functions = [node.name for node in tree.body if type(node) == ast.FunctionDef]
classes = [node.name for node in tree.body if type(node) == ast.ClassDef]
return docstring, functions, classes
def header():
'''
displays current module docstring
'''
f=inspect.stack()[1][1]
m=ast.parse(''.join(open(f)))
print "\n%s" % ast.get_docstring(m)
def _get_file_content(realpath):
"""Helper function to turn a file into HTML"""
result = """<h2>Module: %s</h2>
<div class="alert alert-info">
<div style = "details">path: %s</div>
<div style = "details">created: %s, last modified: %s</div></div>""" %(
os.path.basename(realpath),
urlparse.unquote(_convert_path_to_url(realpath)),
time.ctime(os.path.getmtime(realpath)),
time.ctime(os.path.getctime(realpath))
)
with open(realpath) as sourcefile:
code = sourcefile.readlines()
abstract_syntax_tree = ast.parse(''.join(code))
description = ast.get_docstring(abstract_syntax_tree)
if description:
result += "<h3>Summary:</h3> <div style = 'summary'>%s</div>" % _convert_str_to_html(description)
visitor = DocVisitor()
visitor.visit(abstract_syntax_tree)
parsed_code = visitor.get_doc()
entries = [ parsed_code[key] for key in sorted(parsed_code.keys())]
for entry in entries:
begin, end = entry.get("lines")
result += '<hr /><div><pre>' + "".join(code[begin:end]).rstrip().rstrip(":") +"</pre>"
result += _convert_str_to_html(entry.get("description"))+"</div>"
return result
def parse(self, source, document):
""" Parse ``source``, write results to ``document``.
"""
# This is lame, but seems to be required for python 2
source = CODING.sub("", source)
env = document.settings.env
filename = env.doc2path(env.docname) # e.g. full path to docs/user_guide/examples/layout_vertical
# This code splits the source into two parts: the docstring (or None if
# there is not one), and the remaining source code after
m = ast.parse(source)
docstring = ast.get_docstring(m)
if docstring is not None:
lines = source.split("\n")
lineno = m.body[0].lineno # assumes docstring is m.body[0]
source = "\n".join(lines[lineno:])
js_name = "bokeh-plot-%s.js" % hashlib.md5(env.docname.encode('utf-8')).hexdigest()
(script, js, js_path, source) = _process_script(source, filename, env.bokeh_plot_auxdir, js_name)
env.bokeh_plot_files[env.docname] = (script, js, js_path, source)
rst = PLOT_PAGE.render(source=source,
filename=basename(filename),
docstring=docstring,
script=script)
document['bokeh_plot_include_bokehjs'] = True
# can't use super, Sphinx Parser classes don't inherit object
Parser.parse(self, rst, document)
def _parse_class(symbol, with_docstrings):
docstring = {}
attr = {}
func = {}
name = symbol.name + "("
name += ", ".join([analyzer.expand_attribute(base) for base in symbol.bases])
name += ")"
for sym in symbol.body:
if sym.__class__ is ast.Assign:
result = _parse_assign(sym)
attr.update(result[0])
attr.update(result[1])
elif sym.__class__ is ast.FunctionDef:
result = _parse_function(sym, with_docstrings)
attr.update(result["attrs"])
if with_docstrings:
func[result["name"]] = result["lineno"]
docstring[result["lineno"]] = result["docstring"]
else:
func[result["name"]] = result["lineno"]
if with_docstrings:
docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)
lineno = symbol.lineno
for decorator in symbol.decorator_list:
lineno += 1
return {"name": name, "attributes": attr, "functions": func, "lineno": lineno, "docstring": docstring}
def _parse_class(symbol, with_docstrings):
docstring = {}
attr = {}
func = {}
name = symbol.name + '('
name += ', '.join([
analyzer.expand_attribute(base) for base in symbol.bases])
name += ')'
for sym in symbol.body:
if sym.__class__ is ast.Assign:
result = _parse_assign(sym)
attr.update(result[0])
attr.update(result[1])
elif sym.__class__ is ast.FunctionDef:
result = _parse_function(sym, with_docstrings)
attr.update(result['attrs'])
if with_docstrings:
func[result['name']] = result['lineno']
docstring[result['lineno']] = result['docstring']
else:
func[result['name']] = result['lineno']
if with_docstrings:
docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)
return {'name': name, 'attributes': attr, 'functions': func,
'lineno': symbol.lineno, 'docstring': docstring}
def shortdesc():
filename = os.path.join(PACKAGE, '__init__.py')
data = Setup.read(filename)
node = ast.parse(data, filename)
docstring = ast.get_docstring(node)
desc = docstring.strip().split('\n\n', 1)[0]
return desc.replace('\n', ' ')
def _parse_class(symbol, with_docstrings):
docstring = {}
attr = {}
func = {}
clazz = {}
name = symbol.name + '('
name += ', '.join([
analyzer.expand_attribute(base) for base in symbol.bases])
name += ')'
for sym in symbol.body:
if sym.__class__ is ast.Assign:
result = _parse_assign(sym)
attr.update(result[0])
attr.update(result[1])
elif sym.__class__ is ast.FunctionDef:
result = _parse_function(sym, with_docstrings)
attr.update(result['attrs'])
if with_docstrings:
docstring.update(result['docstring'])
func[result['name']] = (result['lineno'], result['functions'])
elif sym.__class__ is ast.ClassDef:
result = _parse_class(sym, with_docstrings)
clazz[result['name']] = (result['lineno'],
{'attributes': result['attributes'],
'functions': result['functions']})
docstring.update(result['docstring'])
if with_docstrings:
docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)
lineno = symbol.lineno
for decorator in symbol.decorator_list:
lineno += 1
return {'name': name, 'attributes': attr, 'functions': func,
'lineno': lineno, 'docstring': docstring, 'classes': clazz}
def __init__(self, node):
name = node.name
args = []
if node.args.args is not None:
for argument in node.args.args:
args.append(argument.arg)
for default, num_arg in zip(
reversed(node.args.defaults), reversed(range(len(args)))):
if type(default) is _ast.Str:
args[num_arg] += ' = "' + default.s + '"'
elif type(default) is _ast.NameConstant:
args[num_arg] += " = " + str(default.value)
elif type(default) is _ast.Num:
args[num_arg] += " = " + str(default.n)
elif type(default) is _ast.Name:
args[num_arg] += " = " + str(default.id)
if node.args.vararg is not None:
args.append("*" + node.args.vararg.arg)
if node.args.kwarg is not None:
args.append("**" + node.args.kwarg.arg)
doc = ast.get_docstring(node)
super(Function, self).__init__("Function", name, args, doc)
def extract_return(file):
for node in ast.walk(ast.parse(open(file).read())):
if (isinstance(node, ast.FunctionDef)):
print('Method name', node.name)
print('Doc string:', ast.get_docstring(node))
def handle_filters(collection, fullpath):
""" Grab each filter from a filter plugin file and
use the def comment if available
:param collection: The full collection name
:type collection: str
:param fullpath: The full path to the filter plugin file
:type fullpath: str
:return: A doc of filter plugins + descriptions
"""
plugins = {}
with open(fullpath) as fhand:
file_contents = fhand.read()
module = ast.parse(file_contents)
function_definitions = {
node.name: ast.get_docstring(node)
for node in module.body
if isinstance(node, ast.FunctionDef)
}
classdef = [
node
for node in module.body
if isinstance(node, ast.ClassDef) and node.name == "FilterModule"
]
if not classdef:
return plugins
filter_map = next(
(
node
for node in classdef[0].body
if isinstance(node, ast.Assign)
and hasattr(node, "targets")
and node.targets[0].id == "filter_map"
),
None,
)
if not filter_map:
filter_func = [
func
for func in classdef[0].body
if isinstance(func, ast.FunctionDef) and func.name == "filters"
]
if not filter_func:
return plugins
# The filter map is either looked up using the filter_map = {} assignment or if return returns a dict literal.
filter_map = next(
(
node
for node in filter_func[0].body
if isinstance(node, ast.Return) and isinstance(node.value, ast.Dict)
),
None,
)
if not filter_map:
return plugins
keys = [k.value for k in filter_map.value.keys]
logging.info("Adding filter plugins %s", ",".join(keys))
values = [k.id for k in filter_map.value.values]
filter_map = dict(zip(keys, values))
for name, func in filter_map.items():
if func in function_definitions:
comment = function_definitions[
func
] or "{collection} {name} filter plugin".format(
collection=collection, name=name
)
# Get the first line from the docstring for the description and make that the short description.
comment = next(
c for c in comment.splitlines() if c and not c.startswith(":")
)
plugins[
"{collection}.{name}".format(collection=collection, name=name)
] = comment
return plugins
python_file_list = []
module_data = {'deploy': {}, 'update': {}, 'monitor': {}, 'maintain': {}, 'other': {}}
for entry in scandir(categories_dictionary['python_code_path']):
if entry.path.endswith(".py"):
python_file_list.append(entry.path)
logging.info("Scanning Python files for docstrings and extracting them...")
script_tracker = {} # Used to track if a key has Python scripts, PowerShell scripts, or both
for module_path in python_file_list:
print("Processing " + module_path)
with open(module_path) as fd:
module_contents = fd.read()
module = ast.parse(module_contents)
docstring = ast.get_docstring(module)
if docstring is None:
docstring = ""
# Key is the name without py- ex: get_group_details
key = basename(module_path).replace('.py', '')
if key in categories_dictionary['deploy']:
category = 'deploy'
elif key in categories_dictionary['update']:
category = 'update'
elif key in categories_dictionary['monitor']:
category = 'monitor'
elif key in categories_dictionary['maintain']:
category = 'maintain'
else:
def _parse_function(symbol, with_docstrings):
docstring = {}
attrs = {}
func = {'functions': {}}
func_name = symbol.name + '('
# We store the arguments to compare with default backwards
defaults = []
for value in symbol.args.defaults:
# TODO: In some cases we can have something like: a=os.path
defaults.append(value)
arguments = []
for arg in reversed(symbol.args.args):
if not isinstance(arg, ast.Name) or arg.id == "self":
continue
argument = arg.id
if defaults:
value = defaults.pop()
arg_default = _map_type.get(value.__class__, None)
if arg_default is None:
if isinstance(value, ast.Attribute):
arg_default = model.expand_attribute(value)
elif isinstance(value, ast.Name):
arg_default = value.id
else:
arg_default = 'object'
argument += '=' + arg_default
arguments.append(argument)
func_name += ', '.join(reversed(arguments))
if symbol.args.vararg is not None:
if not func_name.endswith('('):
func_name += ', '
func_name += '*'
func_name += symbol.args.vararg.arg
if symbol.args.kwarg is not None:
if not func_name.endswith('('):
func_name += ', '
func_name += '**'
func_name += symbol.args.kwarg.arg
func_name += ')'
for sym in symbol.body:
if isinstance(sym, ast.Assign):
result = _parse_assign(sym)
attrs.update(result[1])
elif isinstance(sym, ast.FunctionDef):
result = _parse_function(sym, with_docstrings)
if with_docstrings:
docstring.update(result['docstring'])
func['functions'][result['name']] = {
'lineno': result['lineno'],
'functions': result['functions']
}
if with_docstrings:
docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)
lineno = symbol.lineno
for decorator in symbol.decorator_list:
lineno += 1
return {'name': func_name, 'lineno': lineno,
'attrs': attrs, 'docstring': docstring, 'functions': func}
def visit_Module(self, node):
ss = ast.get_docstring(node)
if ss:
self.docstring += ss + "\n"
self.generic_visit(node)
def extract_docstring(self, node: ast_fn_def) -> str:
"""Extract the docstring from a function"""
return ast.get_docstring(node) or ""
def fargv(default_switches, argv=None, use_enviromental_variables=True, return_type="SimpleNamespace", return_named_tuple=None,
spaces_are_equals=True, description=None):
"""Parse the argument list and create a dictionary with all parameters.
Argument types:
Strings: The most generic parameter type. If you need more specific data types, you run eval on a string type.
Integers: Anything that can be used to construct an int from a string.
Floating Point: Anything that can be used to construct a float from a string.
Booleans: If set with out a parameter, it is switched to True. Other wise a case incensitive value of true or
false
Choices: Defined as tuples in the parameter dictionary.
Positionals: Defined as sets in the parameter dictionary. They should not contain tabs and a string list will be
returned. This type is designed to work well with wildcards.
:param default_switches: A dictionary with parameters as keys and default values as elements. If the value is a
collection of two elements who's second element is a string.
:param argv: a list of strings which contains all parameters in the form '-PARAM_NAME=PARAM_VALUE'. This list will
be emptied of all switches and their values after processed, if the argv is needed full, pass a copy.
:param use_enviromental_variables: If set to True, before parsing argv elements to override the default settings,
the default settings are first overridden by any assigned environmental variable.
:param return_named_tuple: If set to True, result will be a named tuple instead of a dictionary.
:param spaces_are_equals: If set to True, a space bar is considered a valid separator if a parameter and its value.
:param description: A string describing the overall function of the script. If None (the default parameter) is passed,
the docstring of the calling file will be used if defined.
:return: Dictionary that is the same as the default values with updated values and the help string.
"""
if description is None:
caller_filename = inspect.getframeinfo(inspect.currentframe().f_back).filename # Assuming the caller is the main script file
description = ast.get_docstring(ast.parse(open(caller_filename, "r").read()))
if description is None:
description = ""
if description != "":
description = "\n" + description + "\n"
if return_named_tuple is not None:
sys.stderr.write("fargv.fargv: return return_named_tuple has been deprecated.")
if return_named_tuple == True:
return_type = "namedtuple"
else:
return_type = "dict"
assert return_type in ["SimpleNamespace", "dict", "namedtuple"]
str2type = {bool: lambda x: x.lower() in ["", "true"], # TODO(anguelos) replace lambda with a proper function
tuple: lambda x: x,
int: lambda x: int(x),
float: lambda x: float(x),
str: lambda x: x,
list: lambda x: eval(x),
set: lambda x: x.split("\t") # x will be stiched and than splited
}
if use_enviromental_variables:
for k, default_v in list(default_switches.items()):
if k in list(os.environ.keys()):
if hasattr(default_v, '__len__') and len(default_v) == 2 and isinstance(default_v[1], str):
default_switches[k] = (type(default_v[0])(os.environ[k]), default_v[1])
else:
default_switches[k] = type(default_v)(os.environ[k])
new_default_switches = {}
switches_help = {"help": "Print help and exit.",
"h": "Print help and exit",
"bash_autocomplete": "Print a set of bash commands that enable autocomplete for current program."}
# Removing help strings from arguments and placing them in stiches_help
for k, v in list(default_switches.items()):
if (not isinstance(v, str)) and not isinstance(v, set) and hasattr(v, '__len__') and len(v) == 2 and \
isinstance(v[1], str):
switches_help[k] = v[1]
new_default_switches[k] = v[0]
else:
switches_help[k] = ""
new_default_switches[k] = v
default_switches = new_default_switches
del new_default_switches
default_switches = dict(default_switches, **{"help": False, "bash_autocomplete": False, "h": False})
if argv is None:
argv = sys.argv
# Compiling positional switches and their values into tab separated strings
for switch_name in [k for k, v in default_switches.items() if isinstance(v, set)]:
arg_starts = [arg.split("=")[0] for arg in argv]
if f"-{switch_name}" in arg_starts:
param_list_start = arg_starts.index(f"-{switch_name}")
param_list_end = param_list_start + 1
while param_list_end < len(argv) and not argv[param_list_end].startswith("-"):
param_list_end += 1
if len(argv[param_list_start]) > len(switch_name)+1 and argv[param_list_start][len(switch_name)+1] == "=":
items = [argv[param_list_start][len(switch_name)+2]]
else:
items=[]
items += argv[param_list_start + 1: param_list_end]
packed_params = '\t'.join(items)
packed_params = f"-{switch_name}={packed_params}"
argv[param_list_start:param_list_end] = [packed_params]
argv_switches = dict(default_switches)
# Allowing pure switch behavior for bool and making = optional
for n in range(len(argv)):
if len(argv[n]) > 0 and argv[n][0] == "-":
key_str = argv[n][1:].split("=")[0]
expected_type = type(default_switches[key_str])
if "=" in argv[n]:
val_str = argv[n][argv[n].find("=") + 1:]
if expected_type is tuple:
if not val_str in default_switches[key_str]:
print(f"{val_str} should be one of {repr(default_switches[key_str])}", file=sys.stderr)
raise ValueError()
else:
if expected_type is bool:
argv[n] = "-" + key_str + "=true"
elif spaces_are_equals and n + 1 < len(argv) and argv[n + 1][0] != "-":
argv[n] = "-" + key_str + "=" + argv[n + 1]
argv[n + 1] = ""
if spaces_are_equals:
argv[:] = [arg for arg in argv if len(arg) > 0]
# setting the choice items (defined as tuples) to be the first item by default
for key, val in argv_switches.items():
if type(val) is tuple:
argv_switches[key] = default_switches[key][0]
argv_switches.update({arg[1:arg.find("=")]: arg[arg.find("=") + 1:] for arg in argv if arg[0] == "-"})
if spaces_are_equals:
positionals = [arg for arg in argv if len(arg) and arg[0] != "-"]
else:
positionals = [arg for arg in argv if arg[0] != "-"]
argv[:] = positionals
if set(argv_switches.keys()) > set(default_switches.keys()):
help_str = "\n" + argv[0] + description + " Syntax:\n\n"
for k in list(default_switches.keys()):
help_str += f"\t-{k} = {type(default_switches[k])} {switches_help[k]} Default {repr(default_switches[k])}.\n"
help_str += "\n\nUnrecognized switches: " + repr(tuple(set(default_switches.keys()) - set(argv_switches.keys())))
help_str += "\nAborting.\n"
sys.stderr.write(help_str)
sys.exit(1)
# Setting argv element to the value type of the default.
for k in argv_switches.keys():
if not isinstance(default_switches[k], type(argv_switches[k])):
argv_switches[k] = str2type[type(default_switches[k])](argv_switches[k])
help_str = "\n" + argv[0] + description + " Syntax:\n\n"
for k in list(default_switches.keys()):
help_str += "\t-%s=%s %s Default %s . Passed %s\n" % (
k, repr(type(default_switches[k])), switches_help[k], repr(default_switches[k]), repr(argv_switches[k]))
help_str += "\nAborting.\n"
# replace {blabla} with argv_switches["balbla"] values
# replacable_values = ["{" + k + "}" for k in list(argv_switches.keys())]
while len(re.findall("{[a-z0-9A-Z_]+}", "".join([v for v in list(argv_switches.values()) if isinstance(v, str)]))):
for k, v in list(argv_switches.items()):
if isinstance(v, str):
argv_switches[k] = v.format(**argv_switches)
if argv_switches["help"] or argv_switches["h"]:
sys.stderr.write(help_str)
sys.exit()
elif argv_switches["bash_autocomplete"]:
sys.stdout.write(generate_bash_autocomplete(default_switches, sys.argv[0]))
sys.exit()
else:
del argv_switches["h"]
del argv_switches["help"]
del argv_switches["bash_autocomplete"]
# Verifying choice given is part of allowed choices.
for key in default_switches.keys():
if isinstance(default_switches[key],tuple):
if argv_switches[key] not in default_switches[key]:
print(f"{key} must be one of [{' '.join([repr(v) for v in default_switches[key]])}], value given: {argv_switches[key]}",file=sys.stderr)
raise ValueError
if return_type.lower() == "namedtuple":
params = namedtuple("Parameters", argv_switches.keys())(*argv_switches.values())
elif return_type.lower() == "simplenamespace":
params = types.SimpleNamespace(**argv_switches)
elif return_type == "dict":
params = argv_switches
else:
raise ValueError
return params, help_str
def print_modules(self):
basepath = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
)
rst = {}
if self._format == "rst":
print(".. THIS DOCUMENT IS AUTO-GENERATED, DO NOT MODIFY")
print(".. To change this document, please update the docstrings in the individual modules")
for m in all_modules():
try:
module_type = "core"
filename = os.path.join(basepath, "modules", "core", "{}.py".format(m))
if not os.path.exists(filename):
filename = os.path.join(
basepath, "modules", "contrib", "{}.py".format(m)
)
module_type = "contrib"
if not os.path.exists(filename):
log.warning("module {} not found".format(m))
continue
doc = None
with open(filename) as f:
tree = ast.parse(f.read())
doc = ast.get_docstring(tree)
if not doc:
log.warning("failed to find docstring for {}".format(m))
continue
if self._format == "rst":
if os.path.exists(
os.path.join(basepath, "..", "screenshots", "{}.png".format(m))
):
doc = "{}\n\n.. image:: ../screenshots/{}.png".format(doc, m)
rst[module_type] = rst.get(module_type, [])
rst[module_type].append({"module": m, "content": doc})
else:
print(
textwrap.fill(
"{}:".format(m),
80,
initial_indent=self._indent * 2,
subsequent_indent=self._indent * 2,
)
)
for line in doc.split("\n"):
print(
textwrap.fill(
line,
80,
initial_indent=self._indent * 3,
subsequent_indent=self._indent * 6,
)
)
except Exception as e:
log.warning(e)
if self._format == "rst":
print("List of modules\n===============")
for k in ["core", "contrib"]:
print("\n{}\n{}\n".format(k, "-" * len(k)))
for mod in rst[k]:
print("\n{}\n{}\n".format(mod["module"], "~" * len(mod["module"])))
print(mod["content"])
def get_file_doc(path):
"""Load __doc__ from `.py` file"""
with open(path) as f:
module = ast.parse(f.read())
return ast.get_docstring(module)
def test_get_docstring(self):
tree = ast.parse("'docstring'\nx = 1")
self.assertEqual(ast.get_docstring(tree), 'docstring')
tree.body[0].value = ast.Constant(value='constant docstring')
self.assertEqual(ast.get_docstring(tree), 'constant docstring')
def get_module_docstring(path):
"""get a .py file docstring, without actually executing the file"""
with open(path) as f:
return ast.get_docstring(ast.parse(f.read()))
def _has_doc(node):
"""Return if node has docstrings."""
return (ast.get_docstring(node) is not None
and ast.get_docstring(node).strip() != "")
def find_avocado_tests(path):
"""
Attempts to find Avocado instrumented tests from Python source files
:param path: path to a Python source code file
:type path: str
:returns: tuple where first item is dict with class name and additional
info such as method names and tags; the second item is
set of class names which look like avocado tests but are
force-disabled.
:rtype: tuple
"""
module_name = 'avocado'
class_name = 'Test'
module = PythonModule(path, module_name, class_name)
# The resulting test classes
result = collections.OrderedDict()
disabled = set()
for klass in module.iter_classes():
docstring = ast.get_docstring(klass)
# Looking for a class that has in the docstring either
# ":avocado: enable" or ":avocado: disable
if check_docstring_directive(docstring, 'disable'):
disabled.add(klass.name)
continue
if check_docstring_directive(docstring, 'enable'):
info = get_methods_info(klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_requirements(
docstring))
result[klass.name] = info
continue
# From this point onwards we want to do recursive discovery, but
# for now we don't know whether it is avocado.Test inherited
# (Ifs are optimized for readability, not speed)
# If "recursive" tag is specified, it is forced as Avocado test
if check_docstring_directive(docstring, 'recursive'):
is_avocado = True
else:
is_avocado = module.is_matching_klass(klass)
info = get_methods_info(klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_requirements(
docstring))
_disabled = set()
# Getting the list of parents of the current class
parents = klass.bases
# Searching the parents in the same module
for parent in parents[:]:
# Looking for a 'class FooTest(Parent)'
if not isinstance(parent, ast.Name):
# 'class FooTest(bar.Bar)' not supported withing
# a module
continue
parent_class = parent.id
_info, _dis, _avocado = _examine_class(module.path, parent_class,
is_avocado, module_name,
class_name,
_determine_match_avocado)
if _info:
parents.remove(parent)
_extend_test_list(info, _info)
_disabled.update(_dis)
if _avocado is not is_avocado:
is_avocado = _avocado
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
if hasattr(parent, 'value'):
if hasattr(parent.value, 'id'):
# We know 'parent.Class' or 'asparent.Class' and need
# to get path and original_module_name. Class is given
# by parent definition.
_parent = module.imported_objects.get(parent.value.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path = os.path.dirname(_parent)
parent_module = os.path.basename(_parent)
parent_class = parent.attr
else:
# We don't support multi-level 'parent.parent.Class'
continue
else:
# We only know 'Class' or 'AsClass' and need to get
# path, module and original class_name
_parent = module.imported_objects.get(parent.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path, parent_module, parent_class = (
_parent.rsplit(os.path.sep, 2))
modules_paths = [parent_path,
os.path.dirname(module.path)] + sys.path
try:
_, found_ppath, _ = imp.find_module(parent_module, modules_paths)
except ImportError:
continue
_info, _dis, _avocado = _examine_class(found_ppath,
parent_class,
is_avocado,
module_name,
class_name,
_determine_match_avocado)
if _info:
info.extend(_info)
_disabled.update(_dis)
if _avocado is not is_avocado:
is_avocado = _avocado
# Only update the results if this was detected as 'avocado.Test'
if is_avocado:
result[klass.name] = info
disabled.update(_disabled)
return result, disabled
def visit_FunctionDef(self, node):
ss = ast.get_docstring(node)
if ss:
self.docstring += ss + "\n"
self.generic_visit(node)
def for_functions(self, c, heading, toc_level):
funcs = list(el(c, ast.FunctionDef))
property_operations = {}
property_docstrings = {}
result = ''
toc = []
if len(funcs) > 1:
func_heading = "#" * (toc_level + 1) + ' ' + heading + '\n\n'
result += func_heading
toc.append((heading, toc_level))
for f in funcs:
section_title = self.get_section_title(f.lineno)
if section_title:
result += '\n#### ' + section_title.upper()
toc.append((section_title, toc_level + 1))
if f.name.startswith('_'): continue
line_actual = f.lineno
descriptors = []
property_descriptor = False
while True:
stripped = self.lines[line_actual - 1].strip()
if stripped.startswith('@'):
line_actual += 1
if stripped == '@on_main_thread':
continue
if stripped == '@property':
property_operations[f.name] = ['get']
docstr = ast.get_docstring(f)
if docstr:
property_docstrings[f.name] = docstr
property_descriptor = True
elif stripped.endswith('.setter'):
property_operations[f.name].append('set')
property_descriptor = True
elif stripped.endswith('.deleter'):
property_operations[f.name].append('del')
property_descriptor = True
descriptors.append(stripped)
else:
break
if property_descriptor:
continue
descriptor_str = '`' + ', '.join(descriptors) + '`\n' if len(
descriptors) > 0 else ''
while not stripped.endswith(':'):
line_actual += 1
stripped += self.lines[line_actual - 1].strip()
result += ('\n#### `' + stripped[len('def '):-1] + '`\n' +
descriptor_str + '\n')
docstr = ast.get_docstring(f)
if docstr:
for line in iter(docstr.splitlines()):
result += ' ' + line + '\n'
if len(property_operations) > 0:
prop_heading = "#" * (toc_level + 1) + ' Properties\n\n'
result += prop_heading
toc.append(('Properties', toc_level))
for prop in property_operations:
result += '\n#### `' + prop + ' (' + ', '.join(
property_operations[prop]) + ')`\n\n'
docstr = property_docstrings.get(prop, None)
if docstr:
for line in iter(docstr.splitlines()):
result += ' ' + line + '\n'
return result, toc
def find_python_unittests(path):
"""
Attempts to find methods names from a given Python source file
This is a simpler, albeit more strict and correct, alternative version to
:func:`find_class_and_methods`, in the sense that it checks for the
(immediate) module name base class name.
:param path: path to a Python source code file
:type path: str
:returns: an ordered dictionary with classes as keys and methods as values
:rtype: collections.OrderedDict
"""
module_name = 'unittest'
class_name = 'TestCase'
module = PythonModule(path, module_name, class_name)
result = collections.OrderedDict()
for klass in module.iter_classes():
docstring = ast.get_docstring(klass)
parents = klass.bases
is_unittest = module.is_matching_klass(klass)
info = get_methods_info(klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_requirements(
docstring))
# Searching the parents in the same module
for parent in parents[:]:
# Looking for a 'class FooTest(Parent)'
if not isinstance(parent, ast.Name):
# 'class FooTest(bar.Bar)' not supported within a module
continue
parent_class = parent.id
_info, _dis, _is_unittest = _examine_class(module.path, parent_class,
is_unittest, module_name,
class_name,
_determine_match_unittest)
if _info:
parents.remove(parent)
_extend_test_list(info, _info)
if _is_unittest is not is_unittest:
is_unittest = _is_unittest
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
if hasattr(parent, 'value'):
if hasattr(parent.value, 'id'):
# We know 'parent.Class' or 'asparent.Class' and need
# to get path and original_module_name. Class is given
# by parent definition.
_parent = module.imported_objects.get(parent.value.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path = os.path.dirname(_parent)
parent_module = os.path.basename(_parent)
parent_class = parent.attr
else:
# We don't support multi-level 'parent.parent.Class'
continue
else:
# We only know 'Class' or 'AsClass' and need to get
# path, module and original class_name
_parent = module.imported_objects.get(parent.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path, parent_module, parent_class = (
_parent.rsplit(os.path.sep, 2))
modules_paths = [parent_path,
os.path.dirname(module.path)] + sys.path
try:
_, found_ppath, _ = imp.find_module(parent_module, modules_paths)
except ImportError:
continue
_info, _dis, _is_unittest = _examine_class(found_ppath,
parent_class,
is_unittest,
module_name,
class_name,
_determine_match_unittest)
if _info:
_extend_test_list(info, _info)
if _is_unittest is not is_unittest:
is_unittest = _is_unittest
# Only update the results if this was detected as 'unittest.TestCase'
if is_unittest:
result[klass.name] = info
return result
def _examine_class(target_module, target_class, determine_match, path,
class_name, match):
"""
Examine a class from a given path
:param target_module: the name of the module from which a class should
have come from. When attempting to find a Python
unittest, the target_module will most probably
be "unittest", as per the standard library module
name. When attempting to find Avocado tests, the
target_module will most probably be "avocado".
:type target_module: str
:param target_class: the name of the class that is considered to contain
test methods. When attempting to find Python
unittests, the target_class will most probably be
"TestCase". When attempting to find Avocado tests,
the target_class will most probably be "Test".
:type target_class: str
:param determine_match: a callable that will determine if a match has
occurred or not
:type determine_match: function
:param path: path to a Python source code file
:type path: str
:param class_name: the specific class to be found
:type path: str
:param match: whether the inheritance from <target_module.target_class> has
been determined or not
:type match: bool
:returns: tuple where first item is a list of test methods detected
for given class; second item is set of class names which
look like avocado tests but are force-disabled.
:rtype: tuple
"""
module = PythonModule(path, target_module, target_class)
info = []
disabled = set()
for klass in module.iter_classes(class_name):
if class_name != klass.name:
continue
docstring = ast.get_docstring(klass)
if match is False:
match = module.is_matching_klass(klass)
info = get_methods_info(
klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_dependencies(docstring),
)
# Getting the list of parents of the current class
parents = klass.bases
match = _examine_same_module(
parents,
info,
disabled,
match,
module,
target_module,
target_class,
determine_match,
)
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
try:
(
parent_class,
imported_symbol,
symbol_is_module,
) = _get_attributes_for_further_examination(parent, module)
found_spec = imported_symbol.get_importable_spec(
symbol_is_module)
if found_spec is None:
continue
except ClassNotSuitable:
continue
_info, _disabled, _match = _examine_class(
target_module,
target_class,
determine_match,
found_spec.origin,
parent_class,
match,
)
if _info:
_extend_test_list(info, _info)
disabled.update(_disabled)
if _match is not match:
match = _match
if not match and module.interesting_klass_found:
imported_symbol = module.imported_symbols[class_name]
if imported_symbol:
found_spec = imported_symbol.get_importable_spec()
if found_spec:
_info, _disabled, _match = _examine_class(
target_module,
target_class,
determine_match,
found_spec.origin,
class_name,
match,
)
if _info:
_extend_test_list(info, _info)
disabled.update(_disabled)
if _match is not match:
match = _match
return info, disabled, match
def _examine_class(path, class_name, match, target_module, target_class,
determine_match):
"""
Examine a class from a given path
:param path: path to a Python source code file
:type path: str
:param class_name: the specific class to be found
:type path: str
:param match: whether the inheritance from <target_module.target_class> has
been determined or not
:type match: bool
:param target_module: the module name under which the target_class lives
:type target_module: str
:param target_class: the name of the class that class_name should
ultimately inherit from
:type target_class: str
:param determine_match: a callable that will determine if a match has
occurred or not
:type determine_match: function
:returns: tuple where first item is a list of test methods detected
for given class; second item is set of class names which
look like avocado tests but are force-disabled.
:rtype: tuple
"""
module = PythonModule(path, target_module, target_class)
info = []
disabled = set()
for klass in module.iter_classes():
if class_name != klass.name:
continue
docstring = ast.get_docstring(klass)
if match is False:
match = determine_match(module, klass, docstring)
info = get_methods_info(klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_requirements(
docstring))
# Getting the list of parents of the current class
parents = klass.bases
# From this point we use `_$variable` to name temporary returns
# from method calls that are to-be-assigned/combined with the
# existing `$variable`.
# Searching the parents in the same module
for parent in parents[:]:
# Looking for a 'class FooTest(Parent)'
if not isinstance(parent, ast.Name):
# 'class FooTest(bar.Bar)' not supported withing
# a module
continue
parent_class = parent.id
_info, _disabled, _match = _examine_class(module.path, parent_class,
match, target_module,
target_class,
_determine_match_avocado)
if _info:
parents.remove(parent)
_extend_test_list(info, _info)
disabled.update(_disabled)
if _match is not match:
match = _match
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
if hasattr(parent, 'value'):
if hasattr(parent.value, 'id'):
# We know 'parent.Class' or 'asparent.Class' and need
# to get path and original_module_name. Class is given
# by parent definition.
_parent = module.imported_objects.get(parent.value.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path = os.path.dirname(_parent)
parent_module = os.path.basename(_parent)
parent_class = parent.attr
else:
# We don't support multi-level 'parent.parent.Class'
continue
else:
# We only know 'Class' or 'AsClass' and need to get
# path, module and original class_name
_parent = module.imported_objects.get(parent.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path, parent_module, parent_class = (
_parent.rsplit(os.path.sep, 2))
modules_paths = [parent_path,
os.path.dirname(module.path)] + sys.path
_, found_ppath, _ = imp.find_module(parent_module,
modules_paths)
_info, _disabled, _match = _examine_class(found_ppath,
parent_class,
match,
target_module,
target_class,
_determine_match_avocado)
if _info:
_extend_test_list(info, _info)
disabled.update(_disabled)
if _match is not match:
match = _match
return info, disabled, match
def _examine_class(path, class_name, is_avocado):
"""
Examine a class from a given path
:param path: path to a Python source code file
:type path: str
:param class_name: the specific class to be found
:type path: str
:returns: tuple where first item is a list of test methods detected
for given class; second item is set of class names which
look like avocado tests but are force-disabled.
:rtype: tuple
"""
module = AvocadoModule(path)
path = module.path # path might get updated (__init__.py)
ppath = os.path.dirname(path)
info = []
disabled = []
for klass in module.iter_classes():
if class_name != klass.name:
continue
docstring = ast.get_docstring(klass)
cl_tags = get_docstring_directives_tags(docstring)
# Only detect 'avocado.Test' if not yet decided
if is_avocado is False:
directives = get_docstring_directives(docstring)
if 'disable' in directives:
is_avocado = True
elif 'enable' in directives:
is_avocado = True
elif 'recursive' in directives:
is_avocado = True
if is_avocado is False: # Still not decided, try inheritance
is_avocado = module.is_avocado_test(klass)
info = get_methods_info(klass.body, cl_tags)
disabled = set()
# Getting the list of parents of the current class
parents = klass.bases
# From this point we use `_$variable` to name temporary returns
# from method calls that are to-be-assigned/combined with the
# existing `$variable`.
# Searching the parents in the same module
for parent in parents[:]:
# Looking for a 'class FooTest(Parent)'
if not isinstance(parent, ast.Name):
# 'class FooTest(bar.Bar)' not supported withing
# a module
continue
parent_class = parent.id
_info, _disabled, _avocado = _examine_class(
path, parent_class, is_avocado)
if _info:
parents.remove(parent)
info.extend(_info)
disabled.update(_disabled)
if _avocado is not is_avocado:
is_avocado = _avocado
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
if hasattr(parent, 'value'):
if hasattr(parent.value, 'id'):
# We know 'parent.Class' or 'asparent.Class' and need
# to get path and original_module_name. Class is given
# by parent definition.
_parent = module.imported_objects.get(parent.value.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path = os.path.dirname(_parent)
parent_module = os.path.basename(_parent)
parent_class = parent.attr
else:
# We don't support multi-level 'parent.parent.Class'
continue
else:
# We only know 'Class' or 'AsClass' and need to get
# path, module and original class_name
_parent = module.imported_objects.get(parent.id)
if _parent is None:
# We can't examine this parent (probably broken
# module)
continue
parent_path, parent_module, parent_class = (_parent.rsplit(
os.path.sep, 2))
modules_paths = [parent_path, ppath] + sys.path
_, found_ppath, _ = imp.find_module(parent_module, modules_paths)
_info, _dis, _avocado = _examine_class(found_ppath, parent_class,
is_avocado)
if _info:
info.extend(_info)
_disabled.update(_dis)
if _avocado is not is_avocado:
is_avocado = _avocado
return info, disabled, is_avocado
def find_python_tests(target_module, target_class, determine_match, path):
"""
Attempts to find Python tests from source files
A Python test in this context is a method within a specific type
of class (or that inherits from a specific class).
:param target_module: the name of the module from which a class should
have come from. When attempting to find a Python
unittest, the target_module will most probably
be "unittest", as per the standard library module
name. When attempting to find Avocado tests, the
target_module will most probably be "avocado".
:type target_module: str
:param target_class: the name of the class that is considered to contain
test methods. When attempting to find Python
unittests, the target_class will most probably be
"TestCase". When attempting to find Avocado tests,
the target_class will most probably be "Test".
:type target_class: str
:type determine_match: a callable that will determine if a given module
and class is contains valid Python tests
:type determine_match: function
:param path: path to a Python source code file
:type path: str
:returns: tuple where first item is dict with class name and additional
info such as method names and tags; the second item is
set of class names which look like Python tests but have been
forcefully disabled.
:rtype: tuple
"""
module = PythonModule(path, target_module, target_class)
# The resulting test classes
result = collections.OrderedDict()
disabled = set()
for klass in module.iter_classes():
docstring = ast.get_docstring(klass)
# Looking for a class that has in the docstring either
# ":avocado: enable" or ":avocado: disable
if check_docstring_directive(docstring, "disable"):
disabled.add(klass.name)
continue
if check_docstring_directive(docstring, "enable"):
info = get_methods_info(
klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_dependencies(docstring),
)
result[klass.name] = info
continue
# From this point onwards we want to do recursive discovery, but
# for now we don't know whether it is avocado.Test inherited
# (Ifs are optimized for readability, not speed)
# If "recursive" tag is specified, it is forced as test
if check_docstring_directive(docstring, "recursive"):
match = True
else:
match = module.is_matching_klass(klass)
info = get_methods_info(
klass.body,
get_docstring_directives_tags(docstring),
get_docstring_directives_dependencies(docstring),
)
# Getting the list of parents of the current class
parents = klass.bases
match = _examine_same_module(
parents,
info,
disabled,
match,
module,
target_module,
target_class,
determine_match,
)
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
try:
(
parent_class,
imported_symbol,
symbol_is_module,
) = _get_attributes_for_further_examination(parent, module)
found_spec = imported_symbol.get_importable_spec(
symbol_is_module)
if found_spec is None:
continue
except ClassNotSuitable:
continue
_info, _dis, _match = _examine_class(
target_module,
target_class,
determine_match,
found_spec.origin,
parent_class,
match,
)
if _info:
info.extend(_info)
disabled.update(_dis)
if _match is not match:
match = _match
# Only update the results if this was detected as 'avocado.Test'
if match:
result[klass.name] = info
return result, disabled
def _get_dag_file_docstring(fileloc: str) -> str:
with open(fileloc) as f:
file_contents = f.read()
module = ast.parse(file_contents)
docstring = ast.get_docstring(module)
return docstring
def get_long_description():
tree = get_ast_tree()
return ast.get_docstring(tree)
conf = ConfigParser()
conf.read(['setup.cfg'])
metadata = dict(conf.items('metadata'))
PACKAGENAME = metadata.get('package_name', 'packagename')
DESCRIPTION = metadata.get('description', 'Astropy affiliated package')
AUTHOR = metadata.get('author', '')
AUTHOR_EMAIL = metadata.get('author_email', '')
LICENSE = metadata.get('license', 'unknown')
URL = metadata.get('url', 'http://astropy.org')
# Get the long description from the package's docstring
_, module_path, _ = imp.find_module(PACKAGENAME)
with open(os.path.join(module_path, '__init__.py')) as f:
module_ast = ast.parse(f.read())
LONG_DESCRIPTION = ast.get_docstring(module_ast)
# Store the package name in a built-in variable so it's easy
# to get from other parts of the setup infrastructure
builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME
# VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
VERSION = '0.6.2.dev'
# Indicates if this version is a release version
RELEASE = 'dev' not in VERSION
if not RELEASE:
VERSION += get_git_devstr(False)
# Populate the dict of setup command overrides; this should be done before
def _find_avocado_tests(self, path, class_name=None):
"""
Attempts to find Avocado instrumented tests from Python source files
:param path: path to a Python source code file
:type path: str
:param class_name: the specific class to be found
:type path: str
:returns: tuple where first item is dict with class name and additional
info such as method names and tags; the second item is
set of class names which look like avocado tests but are
force-disabled.
:rtype: tuple
"""
# If only the Test class was imported from the avocado namespace
test_import = False
# The name used, in case of 'from avocado import Test as AvocadoTest'
test_import_name = None
# If the "avocado" module itself was imported
mod_import = False
# The name used, in case of 'import avocado as avocadolib'
mod_import_name = None
# The resulting test classes
result = collections.OrderedDict()
disabled = set()
if os.path.isdir(path):
path = os.path.join(path, "__init__.py")
with open(path) as source_file:
mod = ast.parse(source_file.read(), path)
for statement in mod.body:
# Looking for a 'from avocado import Test'
if (isinstance(statement, ast.ImportFrom) and
statement.module == 'avocado'):
for name in statement.names:
if name.name == 'Test':
test_import = True
if name.asname is not None:
test_import_name = name.asname
else:
test_import_name = name.name
break
# Looking for a 'import avocado'
elif isinstance(statement, ast.Import):
for name in statement.names:
if name.name == 'avocado':
mod_import = True
if name.asname is not None:
mod_import_name = name.nasname
else:
mod_import_name = name.name
# Looking for a 'class Anything(anything):'
elif isinstance(statement, ast.ClassDef):
# class_name will exist only under recursion. In that
# case, we will only process the class if it has the
# expected class_name.
if class_name is not None and class_name != statement.name:
continue
docstring = ast.get_docstring(statement)
# Looking for a class that has in the docstring either
# ":avocado: enable" or ":avocado: disable
has_disable = safeloader.check_docstring_directive(docstring,
'disable')
if (has_disable and class_name is None):
disabled.add(statement.name)
continue
cl_tags = safeloader.get_docstring_directives_tags(docstring)
has_enable = safeloader.check_docstring_directive(docstring,
'enable')
if (has_enable and class_name is None):
info = self._get_methods_info(statement.body, cl_tags)
result[statement.name] = info
continue
# Looking for the 'recursive' docstring or a 'class_name'
# (meaning we are under recursion)
has_recurse = safeloader.check_docstring_directive(docstring,
'recursive')
if (has_recurse or class_name is not None):
info = self._get_methods_info(statement.body, cl_tags)
result[statement.name] = info
# Getting the list of parents of the current class
parents = statement.bases
# Searching the parents in the same module
for parent in parents[:]:
# Looking for a 'class FooTest(module.Parent)'
if isinstance(parent, ast.Attribute):
parent_class = parent.attr
# Looking for a 'class FooTest(Parent)'
else:
parent_class = parent.id
res, dis = self._find_avocado_tests(path, parent_class)
if res:
parents.remove(parent)
for cls in res:
info.extend(res[cls])
disabled.update(dis)
# If there are parents left to be discovered, they
# might be in a different module.
for parent in parents:
if isinstance(parent, ast.Attribute):
# Looking for a 'class FooTest(module.Parent)'
parent_module = parent.value.id
parent_class = parent.attr
else:
# Looking for a 'class FooTest(Parent)'
parent_module = None
parent_class = parent.id
for node in mod.body:
reference = None
# Looking for 'from parent import class'
if isinstance(node, ast.ImportFrom):
reference = parent_class
# Looking for 'import parent'
elif isinstance(node, ast.Import):
reference = parent_module
if reference is None:
continue
for artifact in node.names:
# Looking for a class alias
# ('from parent import class as alias')
if artifact.asname is not None:
parent_class = reference = artifact.name
# If the parent class or the parent module
# is found in the imports, discover the
# parent module path and find the parent
# class there
if artifact.name == reference:
modules_paths = [os.path.dirname(path)]
modules_paths.extend(sys.path)
if parent_module is None:
parent_module = node.module
_, ppath, _ = imp.find_module(parent_module,
modules_paths)
res, dis = self._find_avocado_tests(ppath,
parent_class)
if res:
for cls in res:
info.extend(res[cls])
disabled.update(dis)
continue
if test_import:
base_ids = [base.id for base in statement.bases
if hasattr(base, 'id')]
# Looking for a 'class FooTest(Test):'
if test_import_name in base_ids:
info = self._get_methods_info(statement.body,
cl_tags)
result[statement.name] = info
continue
# Looking for a 'class FooTest(avocado.Test):'
if mod_import:
for base in statement.bases:
module = base.value.id
klass = base.attr
if module == mod_import_name and klass == 'Test':
info = self._get_methods_info(statement.body,
cl_tags)
result[statement.name] = info
continue
return result, disabled
def test_get_docstring(self):
node = ast.parse('def foo():\n """line one\n line two"""')
self.assertEqual(ast.get_docstring(node.body[0]),
'line one\nline two')
def _get_docstring(fun): # type: (ast.AST) -> str
return ast.get_docstring(fun)
def func_doc(self):
if self.func is not None:
return self.func.__doc__
else:
return ast_module.get_docstring(self.ast)
def _find_avocado_tests(self, path):
"""
Attempts to find Avocado instrumented tests from Python source files
:param path: path to a Python source code file
:type path: str
:returns: dictionary with class name and method names
:rtype: dict
"""
# If only the Test class was imported from the avocado namespace
test_import = False
# The name used, in case of 'from avocado import Test as AvocadoTest'
test_import_name = None
# If the "avocado" module itself was imported
mod_import = False
# The name used, in case of 'import avocado as avocadolib'
mod_import_name = None
# The resulting test classes
result = {}
mod = ast.parse(open(path).read(), path)
for statement in mod.body:
# Looking for a 'from avocado import Test'
if (isinstance(statement, ast.ImportFrom)
and statement.module == 'avocado'):
for name in statement.names:
if name.name == 'Test':
test_import = True
if name.asname is not None:
test_import_name = name.asname
else:
test_import_name = name.name
break
# Looking for a 'import avocado'
elif isinstance(statement, ast.Import):
for name in statement.names:
if name.name == 'avocado':
mod_import = True
if name.asname is not None:
mod_import_name = name.nasname
else:
mod_import_name = name.name
# Looking for a 'class Anything(anything):'
elif isinstance(statement, ast.ClassDef):
docstring = ast.get_docstring(statement)
# Looking for a class that has in the docstring either
# ":avocado: enable" or ":avocado: disable
if safeloader.is_docstring_tag_disable(docstring):
continue
elif safeloader.is_docstring_tag_enable(docstring):
functions = [
st.name for st in statement.body
if isinstance(st, ast.FunctionDef)
and st.name.startswith('test')
]
functions = data_structures.ordered_list_unique(functions)
result[statement.name] = functions
continue
if test_import:
base_ids = [
base.id for base in statement.bases
if hasattr(base, 'id')
]
# Looking for a 'class FooTest(Test):'
if test_import_name in base_ids:
functions = [
st.name for st in statement.body
if isinstance(st, ast.FunctionDef)
and st.name.startswith('test')
]
functions = data_structures.ordered_list_unique(
functions)
result[statement.name] = functions
continue
# Looking for a 'class FooTest(avocado.Test):'
if mod_import:
for base in statement.bases:
module = base.value.id
klass = base.attr
if module == mod_import_name and klass == 'Test':
functions = [
st.name for st in statement.body
if isinstance(st, ast.FunctionDef)
and st.name.startswith('test')
]
functions = data_structures.ordered_list_unique(
functions)
result[statement.name] = functions
return result
def module_from_ast(module_name, filename, t):
code = code_for_module(module_name, filename, t)
module = types.ModuleType(module_name, ast.get_docstring(t))
exec(code, module.__dict__)
return module
from .core import driver_not_found as gpu_stump # noqa: F401
import ast
import pathlib
gpu_stump.__doc__ = ""
filepath = pathlib.Path(__file__).parent / "gpu_stump.py"
file_contents = ""
with open(filepath, encoding="utf8") as f:
file_contents = f.read()
module = ast.parse(file_contents)
function_definitions = [
node for node in module.body if isinstance(node, ast.FunctionDef)
]
for fd in function_definitions:
if fd.name == "gpu_stump":
gpu_stump.__doc__ = ast.get_docstring(fd)
try:
_dist = get_distribution("stumpy")
# Normalize case for Windows systems
dist_loc = os.path.normcase(_dist.location)
here = os.path.normcase(__file__)
if not here.startswith(os.path.join(dist_loc, "stumpy")):
# not installed, but there is another version that *is*
raise DistributionNotFound # pragma: no cover
except DistributionNotFound: # pragma: no cover
__version__ = "Please install this project with setup.py"
else: # pragma: no cover
__version__ = _dist.version