def collect_docstrings(self): """Collect list of paths + classes + methods/functions, and docstrings. This function assumes that there are no classes or methods/functions nested within others. """ self.node_sequence = [] # Module-level docstrings. self.node_sequence.append([self.path, ast.get_docstring(self.module)]) # Class-level doc-strings # Function-level doc-strings for class_def in self.class_defs: for node in class_def.body: if isinstance(node, ast.ClassDef): self.node_sequence.append( [self.path + '.' + node.name, ast.get_docstring(node)] ) elif isinstance(node, ast.FunctionDef): if self.tests_only and node.name[:5] != 'test_': continue self.node_sequence.append( [self.path + '.' + class_def.name + '.' + node.name, ast.get_docstring(node)] ) for func_def in self.func_defs: if isinstance(func_def, ast.FunctionDef): if self.tests_only and func_def.name[:5] != 'test_': continue self.node_sequence.append( [self.path + '.' + func_def.name, ast.get_docstring(func_def)] )
def test_get_docstring(self): node = ast.parse('def foo():\n """line one\n line two"""') self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham')
def _parse_docstring(self): """Parses the test docstring extracting expected values. If the expected tags is not spelled right they will not be parsed. """ if self.docstring is None: return # Create the contexts tags, unexpected_tags, _ = self._parse_tags( ast.get_docstring(self.module_def)) class_tags, class_unexpected_tags, _ = self._parse_tags( ast.get_docstring(self.parent_class_def)) function_tags, function_unexpected_tags, self.skipped_lines = ( self._parse_tags(self.docstring)) # Update context dictionaries tags.update(class_tags) tags.update(function_tags) unexpected_tags.update(class_unexpected_tags) unexpected_tags.update(function_unexpected_tags) for tag, value in tags.items(): if tag == 'bz': tag = 'bugs' if tag == 'assert': tag = 'assertion' if tag == 'type': tag = 'test_type' setattr(self, tag, value) self.unexpected_tags = unexpected_tags # Always use the first line of docstring as test case name if self.test is None: self.test = self.docstring.strip().split('\n')[0]
def parse_module(file_path): module = ast.parse(file_path.read_text()) tmpl_str = HEADER.format(file_path.name.rsplit('.',1)[0]) cls_defs = [node for node in module.body if isinstance(node, ast.ClassDef)] mod_func_defs = [node for node in module.body if isinstance(node, ast.FunctionDef)] for cls_def in cls_defs: cls_name = cls_def.name cls_bases = ','.join([parse(each) for each in cls_def.bases]) tmpl_str += f'== {{{{class {cls_name}{":" + cls_bases if cls_bases else ""}}}}}\n\n' method_str = None for fn_def in (fn_def for fn_def in cls_def.body if isinstance(fn_def, ast.FunctionDef)): if fn_def.name == '__init__': tmpl_str += "=== Arguments\n" + parse_args(fn_def.args) + "\n\n" else: if not method_str: method_str = '=== Methods\n\n' doc_str = ast.get_docstring(fn_def) method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n' tmpl_str += method_str if method_str else '' method_str = None for fn_def in mod_func_defs: if not method_str: method_str = '== Module Functions\n\n' doc_str = ast.get_docstring(fn_def) method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n' tmpl_str += method_str if method_str else '' return tmpl_str
def extract_info(self): for node in self.module.body: if isinstance(node, ast.ClassDef): yield { "name": node.name, "lineno": node.lineno, "docstring": ast.get_docstring(node), "type": 'class', } for sub_node in node.body: if isinstance(sub_node, ast.FunctionDef): yield { "name": sub_node.name, "lineno": sub_node.lineno, "docstring": ast.get_docstring(sub_node), "type": 'attribute', "args": [arg.id for arg in sub_node.args.args], "header": '' } elif isinstance(node, ast.FunctionDef): yield { "name": node.name, "lineno": node.lineno, "docstring": ast.get_docstring(node), "type": 'function', "args": [arg.id for arg in node.args.args], }
def get_doc(self, module): for node in module.body: if isinstance(node, ast.ClassDef): yield ast.get_docstring(node) for sub_node in node.body: if isinstance(sub_node, ast.FunctionDef): yield ast.get_docstring(sub_node)
def __init__(self, function_def, parent_class=None, testmodule=None): """Wrap a ``ast.FunctionDef`` instance used to extract information.""" self.docstring = ast.get_docstring(function_def) self.function_def = function_def self.name = function_def.name if parent_class: self.parent_class = parent_class.name self.parent_class_def = parent_class self.class_docstring = ast.get_docstring(self.parent_class_def) else: self.parent_class = None self.parent_class_def = None self.class_docstring = None self.testmodule = testmodule.path self.module_def = testmodule self.module_docstring = ast.get_docstring(self.module_def) self.pkginit = os.path.join( os.path.dirname(self.testmodule), '__init__.py') if os.path.exists(self.pkginit): self.pkginit_def = ast.parse(''.join(open(self.pkginit))) self.pkginit_docstring = ast.get_docstring(self.pkginit_def) else: self.pkginit_def = None self.pkginit_docstring = None self.tokens = {} self.invalid_tokens = {} self._rst_parser_messages = [] self.parser = DocstringParser( SETTINGS.get('tokens'), SETTINGS.get('minimum_tokens'), ) self._parse_docstring()
def _getMarvinTestDocStrings(classname, testnames, marvinCodePath): pathToClass = os.path.join(marvinCodePath, *classname.split('.')[1:-1])+'.py' astData = ast.parse(open(pathToClass).read()) classElement = filter(lambda x:isinstance(x, ast.ClassDef) and x.name == classname.split('.')[-1], astData.body)[0] classDocString = ast.get_docstring(classElement) classDocString = classDocString and classDocString.rstrip() or '' testMethodElements = filter(lambda x:isinstance(x, ast.FunctionDef) and x.name in testnames, classElement.body) testMethodDocStrings = [] for testMethod in testMethodElements: docStr = ast.get_docstring(testMethod) docStr = docStr and docStr.rstrip() or '' testMethodDocStrings.append((testMethod.name, docStr)) return (classDocString, testMethodDocStrings)
def _extract_default_fits_file(self, path): with open(path, 'r') as rfile: m = ast.parse(rfile.read()) docstr = ast.get_docstring(m) yd = yaml.load(docstr) if yd: return yd.get('default_fits', None)
def _parse_function(symbol, with_docstrings): docstring = {} attrs = {} func = {'functions': {}} func_name = symbol.name + '(' #We store the arguments to compare with default backwards defaults = [] for value in symbol.args.defaults: #TODO: In some cases we can have something like: a=os.path defaults.append(value) arguments = [] for arg in reversed(symbol.args.args): if arg.__class__ is not _ast.Name or arg.id == 'self': continue argument = arg.id if defaults: value = defaults.pop() arg_default = _map_type.get(value.__class__, None) if arg_default is None: if value.__class__ is _ast.Attribute: arg_default = analyzer.expand_attribute(value) elif value.__class__ is _ast.Name: arg_default = value.id else: arg_default = 'object' argument += '=' + arg_default arguments.append(argument) func_name += ', '.join(reversed(arguments)) if symbol.args.vararg is not None: if not func_name.endswith('('): func_name += ', ' func_name += '*' + symbol.args.vararg if symbol.args.kwarg is not None: if not func_name.endswith('('): func_name += ', ' func_name += '**' + symbol.args.kwarg func_name += ')' for sym in symbol.body: if sym.__class__ is ast.Assign: result = _parse_assign(sym) attrs.update(result[1]) elif sym.__class__ is ast.FunctionDef: result = _parse_function(sym, with_docstrings) if with_docstrings: docstring.update(result['docstring']) func['functions'][result['name']] = {'lineno': result['lineno'], 'functions': result['functions']} if with_docstrings: docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True) lineno = symbol.lineno for decorator in symbol.decorator_list: lineno += 1 return {'name': func_name, 'lineno': lineno, 'attrs': attrs, 'docstring': docstring, 'functions': func}
def parse_tree (self, name, node): """ Recursive function that explores the python parse tree and constructs our TreeNode datastructure to represent a document. Takes the root node of the tree provided by the std ast module and the file name. """ logging.debug('Exploring node %s of %s' % (node, name)) tree_node = ParserNode( type = type_lookup[type(node)], name = node.name if 'name' in node.__dict__ else \ name.split('.')[0], docstring = ast.get_docstring(node), children = [], terms = [] ) for child in node.body: # We only want to look for modules, functions and classes as # these are the only items that have doc strings and therefore # terms at the moment. Later on, we'll want to use imports and # function names and other elements of the parse tree. if type(child) not in [ast.Module, ast.FunctionDef, ast.ClassDef]: continue tree_node.children.append(self.parse_tree(name, child)) return tree_node
def visit_FunctionDef(node): """ https://docs.python.org/2/library/ast.html#ast.NodeVisitor.visit """ if node.name != self.controller_method_name: return doc = ast.get_docstring(node) raise StopIteration(doc if doc else u"")
def crop_to_docstring(strlines, outputLineNums=False, includeMarks=False): '''Gets the first found docstring from the input code. .. NOTE:: If the input contains the outer quotes ""/'', then the output will, too :TODO: Make an optional output for line nums, which will be necessary for Style Guide classes later on :TODO: Force the output of the cropped docstring to include its original '' Args: strlines (str or iter): The snippet of code that contains the docstring Returns: str : The full docstring ''' try: iter(strlines) if not isinstance(strlines, six.string_types): raise TypeError except TypeError: strlines = '\n'.join(strlines) module = ast.parse(strlines) defs = [node for node in module.body if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Module))] for funcDef in defs: return ast.get_docstring(funcDef)
def get_init_code(tree, table): """Get one-lined code from `tree` and `table. Calculate the helper variables that we will need, and wrap the output code (computed from `tree`) with definitions of those variables. TODO: Short-circuit to something far simpler if the program has only one print statement. Arguments: tree: Python AST node table: symtable.symtable Returns: string - valid one-line code """ output = Namespace(table).many_to_one(tree.body) doc = ast.get_docstring(tree, clean=False) if doc is not None: output = assignment_component(output, T("{__g}['__doc__']"), repr(doc)) output = provide( output.format(__l=T('{__g}')), __print=T("__import__('__builtin__').__dict__['print']"), __exec="__import__('trace').Trace(count=False," " trace=False).runctx", __y="(lambda f: (lambda x: x(x))(lambda y:" " f(lambda: y(y)())))", __g=T("globals()"), __contextlib="__import__('contextlib')", __sys="__import__('sys')", __types="__import__('types')") return output.close()
def find_info(*path_parts): finder = VersionFinder() node = ast.parse(read(*path_parts)) finder.visit(node) info = finder.data info['docstring'] = ast.get_docstring(node) return info
def scripts2rst(path, f): """ creates rst summary documentation for files in scripts folder which is not a package so cannot be imported """ # get list of script files try: files = [name for name in os.listdir(path) if not name.startswith("_") and name.endswith(".py")] except: return f.write("**Scripts**\n\n") # loop over script files for name in files: sfile = os.path.join(path, name) try: try: source = ast.parse(open(sfile, "r", encoding="utf8").read()) except: #py2 fails if encoding in string source = ast.parse(open(sfile, "r").read()) except Exception as e: log.warning("Problem parsing %s\n%s"%(name, e)) f.write(name+"\n") doc = i.cleandoc(ast.get_docstring(source)) or "." doc = py2decode(doc) #py2 f.write(" "+doc.splitlines()[0]+"\n") f.write("\n")
def main(): with open(FILENAME) as fd: file_contents = fd.read() module = ast.parse(file_contents) print('loaded file') function_definitions = [node for node in module.body if isinstance(node, ast.FunctionDef)] for f in function_definitions: docstring = ast.get_docstring(f) params = {} returntype = "" for line in docstring.splitlines(): # if ":param" in line: # param = line.split()[2] # params[param] = "Any" if ":type" in line: # this line is a type tokens = line.split() param = tokens[1][:-1] ty = tokens[2] # print ("param is " + param + " with type " + ty) params[param] = ty string = "def {0}({1})".format( f.name, ", ".join( ["%s: %s" % (param, params[param]) for param in params])) ''' string = "def " + f.name + "(" for param in params: string += "" + param + ": " + params[param] + ", " string += "):" ''' print string
def visit_FunctionDef(self, tree): # self.visit() returns something of the form # ('lambda x, y, z=5, *args: ', ['x', 'y', 'z', 'args']) args, arg_names = self.visit(tree.args) decoration = T('{}') for decorator in tree.decorator_list: decoration = decoration.format(T('{}({})').format(self.visit(decorator), T('{}'))) ns = self.next_child() body = ns.many_to_one(tree.body).format(pre_return='', post_return='') if arg_names: body = assignment_component(body, T(', ').join(ns.var(name) for name in arg_names), T(', ').join(arg_names)) body = self.close(ns, '{}', body) function_code = args + body doc = ast.get_docstring(tree, clean=False) if tree.decorator_list: return assignment_component( T('{after}'), self.store_var(tree.name), decoration.format(assignment_component( '__func', '__func, __func.__name__' + ('' if doc is None else ', __func.__doc__'), T('{}, {!r}' + ('' if doc is None else ', {!r}')).format( function_code, tree.name, doc)))) else: return assignment_component( T('{after}'), T('{}, {}.__name__' + ('' if doc is None else ', {}.__doc__')).format( self.store_var(tree.name), self.var(tree.name), self.var(tree.name)), T('{}, {!r}' + ('' if doc is None else ', {!r}')).format( function_code, tree.name, doc))
def ast(self, value): #@DuplicatedSignature if not isinstance(value, _ast.FunctionDef): raise Error("Root of ast must be a FunctionDef, but got a %s." % value.__class__.__name__) self._ast = value self.__name__ = value.name self.__doc__ = _ast.get_docstring(value, clean=False)
def visit_FunctionDef(self, node, **kwargs): """ Handles function definitions within code. Process a function's docstring, keeping well aware of the function's context and whether or not it's part of an interface definition. """ if self.options.debug: stderr.write("# Function {0.name}{1}".format(node, linesep)) # Push either 'interface' or 'class' onto our containing nodes # hierarchy so we can keep track of context. This will let us tell # if a function is nested within another function or even if a class # is nested within a function. containingNodes = kwargs.get('containingNodes', []) or [] containingNodes.append((node.name, 'function')) if self.options.topLevelNamespace: fullPathNamespace = self._getFullPathName(containingNodes) contextTag = '.'.join(pathTuple[0] for pathTuple in fullPathNamespace) modifiedContextTag = self._processMembers(node, contextTag) tail = '@namespace {0}'.format(modifiedContextTag) else: tail = self._processMembers(node, '') if get_docstring(node): self._processDocstring(node, tail, containingNodes=containingNodes) # Visit any contained nodes. self.generic_visit(node, containingNodes=containingNodes) # Remove the item we pushed onto the containing nodes hierarchy. containingNodes.pop()
def _get_service_classes(self, node, path): ret = [] classes = [c for c in node.body if isinstance(c, ast.ClassDef)] for cls in classes: exposed = False bases = [] for cls_base in cls.bases: if cls_base.id in ['StorageService', 'ComputationalService']: exposed = True bases.append(cls_base.id) if not exposed: continue ret.append({ 'defs': self._get_exposed_funcs(cls, path, method=True), 'name': cls.name, 'doc': ast.get_docstring(cls, clean=True), 'bases': bases, 'path': path }) return ret
def _parse_py_file(py_file): tree = ast.parse("".join(open(py_file))) # noinspection PyArgumentEqualDefault docstring = (ast.get_docstring(tree, clean=True) or "").strip() functions = [node.name for node in tree.body if type(node) == ast.FunctionDef] classes = [node.name for node in tree.body if type(node) == ast.ClassDef] return docstring, functions, classes
def header(): ''' displays current module docstring ''' f=inspect.stack()[1][1] m=ast.parse(''.join(open(f))) print "\n%s" % ast.get_docstring(m)
def _get_file_content(realpath): """Helper function to turn a file into HTML""" result = """<h2>Module: %s</h2> <div class="alert alert-info"> <div style = "details">path: %s</div> <div style = "details">created: %s, last modified: %s</div></div>""" %( os.path.basename(realpath), urlparse.unquote(_convert_path_to_url(realpath)), time.ctime(os.path.getmtime(realpath)), time.ctime(os.path.getctime(realpath)) ) with open(realpath) as sourcefile: code = sourcefile.readlines() abstract_syntax_tree = ast.parse(''.join(code)) description = ast.get_docstring(abstract_syntax_tree) if description: result += "<h3>Summary:</h3> <div style = 'summary'>%s</div>" % _convert_str_to_html(description) visitor = DocVisitor() visitor.visit(abstract_syntax_tree) parsed_code = visitor.get_doc() entries = [ parsed_code[key] for key in sorted(parsed_code.keys())] for entry in entries: begin, end = entry.get("lines") result += '<hr /><div><pre>' + "".join(code[begin:end]).rstrip().rstrip(":") +"</pre>" result += _convert_str_to_html(entry.get("description"))+"</div>" return result
def parse(self, source, document): """ Parse ``source``, write results to ``document``. """ # This is lame, but seems to be required for python 2 source = CODING.sub("", source) env = document.settings.env filename = env.doc2path(env.docname) # e.g. full path to docs/user_guide/examples/layout_vertical # This code splits the source into two parts: the docstring (or None if # there is not one), and the remaining source code after m = ast.parse(source) docstring = ast.get_docstring(m) if docstring is not None: lines = source.split("\n") lineno = m.body[0].lineno # assumes docstring is m.body[0] source = "\n".join(lines[lineno:]) js_name = "bokeh-plot-%s.js" % hashlib.md5(env.docname.encode('utf-8')).hexdigest() (script, js, js_path, source) = _process_script(source, filename, env.bokeh_plot_auxdir, js_name) env.bokeh_plot_files[env.docname] = (script, js, js_path, source) rst = PLOT_PAGE.render(source=source, filename=basename(filename), docstring=docstring, script=script) document['bokeh_plot_include_bokehjs'] = True # can't use super, Sphinx Parser classes don't inherit object Parser.parse(self, rst, document)
def _parse_class(symbol, with_docstrings): docstring = {} attr = {} func = {} name = symbol.name + "(" name += ", ".join([analyzer.expand_attribute(base) for base in symbol.bases]) name += ")" for sym in symbol.body: if sym.__class__ is ast.Assign: result = _parse_assign(sym) attr.update(result[0]) attr.update(result[1]) elif sym.__class__ is ast.FunctionDef: result = _parse_function(sym, with_docstrings) attr.update(result["attrs"]) if with_docstrings: func[result["name"]] = result["lineno"] docstring[result["lineno"]] = result["docstring"] else: func[result["name"]] = result["lineno"] if with_docstrings: docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True) lineno = symbol.lineno for decorator in symbol.decorator_list: lineno += 1 return {"name": name, "attributes": attr, "functions": func, "lineno": lineno, "docstring": docstring}
def _parse_class(symbol, with_docstrings): docstring = {} attr = {} func = {} name = symbol.name + '(' name += ', '.join([ analyzer.expand_attribute(base) for base in symbol.bases]) name += ')' for sym in symbol.body: if sym.__class__ is ast.Assign: result = _parse_assign(sym) attr.update(result[0]) attr.update(result[1]) elif sym.__class__ is ast.FunctionDef: result = _parse_function(sym, with_docstrings) attr.update(result['attrs']) if with_docstrings: func[result['name']] = result['lineno'] docstring[result['lineno']] = result['docstring'] else: func[result['name']] = result['lineno'] if with_docstrings: docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True) return {'name': name, 'attributes': attr, 'functions': func, 'lineno': symbol.lineno, 'docstring': docstring}
def shortdesc(): filename = os.path.join(PACKAGE, '__init__.py') data = Setup.read(filename) node = ast.parse(data, filename) docstring = ast.get_docstring(node) desc = docstring.strip().split('\n\n', 1)[0] return desc.replace('\n', ' ')
def _parse_class(symbol, with_docstrings): docstring = {} attr = {} func = {} clazz = {} name = symbol.name + '(' name += ', '.join([ analyzer.expand_attribute(base) for base in symbol.bases]) name += ')' for sym in symbol.body: if sym.__class__ is ast.Assign: result = _parse_assign(sym) attr.update(result[0]) attr.update(result[1]) elif sym.__class__ is ast.FunctionDef: result = _parse_function(sym, with_docstrings) attr.update(result['attrs']) if with_docstrings: docstring.update(result['docstring']) func[result['name']] = (result['lineno'], result['functions']) elif sym.__class__ is ast.ClassDef: result = _parse_class(sym, with_docstrings) clazz[result['name']] = (result['lineno'], {'attributes': result['attributes'], 'functions': result['functions']}) docstring.update(result['docstring']) if with_docstrings: docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True) lineno = symbol.lineno for decorator in symbol.decorator_list: lineno += 1 return {'name': name, 'attributes': attr, 'functions': func, 'lineno': lineno, 'docstring': docstring, 'classes': clazz}
def __init__(self, node): name = node.name args = [] if node.args.args is not None: for argument in node.args.args: args.append(argument.arg) for default, num_arg in zip( reversed(node.args.defaults), reversed(range(len(args)))): if type(default) is _ast.Str: args[num_arg] += ' = "' + default.s + '"' elif type(default) is _ast.NameConstant: args[num_arg] += " = " + str(default.value) elif type(default) is _ast.Num: args[num_arg] += " = " + str(default.n) elif type(default) is _ast.Name: args[num_arg] += " = " + str(default.id) if node.args.vararg is not None: args.append("*" + node.args.vararg.arg) if node.args.kwarg is not None: args.append("**" + node.args.kwarg.arg) doc = ast.get_docstring(node) super(Function, self).__init__("Function", name, args, doc)
def extract_return(file): for node in ast.walk(ast.parse(open(file).read())): if (isinstance(node, ast.FunctionDef)): print('Method name', node.name) print('Doc string:', ast.get_docstring(node))
def handle_filters(collection, fullpath): """ Grab each filter from a filter plugin file and use the def comment if available :param collection: The full collection name :type collection: str :param fullpath: The full path to the filter plugin file :type fullpath: str :return: A doc of filter plugins + descriptions """ plugins = {} with open(fullpath) as fhand: file_contents = fhand.read() module = ast.parse(file_contents) function_definitions = { node.name: ast.get_docstring(node) for node in module.body if isinstance(node, ast.FunctionDef) } classdef = [ node for node in module.body if isinstance(node, ast.ClassDef) and node.name == "FilterModule" ] if not classdef: return plugins filter_map = next( ( node for node in classdef[0].body if isinstance(node, ast.Assign) and hasattr(node, "targets") and node.targets[0].id == "filter_map" ), None, ) if not filter_map: filter_func = [ func for func in classdef[0].body if isinstance(func, ast.FunctionDef) and func.name == "filters" ] if not filter_func: return plugins # The filter map is either looked up using the filter_map = {} assignment or if return returns a dict literal. filter_map = next( ( node for node in filter_func[0].body if isinstance(node, ast.Return) and isinstance(node.value, ast.Dict) ), None, ) if not filter_map: return plugins keys = [k.value for k in filter_map.value.keys] logging.info("Adding filter plugins %s", ",".join(keys)) values = [k.id for k in filter_map.value.values] filter_map = dict(zip(keys, values)) for name, func in filter_map.items(): if func in function_definitions: comment = function_definitions[ func ] or "{collection} {name} filter plugin".format( collection=collection, name=name ) # Get the first line from the docstring for the description and make that the short description. comment = next( c for c in comment.splitlines() if c and not c.startswith(":") ) plugins[ "{collection}.{name}".format(collection=collection, name=name) ] = comment return plugins
python_file_list = [] module_data = {'deploy': {}, 'update': {}, 'monitor': {}, 'maintain': {}, 'other': {}} for entry in scandir(categories_dictionary['python_code_path']): if entry.path.endswith(".py"): python_file_list.append(entry.path) logging.info("Scanning Python files for docstrings and extracting them...") script_tracker = {} # Used to track if a key has Python scripts, PowerShell scripts, or both for module_path in python_file_list: print("Processing " + module_path) with open(module_path) as fd: module_contents = fd.read() module = ast.parse(module_contents) docstring = ast.get_docstring(module) if docstring is None: docstring = "" # Key is the name without py- ex: get_group_details key = basename(module_path).replace('.py', '') if key in categories_dictionary['deploy']: category = 'deploy' elif key in categories_dictionary['update']: category = 'update' elif key in categories_dictionary['monitor']: category = 'monitor' elif key in categories_dictionary['maintain']: category = 'maintain' else:
def _parse_function(symbol, with_docstrings): docstring = {} attrs = {} func = {'functions': {}} func_name = symbol.name + '(' # We store the arguments to compare with default backwards defaults = [] for value in symbol.args.defaults: # TODO: In some cases we can have something like: a=os.path defaults.append(value) arguments = [] for arg in reversed(symbol.args.args): if not isinstance(arg, ast.Name) or arg.id == "self": continue argument = arg.id if defaults: value = defaults.pop() arg_default = _map_type.get(value.__class__, None) if arg_default is None: if isinstance(value, ast.Attribute): arg_default = model.expand_attribute(value) elif isinstance(value, ast.Name): arg_default = value.id else: arg_default = 'object' argument += '=' + arg_default arguments.append(argument) func_name += ', '.join(reversed(arguments)) if symbol.args.vararg is not None: if not func_name.endswith('('): func_name += ', ' func_name += '*' func_name += symbol.args.vararg.arg if symbol.args.kwarg is not None: if not func_name.endswith('('): func_name += ', ' func_name += '**' func_name += symbol.args.kwarg.arg func_name += ')' for sym in symbol.body: if isinstance(sym, ast.Assign): result = _parse_assign(sym) attrs.update(result[1]) elif isinstance(sym, ast.FunctionDef): result = _parse_function(sym, with_docstrings) if with_docstrings: docstring.update(result['docstring']) func['functions'][result['name']] = { 'lineno': result['lineno'], 'functions': result['functions'] } if with_docstrings: docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True) lineno = symbol.lineno for decorator in symbol.decorator_list: lineno += 1 return {'name': func_name, 'lineno': lineno, 'attrs': attrs, 'docstring': docstring, 'functions': func}
def visit_Module(self, node): ss = ast.get_docstring(node) if ss: self.docstring += ss + "\n" self.generic_visit(node)
def extract_docstring(self, node: ast_fn_def) -> str: """Extract the docstring from a function""" return ast.get_docstring(node) or ""
def fargv(default_switches, argv=None, use_enviromental_variables=True, return_type="SimpleNamespace", return_named_tuple=None, spaces_are_equals=True, description=None): """Parse the argument list and create a dictionary with all parameters. Argument types: Strings: The most generic parameter type. If you need more specific data types, you run eval on a string type. Integers: Anything that can be used to construct an int from a string. Floating Point: Anything that can be used to construct a float from a string. Booleans: If set with out a parameter, it is switched to True. Other wise a case incensitive value of true or false Choices: Defined as tuples in the parameter dictionary. Positionals: Defined as sets in the parameter dictionary. They should not contain tabs and a string list will be returned. This type is designed to work well with wildcards. :param default_switches: A dictionary with parameters as keys and default values as elements. If the value is a collection of two elements who's second element is a string. :param argv: a list of strings which contains all parameters in the form '-PARAM_NAME=PARAM_VALUE'. This list will be emptied of all switches and their values after processed, if the argv is needed full, pass a copy. :param use_enviromental_variables: If set to True, before parsing argv elements to override the default settings, the default settings are first overridden by any assigned environmental variable. :param return_named_tuple: If set to True, result will be a named tuple instead of a dictionary. :param spaces_are_equals: If set to True, a space bar is considered a valid separator if a parameter and its value. :param description: A string describing the overall function of the script. If None (the default parameter) is passed, the docstring of the calling file will be used if defined. :return: Dictionary that is the same as the default values with updated values and the help string. """ if description is None: caller_filename = inspect.getframeinfo(inspect.currentframe().f_back).filename # Assuming the caller is the main script file description = ast.get_docstring(ast.parse(open(caller_filename, "r").read())) if description is None: description = "" if description != "": description = "\n" + description + "\n" if return_named_tuple is not None: sys.stderr.write("fargv.fargv: return return_named_tuple has been deprecated.") if return_named_tuple == True: return_type = "namedtuple" else: return_type = "dict" assert return_type in ["SimpleNamespace", "dict", "namedtuple"] str2type = {bool: lambda x: x.lower() in ["", "true"], # TODO(anguelos) replace lambda with a proper function tuple: lambda x: x, int: lambda x: int(x), float: lambda x: float(x), str: lambda x: x, list: lambda x: eval(x), set: lambda x: x.split("\t") # x will be stiched and than splited } if use_enviromental_variables: for k, default_v in list(default_switches.items()): if k in list(os.environ.keys()): if hasattr(default_v, '__len__') and len(default_v) == 2 and isinstance(default_v[1], str): default_switches[k] = (type(default_v[0])(os.environ[k]), default_v[1]) else: default_switches[k] = type(default_v)(os.environ[k]) new_default_switches = {} switches_help = {"help": "Print help and exit.", "h": "Print help and exit", "bash_autocomplete": "Print a set of bash commands that enable autocomplete for current program."} # Removing help strings from arguments and placing them in stiches_help for k, v in list(default_switches.items()): if (not isinstance(v, str)) and not isinstance(v, set) and hasattr(v, '__len__') and len(v) == 2 and \ isinstance(v[1], str): switches_help[k] = v[1] new_default_switches[k] = v[0] else: switches_help[k] = "" new_default_switches[k] = v default_switches = new_default_switches del new_default_switches default_switches = dict(default_switches, **{"help": False, "bash_autocomplete": False, "h": False}) if argv is None: argv = sys.argv # Compiling positional switches and their values into tab separated strings for switch_name in [k for k, v in default_switches.items() if isinstance(v, set)]: arg_starts = [arg.split("=")[0] for arg in argv] if f"-{switch_name}" in arg_starts: param_list_start = arg_starts.index(f"-{switch_name}") param_list_end = param_list_start + 1 while param_list_end < len(argv) and not argv[param_list_end].startswith("-"): param_list_end += 1 if len(argv[param_list_start]) > len(switch_name)+1 and argv[param_list_start][len(switch_name)+1] == "=": items = [argv[param_list_start][len(switch_name)+2]] else: items=[] items += argv[param_list_start + 1: param_list_end] packed_params = '\t'.join(items) packed_params = f"-{switch_name}={packed_params}" argv[param_list_start:param_list_end] = [packed_params] argv_switches = dict(default_switches) # Allowing pure switch behavior for bool and making = optional for n in range(len(argv)): if len(argv[n]) > 0 and argv[n][0] == "-": key_str = argv[n][1:].split("=")[0] expected_type = type(default_switches[key_str]) if "=" in argv[n]: val_str = argv[n][argv[n].find("=") + 1:] if expected_type is tuple: if not val_str in default_switches[key_str]: print(f"{val_str} should be one of {repr(default_switches[key_str])}", file=sys.stderr) raise ValueError() else: if expected_type is bool: argv[n] = "-" + key_str + "=true" elif spaces_are_equals and n + 1 < len(argv) and argv[n + 1][0] != "-": argv[n] = "-" + key_str + "=" + argv[n + 1] argv[n + 1] = "" if spaces_are_equals: argv[:] = [arg for arg in argv if len(arg) > 0] # setting the choice items (defined as tuples) to be the first item by default for key, val in argv_switches.items(): if type(val) is tuple: argv_switches[key] = default_switches[key][0] argv_switches.update({arg[1:arg.find("=")]: arg[arg.find("=") + 1:] for arg in argv if arg[0] == "-"}) if spaces_are_equals: positionals = [arg for arg in argv if len(arg) and arg[0] != "-"] else: positionals = [arg for arg in argv if arg[0] != "-"] argv[:] = positionals if set(argv_switches.keys()) > set(default_switches.keys()): help_str = "\n" + argv[0] + description + " Syntax:\n\n" for k in list(default_switches.keys()): help_str += f"\t-{k} = {type(default_switches[k])} {switches_help[k]} Default {repr(default_switches[k])}.\n" help_str += "\n\nUnrecognized switches: " + repr(tuple(set(default_switches.keys()) - set(argv_switches.keys()))) help_str += "\nAborting.\n" sys.stderr.write(help_str) sys.exit(1) # Setting argv element to the value type of the default. for k in argv_switches.keys(): if not isinstance(default_switches[k], type(argv_switches[k])): argv_switches[k] = str2type[type(default_switches[k])](argv_switches[k]) help_str = "\n" + argv[0] + description + " Syntax:\n\n" for k in list(default_switches.keys()): help_str += "\t-%s=%s %s Default %s . Passed %s\n" % ( k, repr(type(default_switches[k])), switches_help[k], repr(default_switches[k]), repr(argv_switches[k])) help_str += "\nAborting.\n" # replace {blabla} with argv_switches["balbla"] values # replacable_values = ["{" + k + "}" for k in list(argv_switches.keys())] while len(re.findall("{[a-z0-9A-Z_]+}", "".join([v for v in list(argv_switches.values()) if isinstance(v, str)]))): for k, v in list(argv_switches.items()): if isinstance(v, str): argv_switches[k] = v.format(**argv_switches) if argv_switches["help"] or argv_switches["h"]: sys.stderr.write(help_str) sys.exit() elif argv_switches["bash_autocomplete"]: sys.stdout.write(generate_bash_autocomplete(default_switches, sys.argv[0])) sys.exit() else: del argv_switches["h"] del argv_switches["help"] del argv_switches["bash_autocomplete"] # Verifying choice given is part of allowed choices. for key in default_switches.keys(): if isinstance(default_switches[key],tuple): if argv_switches[key] not in default_switches[key]: print(f"{key} must be one of [{' '.join([repr(v) for v in default_switches[key]])}], value given: {argv_switches[key]}",file=sys.stderr) raise ValueError if return_type.lower() == "namedtuple": params = namedtuple("Parameters", argv_switches.keys())(*argv_switches.values()) elif return_type.lower() == "simplenamespace": params = types.SimpleNamespace(**argv_switches) elif return_type == "dict": params = argv_switches else: raise ValueError return params, help_str
def print_modules(self): basepath = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") ) rst = {} if self._format == "rst": print(".. THIS DOCUMENT IS AUTO-GENERATED, DO NOT MODIFY") print(".. To change this document, please update the docstrings in the individual modules") for m in all_modules(): try: module_type = "core" filename = os.path.join(basepath, "modules", "core", "{}.py".format(m)) if not os.path.exists(filename): filename = os.path.join( basepath, "modules", "contrib", "{}.py".format(m) ) module_type = "contrib" if not os.path.exists(filename): log.warning("module {} not found".format(m)) continue doc = None with open(filename) as f: tree = ast.parse(f.read()) doc = ast.get_docstring(tree) if not doc: log.warning("failed to find docstring for {}".format(m)) continue if self._format == "rst": if os.path.exists( os.path.join(basepath, "..", "screenshots", "{}.png".format(m)) ): doc = "{}\n\n.. image:: ../screenshots/{}.png".format(doc, m) rst[module_type] = rst.get(module_type, []) rst[module_type].append({"module": m, "content": doc}) else: print( textwrap.fill( "{}:".format(m), 80, initial_indent=self._indent * 2, subsequent_indent=self._indent * 2, ) ) for line in doc.split("\n"): print( textwrap.fill( line, 80, initial_indent=self._indent * 3, subsequent_indent=self._indent * 6, ) ) except Exception as e: log.warning(e) if self._format == "rst": print("List of modules\n===============") for k in ["core", "contrib"]: print("\n{}\n{}\n".format(k, "-" * len(k))) for mod in rst[k]: print("\n{}\n{}\n".format(mod["module"], "~" * len(mod["module"]))) print(mod["content"])
def get_file_doc(path): """Load __doc__ from `.py` file""" with open(path) as f: module = ast.parse(f.read()) return ast.get_docstring(module)
def test_get_docstring(self): tree = ast.parse("'docstring'\nx = 1") self.assertEqual(ast.get_docstring(tree), 'docstring') tree.body[0].value = ast.Constant(value='constant docstring') self.assertEqual(ast.get_docstring(tree), 'constant docstring')
def get_module_docstring(path): """get a .py file docstring, without actually executing the file""" with open(path) as f: return ast.get_docstring(ast.parse(f.read()))
def _has_doc(node): """Return if node has docstrings.""" return (ast.get_docstring(node) is not None and ast.get_docstring(node).strip() != "")
def find_avocado_tests(path): """ Attempts to find Avocado instrumented tests from Python source files :param path: path to a Python source code file :type path: str :returns: tuple where first item is dict with class name and additional info such as method names and tags; the second item is set of class names which look like avocado tests but are force-disabled. :rtype: tuple """ module_name = 'avocado' class_name = 'Test' module = PythonModule(path, module_name, class_name) # The resulting test classes result = collections.OrderedDict() disabled = set() for klass in module.iter_classes(): docstring = ast.get_docstring(klass) # Looking for a class that has in the docstring either # ":avocado: enable" or ":avocado: disable if check_docstring_directive(docstring, 'disable'): disabled.add(klass.name) continue if check_docstring_directive(docstring, 'enable'): info = get_methods_info(klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_requirements( docstring)) result[klass.name] = info continue # From this point onwards we want to do recursive discovery, but # for now we don't know whether it is avocado.Test inherited # (Ifs are optimized for readability, not speed) # If "recursive" tag is specified, it is forced as Avocado test if check_docstring_directive(docstring, 'recursive'): is_avocado = True else: is_avocado = module.is_matching_klass(klass) info = get_methods_info(klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_requirements( docstring)) _disabled = set() # Getting the list of parents of the current class parents = klass.bases # Searching the parents in the same module for parent in parents[:]: # Looking for a 'class FooTest(Parent)' if not isinstance(parent, ast.Name): # 'class FooTest(bar.Bar)' not supported withing # a module continue parent_class = parent.id _info, _dis, _avocado = _examine_class(module.path, parent_class, is_avocado, module_name, class_name, _determine_match_avocado) if _info: parents.remove(parent) _extend_test_list(info, _info) _disabled.update(_dis) if _avocado is not is_avocado: is_avocado = _avocado # If there are parents left to be discovered, they # might be in a different module. for parent in parents: if hasattr(parent, 'value'): if hasattr(parent.value, 'id'): # We know 'parent.Class' or 'asparent.Class' and need # to get path and original_module_name. Class is given # by parent definition. _parent = module.imported_objects.get(parent.value.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path = os.path.dirname(_parent) parent_module = os.path.basename(_parent) parent_class = parent.attr else: # We don't support multi-level 'parent.parent.Class' continue else: # We only know 'Class' or 'AsClass' and need to get # path, module and original class_name _parent = module.imported_objects.get(parent.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path, parent_module, parent_class = ( _parent.rsplit(os.path.sep, 2)) modules_paths = [parent_path, os.path.dirname(module.path)] + sys.path try: _, found_ppath, _ = imp.find_module(parent_module, modules_paths) except ImportError: continue _info, _dis, _avocado = _examine_class(found_ppath, parent_class, is_avocado, module_name, class_name, _determine_match_avocado) if _info: info.extend(_info) _disabled.update(_dis) if _avocado is not is_avocado: is_avocado = _avocado # Only update the results if this was detected as 'avocado.Test' if is_avocado: result[klass.name] = info disabled.update(_disabled) return result, disabled
def visit_FunctionDef(self, node): ss = ast.get_docstring(node) if ss: self.docstring += ss + "\n" self.generic_visit(node)
def for_functions(self, c, heading, toc_level): funcs = list(el(c, ast.FunctionDef)) property_operations = {} property_docstrings = {} result = '' toc = [] if len(funcs) > 1: func_heading = "#" * (toc_level + 1) + ' ' + heading + '\n\n' result += func_heading toc.append((heading, toc_level)) for f in funcs: section_title = self.get_section_title(f.lineno) if section_title: result += '\n#### ' + section_title.upper() toc.append((section_title, toc_level + 1)) if f.name.startswith('_'): continue line_actual = f.lineno descriptors = [] property_descriptor = False while True: stripped = self.lines[line_actual - 1].strip() if stripped.startswith('@'): line_actual += 1 if stripped == '@on_main_thread': continue if stripped == '@property': property_operations[f.name] = ['get'] docstr = ast.get_docstring(f) if docstr: property_docstrings[f.name] = docstr property_descriptor = True elif stripped.endswith('.setter'): property_operations[f.name].append('set') property_descriptor = True elif stripped.endswith('.deleter'): property_operations[f.name].append('del') property_descriptor = True descriptors.append(stripped) else: break if property_descriptor: continue descriptor_str = '`' + ', '.join(descriptors) + '`\n' if len( descriptors) > 0 else '' while not stripped.endswith(':'): line_actual += 1 stripped += self.lines[line_actual - 1].strip() result += ('\n#### `' + stripped[len('def '):-1] + '`\n' + descriptor_str + '\n') docstr = ast.get_docstring(f) if docstr: for line in iter(docstr.splitlines()): result += ' ' + line + '\n' if len(property_operations) > 0: prop_heading = "#" * (toc_level + 1) + ' Properties\n\n' result += prop_heading toc.append(('Properties', toc_level)) for prop in property_operations: result += '\n#### `' + prop + ' (' + ', '.join( property_operations[prop]) + ')`\n\n' docstr = property_docstrings.get(prop, None) if docstr: for line in iter(docstr.splitlines()): result += ' ' + line + '\n' return result, toc
def find_python_unittests(path): """ Attempts to find methods names from a given Python source file This is a simpler, albeit more strict and correct, alternative version to :func:`find_class_and_methods`, in the sense that it checks for the (immediate) module name base class name. :param path: path to a Python source code file :type path: str :returns: an ordered dictionary with classes as keys and methods as values :rtype: collections.OrderedDict """ module_name = 'unittest' class_name = 'TestCase' module = PythonModule(path, module_name, class_name) result = collections.OrderedDict() for klass in module.iter_classes(): docstring = ast.get_docstring(klass) parents = klass.bases is_unittest = module.is_matching_klass(klass) info = get_methods_info(klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_requirements( docstring)) # Searching the parents in the same module for parent in parents[:]: # Looking for a 'class FooTest(Parent)' if not isinstance(parent, ast.Name): # 'class FooTest(bar.Bar)' not supported within a module continue parent_class = parent.id _info, _dis, _is_unittest = _examine_class(module.path, parent_class, is_unittest, module_name, class_name, _determine_match_unittest) if _info: parents.remove(parent) _extend_test_list(info, _info) if _is_unittest is not is_unittest: is_unittest = _is_unittest # If there are parents left to be discovered, they # might be in a different module. for parent in parents: if hasattr(parent, 'value'): if hasattr(parent.value, 'id'): # We know 'parent.Class' or 'asparent.Class' and need # to get path and original_module_name. Class is given # by parent definition. _parent = module.imported_objects.get(parent.value.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path = os.path.dirname(_parent) parent_module = os.path.basename(_parent) parent_class = parent.attr else: # We don't support multi-level 'parent.parent.Class' continue else: # We only know 'Class' or 'AsClass' and need to get # path, module and original class_name _parent = module.imported_objects.get(parent.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path, parent_module, parent_class = ( _parent.rsplit(os.path.sep, 2)) modules_paths = [parent_path, os.path.dirname(module.path)] + sys.path try: _, found_ppath, _ = imp.find_module(parent_module, modules_paths) except ImportError: continue _info, _dis, _is_unittest = _examine_class(found_ppath, parent_class, is_unittest, module_name, class_name, _determine_match_unittest) if _info: _extend_test_list(info, _info) if _is_unittest is not is_unittest: is_unittest = _is_unittest # Only update the results if this was detected as 'unittest.TestCase' if is_unittest: result[klass.name] = info return result
def _examine_class(target_module, target_class, determine_match, path, class_name, match): """ Examine a class from a given path :param target_module: the name of the module from which a class should have come from. When attempting to find a Python unittest, the target_module will most probably be "unittest", as per the standard library module name. When attempting to find Avocado tests, the target_module will most probably be "avocado". :type target_module: str :param target_class: the name of the class that is considered to contain test methods. When attempting to find Python unittests, the target_class will most probably be "TestCase". When attempting to find Avocado tests, the target_class will most probably be "Test". :type target_class: str :param determine_match: a callable that will determine if a match has occurred or not :type determine_match: function :param path: path to a Python source code file :type path: str :param class_name: the specific class to be found :type path: str :param match: whether the inheritance from <target_module.target_class> has been determined or not :type match: bool :returns: tuple where first item is a list of test methods detected for given class; second item is set of class names which look like avocado tests but are force-disabled. :rtype: tuple """ module = PythonModule(path, target_module, target_class) info = [] disabled = set() for klass in module.iter_classes(class_name): if class_name != klass.name: continue docstring = ast.get_docstring(klass) if match is False: match = module.is_matching_klass(klass) info = get_methods_info( klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_dependencies(docstring), ) # Getting the list of parents of the current class parents = klass.bases match = _examine_same_module( parents, info, disabled, match, module, target_module, target_class, determine_match, ) # If there are parents left to be discovered, they # might be in a different module. for parent in parents: try: ( parent_class, imported_symbol, symbol_is_module, ) = _get_attributes_for_further_examination(parent, module) found_spec = imported_symbol.get_importable_spec( symbol_is_module) if found_spec is None: continue except ClassNotSuitable: continue _info, _disabled, _match = _examine_class( target_module, target_class, determine_match, found_spec.origin, parent_class, match, ) if _info: _extend_test_list(info, _info) disabled.update(_disabled) if _match is not match: match = _match if not match and module.interesting_klass_found: imported_symbol = module.imported_symbols[class_name] if imported_symbol: found_spec = imported_symbol.get_importable_spec() if found_spec: _info, _disabled, _match = _examine_class( target_module, target_class, determine_match, found_spec.origin, class_name, match, ) if _info: _extend_test_list(info, _info) disabled.update(_disabled) if _match is not match: match = _match return info, disabled, match
def _examine_class(path, class_name, match, target_module, target_class, determine_match): """ Examine a class from a given path :param path: path to a Python source code file :type path: str :param class_name: the specific class to be found :type path: str :param match: whether the inheritance from <target_module.target_class> has been determined or not :type match: bool :param target_module: the module name under which the target_class lives :type target_module: str :param target_class: the name of the class that class_name should ultimately inherit from :type target_class: str :param determine_match: a callable that will determine if a match has occurred or not :type determine_match: function :returns: tuple where first item is a list of test methods detected for given class; second item is set of class names which look like avocado tests but are force-disabled. :rtype: tuple """ module = PythonModule(path, target_module, target_class) info = [] disabled = set() for klass in module.iter_classes(): if class_name != klass.name: continue docstring = ast.get_docstring(klass) if match is False: match = determine_match(module, klass, docstring) info = get_methods_info(klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_requirements( docstring)) # Getting the list of parents of the current class parents = klass.bases # From this point we use `_$variable` to name temporary returns # from method calls that are to-be-assigned/combined with the # existing `$variable`. # Searching the parents in the same module for parent in parents[:]: # Looking for a 'class FooTest(Parent)' if not isinstance(parent, ast.Name): # 'class FooTest(bar.Bar)' not supported withing # a module continue parent_class = parent.id _info, _disabled, _match = _examine_class(module.path, parent_class, match, target_module, target_class, _determine_match_avocado) if _info: parents.remove(parent) _extend_test_list(info, _info) disabled.update(_disabled) if _match is not match: match = _match # If there are parents left to be discovered, they # might be in a different module. for parent in parents: if hasattr(parent, 'value'): if hasattr(parent.value, 'id'): # We know 'parent.Class' or 'asparent.Class' and need # to get path and original_module_name. Class is given # by parent definition. _parent = module.imported_objects.get(parent.value.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path = os.path.dirname(_parent) parent_module = os.path.basename(_parent) parent_class = parent.attr else: # We don't support multi-level 'parent.parent.Class' continue else: # We only know 'Class' or 'AsClass' and need to get # path, module and original class_name _parent = module.imported_objects.get(parent.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path, parent_module, parent_class = ( _parent.rsplit(os.path.sep, 2)) modules_paths = [parent_path, os.path.dirname(module.path)] + sys.path _, found_ppath, _ = imp.find_module(parent_module, modules_paths) _info, _disabled, _match = _examine_class(found_ppath, parent_class, match, target_module, target_class, _determine_match_avocado) if _info: _extend_test_list(info, _info) disabled.update(_disabled) if _match is not match: match = _match return info, disabled, match
def _examine_class(path, class_name, is_avocado): """ Examine a class from a given path :param path: path to a Python source code file :type path: str :param class_name: the specific class to be found :type path: str :returns: tuple where first item is a list of test methods detected for given class; second item is set of class names which look like avocado tests but are force-disabled. :rtype: tuple """ module = AvocadoModule(path) path = module.path # path might get updated (__init__.py) ppath = os.path.dirname(path) info = [] disabled = [] for klass in module.iter_classes(): if class_name != klass.name: continue docstring = ast.get_docstring(klass) cl_tags = get_docstring_directives_tags(docstring) # Only detect 'avocado.Test' if not yet decided if is_avocado is False: directives = get_docstring_directives(docstring) if 'disable' in directives: is_avocado = True elif 'enable' in directives: is_avocado = True elif 'recursive' in directives: is_avocado = True if is_avocado is False: # Still not decided, try inheritance is_avocado = module.is_avocado_test(klass) info = get_methods_info(klass.body, cl_tags) disabled = set() # Getting the list of parents of the current class parents = klass.bases # From this point we use `_$variable` to name temporary returns # from method calls that are to-be-assigned/combined with the # existing `$variable`. # Searching the parents in the same module for parent in parents[:]: # Looking for a 'class FooTest(Parent)' if not isinstance(parent, ast.Name): # 'class FooTest(bar.Bar)' not supported withing # a module continue parent_class = parent.id _info, _disabled, _avocado = _examine_class( path, parent_class, is_avocado) if _info: parents.remove(parent) info.extend(_info) disabled.update(_disabled) if _avocado is not is_avocado: is_avocado = _avocado # If there are parents left to be discovered, they # might be in a different module. for parent in parents: if hasattr(parent, 'value'): if hasattr(parent.value, 'id'): # We know 'parent.Class' or 'asparent.Class' and need # to get path and original_module_name. Class is given # by parent definition. _parent = module.imported_objects.get(parent.value.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path = os.path.dirname(_parent) parent_module = os.path.basename(_parent) parent_class = parent.attr else: # We don't support multi-level 'parent.parent.Class' continue else: # We only know 'Class' or 'AsClass' and need to get # path, module and original class_name _parent = module.imported_objects.get(parent.id) if _parent is None: # We can't examine this parent (probably broken # module) continue parent_path, parent_module, parent_class = (_parent.rsplit( os.path.sep, 2)) modules_paths = [parent_path, ppath] + sys.path _, found_ppath, _ = imp.find_module(parent_module, modules_paths) _info, _dis, _avocado = _examine_class(found_ppath, parent_class, is_avocado) if _info: info.extend(_info) _disabled.update(_dis) if _avocado is not is_avocado: is_avocado = _avocado return info, disabled, is_avocado
def find_python_tests(target_module, target_class, determine_match, path): """ Attempts to find Python tests from source files A Python test in this context is a method within a specific type of class (or that inherits from a specific class). :param target_module: the name of the module from which a class should have come from. When attempting to find a Python unittest, the target_module will most probably be "unittest", as per the standard library module name. When attempting to find Avocado tests, the target_module will most probably be "avocado". :type target_module: str :param target_class: the name of the class that is considered to contain test methods. When attempting to find Python unittests, the target_class will most probably be "TestCase". When attempting to find Avocado tests, the target_class will most probably be "Test". :type target_class: str :type determine_match: a callable that will determine if a given module and class is contains valid Python tests :type determine_match: function :param path: path to a Python source code file :type path: str :returns: tuple where first item is dict with class name and additional info such as method names and tags; the second item is set of class names which look like Python tests but have been forcefully disabled. :rtype: tuple """ module = PythonModule(path, target_module, target_class) # The resulting test classes result = collections.OrderedDict() disabled = set() for klass in module.iter_classes(): docstring = ast.get_docstring(klass) # Looking for a class that has in the docstring either # ":avocado: enable" or ":avocado: disable if check_docstring_directive(docstring, "disable"): disabled.add(klass.name) continue if check_docstring_directive(docstring, "enable"): info = get_methods_info( klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_dependencies(docstring), ) result[klass.name] = info continue # From this point onwards we want to do recursive discovery, but # for now we don't know whether it is avocado.Test inherited # (Ifs are optimized for readability, not speed) # If "recursive" tag is specified, it is forced as test if check_docstring_directive(docstring, "recursive"): match = True else: match = module.is_matching_klass(klass) info = get_methods_info( klass.body, get_docstring_directives_tags(docstring), get_docstring_directives_dependencies(docstring), ) # Getting the list of parents of the current class parents = klass.bases match = _examine_same_module( parents, info, disabled, match, module, target_module, target_class, determine_match, ) # If there are parents left to be discovered, they # might be in a different module. for parent in parents: try: ( parent_class, imported_symbol, symbol_is_module, ) = _get_attributes_for_further_examination(parent, module) found_spec = imported_symbol.get_importable_spec( symbol_is_module) if found_spec is None: continue except ClassNotSuitable: continue _info, _dis, _match = _examine_class( target_module, target_class, determine_match, found_spec.origin, parent_class, match, ) if _info: info.extend(_info) disabled.update(_dis) if _match is not match: match = _match # Only update the results if this was detected as 'avocado.Test' if match: result[klass.name] = info return result, disabled
def _get_dag_file_docstring(fileloc: str) -> str: with open(fileloc) as f: file_contents = f.read() module = ast.parse(file_contents) docstring = ast.get_docstring(module) return docstring
def get_long_description(): tree = get_ast_tree() return ast.get_docstring(tree)
conf = ConfigParser() conf.read(['setup.cfg']) metadata = dict(conf.items('metadata')) PACKAGENAME = metadata.get('package_name', 'packagename') DESCRIPTION = metadata.get('description', 'Astropy affiliated package') AUTHOR = metadata.get('author', '') AUTHOR_EMAIL = metadata.get('author_email', '') LICENSE = metadata.get('license', 'unknown') URL = metadata.get('url', 'http://astropy.org') # Get the long description from the package's docstring _, module_path, _ = imp.find_module(PACKAGENAME) with open(os.path.join(module_path, '__init__.py')) as f: module_ast = ast.parse(f.read()) LONG_DESCRIPTION = ast.get_docstring(module_ast) # Store the package name in a built-in variable so it's easy # to get from other parts of the setup infrastructure builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) VERSION = '0.6.2.dev' # Indicates if this version is a release version RELEASE = 'dev' not in VERSION if not RELEASE: VERSION += get_git_devstr(False) # Populate the dict of setup command overrides; this should be done before
def _find_avocado_tests(self, path, class_name=None): """ Attempts to find Avocado instrumented tests from Python source files :param path: path to a Python source code file :type path: str :param class_name: the specific class to be found :type path: str :returns: tuple where first item is dict with class name and additional info such as method names and tags; the second item is set of class names which look like avocado tests but are force-disabled. :rtype: tuple """ # If only the Test class was imported from the avocado namespace test_import = False # The name used, in case of 'from avocado import Test as AvocadoTest' test_import_name = None # If the "avocado" module itself was imported mod_import = False # The name used, in case of 'import avocado as avocadolib' mod_import_name = None # The resulting test classes result = collections.OrderedDict() disabled = set() if os.path.isdir(path): path = os.path.join(path, "__init__.py") with open(path) as source_file: mod = ast.parse(source_file.read(), path) for statement in mod.body: # Looking for a 'from avocado import Test' if (isinstance(statement, ast.ImportFrom) and statement.module == 'avocado'): for name in statement.names: if name.name == 'Test': test_import = True if name.asname is not None: test_import_name = name.asname else: test_import_name = name.name break # Looking for a 'import avocado' elif isinstance(statement, ast.Import): for name in statement.names: if name.name == 'avocado': mod_import = True if name.asname is not None: mod_import_name = name.nasname else: mod_import_name = name.name # Looking for a 'class Anything(anything):' elif isinstance(statement, ast.ClassDef): # class_name will exist only under recursion. In that # case, we will only process the class if it has the # expected class_name. if class_name is not None and class_name != statement.name: continue docstring = ast.get_docstring(statement) # Looking for a class that has in the docstring either # ":avocado: enable" or ":avocado: disable has_disable = safeloader.check_docstring_directive(docstring, 'disable') if (has_disable and class_name is None): disabled.add(statement.name) continue cl_tags = safeloader.get_docstring_directives_tags(docstring) has_enable = safeloader.check_docstring_directive(docstring, 'enable') if (has_enable and class_name is None): info = self._get_methods_info(statement.body, cl_tags) result[statement.name] = info continue # Looking for the 'recursive' docstring or a 'class_name' # (meaning we are under recursion) has_recurse = safeloader.check_docstring_directive(docstring, 'recursive') if (has_recurse or class_name is not None): info = self._get_methods_info(statement.body, cl_tags) result[statement.name] = info # Getting the list of parents of the current class parents = statement.bases # Searching the parents in the same module for parent in parents[:]: # Looking for a 'class FooTest(module.Parent)' if isinstance(parent, ast.Attribute): parent_class = parent.attr # Looking for a 'class FooTest(Parent)' else: parent_class = parent.id res, dis = self._find_avocado_tests(path, parent_class) if res: parents.remove(parent) for cls in res: info.extend(res[cls]) disabled.update(dis) # If there are parents left to be discovered, they # might be in a different module. for parent in parents: if isinstance(parent, ast.Attribute): # Looking for a 'class FooTest(module.Parent)' parent_module = parent.value.id parent_class = parent.attr else: # Looking for a 'class FooTest(Parent)' parent_module = None parent_class = parent.id for node in mod.body: reference = None # Looking for 'from parent import class' if isinstance(node, ast.ImportFrom): reference = parent_class # Looking for 'import parent' elif isinstance(node, ast.Import): reference = parent_module if reference is None: continue for artifact in node.names: # Looking for a class alias # ('from parent import class as alias') if artifact.asname is not None: parent_class = reference = artifact.name # If the parent class or the parent module # is found in the imports, discover the # parent module path and find the parent # class there if artifact.name == reference: modules_paths = [os.path.dirname(path)] modules_paths.extend(sys.path) if parent_module is None: parent_module = node.module _, ppath, _ = imp.find_module(parent_module, modules_paths) res, dis = self._find_avocado_tests(ppath, parent_class) if res: for cls in res: info.extend(res[cls]) disabled.update(dis) continue if test_import: base_ids = [base.id for base in statement.bases if hasattr(base, 'id')] # Looking for a 'class FooTest(Test):' if test_import_name in base_ids: info = self._get_methods_info(statement.body, cl_tags) result[statement.name] = info continue # Looking for a 'class FooTest(avocado.Test):' if mod_import: for base in statement.bases: module = base.value.id klass = base.attr if module == mod_import_name and klass == 'Test': info = self._get_methods_info(statement.body, cl_tags) result[statement.name] = info continue return result, disabled
def test_get_docstring(self): node = ast.parse('def foo():\n """line one\n line two"""') self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two')
def _get_docstring(fun): # type: (ast.AST) -> str return ast.get_docstring(fun)
def func_doc(self): if self.func is not None: return self.func.__doc__ else: return ast_module.get_docstring(self.ast)
def _find_avocado_tests(self, path): """ Attempts to find Avocado instrumented tests from Python source files :param path: path to a Python source code file :type path: str :returns: dictionary with class name and method names :rtype: dict """ # If only the Test class was imported from the avocado namespace test_import = False # The name used, in case of 'from avocado import Test as AvocadoTest' test_import_name = None # If the "avocado" module itself was imported mod_import = False # The name used, in case of 'import avocado as avocadolib' mod_import_name = None # The resulting test classes result = {} mod = ast.parse(open(path).read(), path) for statement in mod.body: # Looking for a 'from avocado import Test' if (isinstance(statement, ast.ImportFrom) and statement.module == 'avocado'): for name in statement.names: if name.name == 'Test': test_import = True if name.asname is not None: test_import_name = name.asname else: test_import_name = name.name break # Looking for a 'import avocado' elif isinstance(statement, ast.Import): for name in statement.names: if name.name == 'avocado': mod_import = True if name.asname is not None: mod_import_name = name.nasname else: mod_import_name = name.name # Looking for a 'class Anything(anything):' elif isinstance(statement, ast.ClassDef): docstring = ast.get_docstring(statement) # Looking for a class that has in the docstring either # ":avocado: enable" or ":avocado: disable if safeloader.is_docstring_tag_disable(docstring): continue elif safeloader.is_docstring_tag_enable(docstring): functions = [ st.name for st in statement.body if isinstance(st, ast.FunctionDef) and st.name.startswith('test') ] functions = data_structures.ordered_list_unique(functions) result[statement.name] = functions continue if test_import: base_ids = [ base.id for base in statement.bases if hasattr(base, 'id') ] # Looking for a 'class FooTest(Test):' if test_import_name in base_ids: functions = [ st.name for st in statement.body if isinstance(st, ast.FunctionDef) and st.name.startswith('test') ] functions = data_structures.ordered_list_unique( functions) result[statement.name] = functions continue # Looking for a 'class FooTest(avocado.Test):' if mod_import: for base in statement.bases: module = base.value.id klass = base.attr if module == mod_import_name and klass == 'Test': functions = [ st.name for st in statement.body if isinstance(st, ast.FunctionDef) and st.name.startswith('test') ] functions = data_structures.ordered_list_unique( functions) result[statement.name] = functions return result
def module_from_ast(module_name, filename, t): code = code_for_module(module_name, filename, t) module = types.ModuleType(module_name, ast.get_docstring(t)) exec(code, module.__dict__) return module
from .core import driver_not_found as gpu_stump # noqa: F401 import ast import pathlib gpu_stump.__doc__ = "" filepath = pathlib.Path(__file__).parent / "gpu_stump.py" file_contents = "" with open(filepath, encoding="utf8") as f: file_contents = f.read() module = ast.parse(file_contents) function_definitions = [ node for node in module.body if isinstance(node, ast.FunctionDef) ] for fd in function_definitions: if fd.name == "gpu_stump": gpu_stump.__doc__ = ast.get_docstring(fd) try: _dist = get_distribution("stumpy") # Normalize case for Windows systems dist_loc = os.path.normcase(_dist.location) here = os.path.normcase(__file__) if not here.startswith(os.path.join(dist_loc, "stumpy")): # not installed, but there is another version that *is* raise DistributionNotFound # pragma: no cover except DistributionNotFound: # pragma: no cover __version__ = "Please install this project with setup.py" else: # pragma: no cover __version__ = _dist.version