def _find_root_urls_file(self): # Attempt to find setting folder or settings.py manage_path = os.path.join(self.project_root_dir, 'manage.py') manage_ast = self.processor.python_file_asts[manage_path] settings_module = None for call in self.processor.filter_ast(manage_ast, _ast3.Call): if len(call.args) == 2 and ast3.literal_eval( call.args[0]) == 'DJANGO_SETTINGS_MODULE': settings_module = ast3.literal_eval(call.args[1]) break if settings_module: setting_module = os.path.join(self.project_root_dir, settings_module.replace('.', '/')) if setting_module + '.py' in self.processor.python_file_asts: my_ast = self.processor.python_file_asts[setting_module + '.py'] for assign in self.processor.filter_ast(my_ast, _ast3.Assign): # parse through AST of settings.py if len(assign.targets ) == 1 and assign.targets[0].id == 'ROOT_URLCONF': root_urls = os.path.join( self.project_root_dir, ast3.literal_eval(assign.value).replace( '.', '/')) + '.py' if root_urls in self.processor.pythonFileAsts: return root_urls elif os.path.isdir(setting_module): settings_files = list( filter(lambda x: setting_module in x, self.processor.pythonFileAsts.keys())) for sf in settings_files: my_ast = self.processor.python_file_asts[sf] for assign in self.processor.filter_ast( my_ast, _ast3.Assign): # parse through AST of settings.py if len( assign.targets ) == 1 and assign.targets[0].id == 'ROOT_URLCONF': root_urls = os.path.join( self.project_root_dir, ast3.literal_eval(assign.value).replace( '.', '/')) + '.py' if root_urls in self.processor.python_file_asts: return root_urls url_file = None url_files = list( filter(lambda x: 'urls.py' in x, self.processor.python_file_asts.keys())) for uf in url_files: if url_file is None or uf.count('/') < url_file.count('/'): url_file = uf return url_file
def cli_wrapper(args): db_path = Path(args["DB_PATH"]) parent_path = db_path.parent m = regex.fullmatch(r"(.+)[_-]db\.json", db_path.name) prefix = m[1] if m else None pipeline_path = Path(args["--pipe"] or parent_path / f"{prefix}_pipe.py") if pipeline_path.is_file(): try: commands = literal_eval(pipeline_path.read_text()) except Exception: # Too many possible exceptions sys.exit(f"The pipeline '{pipeline_path}' is malformed: aborted.") elif args["--pipe"]: sys.exit(f"No pipeline at '{pipeline_path}': aborted.") else: commands = [] rec = Recommendations( commands=commands, db=json.loads(db_path.read_text()), base_path=Path(args["--base"] or parent_path), cost_assessment_strategy=args["--cost"], ) rec.run_pipeline() output_path = Path(args["--output"] or parent_path / f"{prefix}_recommendations.md") output_path.write_text(rec.get_markdown()) print(f"Dumped: {output_path.resolve()}.\n")
def evaluate_call(node: ast3.Call, *, scope: Scope, module_path: catalog.Path) -> Node: if not is_named_tuple_definition(node, scope=scope, module_path=module_path): return node class_name_node, fields_node = node.args def field_to_parameter(field_node: ast3.expr) -> ast3.arg: name_node, annotation_node = field_node.elts return ast3.arg(ast3.literal_eval(name_node), annotation_node) initializer_node = ast3.FunctionDef( '__init__', ast3.arguments([ast3.arg('self', None)] + list(map(field_to_parameter, fields_node.elts)), None, [], [], None, []), [ast3.Pass()], [], None) function_path = evaluate_node(node.func, scope=scope, module_path=module_path) class_def = ast3.ClassDef(ast3.literal_eval(class_name_node), [ast3.Name(str(function_path), ast3.Load())], [], [initializer_node], []) return ast3.fix_missing_locations(ast3.copy_location(class_def, node))
def normalize_strings_to_repr(node): """Normalize string leaf nodes to a repr since that is what we generate.""" if isinstance(node, Leaf) and node.type == token.STRING: node.value = repr(ast3.literal_eval(node.value)) elif isinstance(node, Node): node.children = [normalize_strings_to_repr(i) for i in node.children] return node
def cli_wrapper(args): db_path = Path(args["DB_PATH"]) if not db_path.exists(): print_exit(f"no file or directory at '{db_path.absolute()}': aborted.") parent_path = db_path.parent if db_path.is_dir(): for suffix in ("_db", "-db"): db_path = parent_path / f"{db_path.name}{suffix}.json" if db_path.is_file(): break else: print_exit( f"unable to locate a tag database in '{parent_path}': aborted." ) m = regex.fullmatch(r"(.+[_-])db\.json", db_path.name) prefix = m[1] if m else "" pipeline_path = Path(args["--pipe"] or parent_path / f"{prefix}pipe.py") if pipeline_path.is_file(): try: commands = literal_eval(pipeline_path.read_text()) except Exception: # Too many possible exceptions print_exit( f"the pipeline '{pipeline_path}' is malformed: aborted.") elif args["--pipe"]: print_exit(f"no pipeline at '{pipeline_path}': aborted.") else: commands = [] stdout_backup = sys.stdout if args["--output"].upper() == "STDOUT": sys.stdout = sys.stderr title_format = args["--format"] if title_format.lower() == "vscode": title_format = "[`{name}`](vscode://file/{absolute}/{prefix}/{path})" title_format = title_format.format( name="{name}", path="{path}", prefix=prefix.rstrip("_-"), absolute=parent_path.resolve(), relative=parent_path, ) rec = Recommendations( db=json.loads(db_path.read_text()), base_path=Path(args["--base"] or parent_path), assessment_strategy=args["--cost"], title_format=title_format, ) rec.run_pipeline(commands) if args["--output"].upper() == "STDOUT": sys.stdout = stdout_backup return print("\n".join( sorted(rec.selected_programs - rec.hidden_programs))) output_path = Path(args["--output"] or parent_path / f"{prefix}recommendations.md") output_path.write_text(rec.get_markdown()) print_success(f"Dumped: {output_path.resolve()}\n")
def parse_python_method_args(self, ast_call_object, ordered_args): """ This method is used to parse out a python method/function call arguments into a dictionary. It takes an ast call object and an ordered list of args that it will try to pull out. :param ast_call_object: The object whose parameters you're trying to parse :param ordered_args: A list with the names of the arguments. ex. ["request", "parameter", "otherParam"] :return: A dictionary with the keys being the argument names and the values being the values passed in """ results = {} # loop through all the positional arguments for i in range(len(ast_call_object.args)): # Let's make our life easier and resolve the literals arg = ast_call_object.args[i] if type(arg) in [ _ast3.Str, _ast3.Bytes, _ast3.Tuple, _ast3.Num, _ast3.List, _ast3.Set, _ast3.Dict ]: try: results[ordered_args[i]] = ast3.literal_eval(arg) except IndexError: pass except ValueError: results[ordered_args[i]] = arg else: results[ordered_args[i]] = arg # Now let's do that same for the keyword args for k in ast_call_object.keywords: if type(k.value) in [ _ast3.Str, _ast3.Bytes, _ast3.Tuple, _ast3.Num, _ast3.List, _ast3.Set, _ast3.Dict ]: results[k.arg] = ast3.literal_eval(k.value) else: results[k.arg] = k.value return results
def _find_methods(self, endpoints): for i in range(len(endpoints)): view_context = endpoints[i]['view_context'] methods = set() # Find out if the view is a class or a method/function if type(view_context) is _ast3.ClassDef: # find all class functions/methods functions = list( filter(lambda x: type(x) is _ast3.FunctionDef, view_context.body)) for func in functions: if func.name == 'get': methods.add('GET') elif func.name == 'post': methods.add('POST') elif func.name == 'put': methods.add('PUT') elif func.name == 'patch': methods.add('PATCH') elif func.name == 'delete': methods.add('DELETE') elif func.name == 'head': methods.add('HEAD') elif func.name == 'options': methods.add('OPTIONS') elif func.name == 'trace': methods.add('TRACE') elif func.name == 'form_valid': methods.add('POST') elif type(view_context) is _ast3.FunctionDef: # Try to find comparators within the function # ex: if request.method == 'METHOD': compares = self.processor.filter_ast(view_context, _ast3.Compare) for compare in compares: if (type(compare.left) is _ast3.Attribute and compare.left.attr == 'method' and type(compare.left.value) is _ast3.Name and compare.left.value.id == 'request' and type(compare.comparators[0]) is _ast3.Str): methods.add(ast3.literal_eval(compare.comparators[0])) endpoints[i]['methods'] = methods return endpoints
def _find_parameters(self, endpoints): for i in range(len(endpoints)): # Find out if the view is a class or a method/function my_ast = self.processor.python_file_asts[endpoints[i] ['view_filepath']] view_context = None for x in ast3.walk(my_ast): if hasattr(x, 'name') and x.name == endpoints[i]['view_name']: view_context = x break # Let's add the view_context to our endpoints dictionary endpoints[i]['view_context'] = view_context render_methods = [] method_names = [ 'get', 'post', 'put', 'patch', 'delete', 'head', 'options', 'trace', 'form_valid' ] # Find all the methods/function we have to process if type(view_context) is _ast3.ClassDef: for b in view_context.body: if type(b) is _ast3.FunctionDef and b.name in method_names: render_methods.append(b) elif type(view_context) is _ast3.FunctionDef: render_methods.append(view_context) params = [] for method in render_methods: # Find the name of the request object within the method req_name = None if method.args.args[0].arg != 'self': req_name = method.args.args[0].arg else: if len(method.args.args) > 1: req_name = method.args.args[1].arg else: pass http_methods = [ 'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE' ] # Now lets parse out the params # This section processes the following: # <req_name>.cleaned_data['first_name'] # <req_name>.<method_in_caps>["id"] # self.request.<method_in_caps>["id"] subscripts = self.processor.filter_ast(method, _ast3.Subscript) for subscript in subscripts: if (type(subscript.value) is _ast3.Attribute and subscript.value.attr == 'cleaned_data' and type(subscript.value.value) is _ast3.Name and subscript.value.value.id): # This processes the following: # <reqName>.cleaned_data['first_name'] try: value = ast3.literal_eval(subscript.slice.value) except ValueError: # Happens when the parameter name is dynamically generated # <reqName>.cleaned_data['first_name' + i] msg = "Couldn't resolve parameter name. File '%s' line '%d'" logger.warning(msg, endpoints[i]['view_filepath'], subscript.lineno) continue if type(value) is bytes: value = value.decode( "utf-8" ) # Accounting for weird bug in typed-ast library param_dict = { 'name': value, 'filepath': endpoints[i]['view_filepath'], 'line_number': subscript.lineno } params.append(param_dict) elif (type(subscript.value) is _ast3.Attribute and subscript.value.attr in http_methods and type(subscript.value.value) is _ast3.Name and subscript.value.value.id == req_name): # This processes the following: # <reqName>.<method_in_caps>["id"] try: value = ast3.literal_eval(subscript.slice.value) except ValueError: # Happens when the parameter name is dynamically generated # <reqName>.<method_in_caps>["id" + i] msg = "Couldn't resolve parameter name. File '%s' line '%d'" logger.warning(msg, endpoints[i]['view_filepath'], subscript.lineno) continue if type(value) is bytes: value = value.decode( "utf-8" ) # Accounting for weird bug in typed-ast library param_dict = { 'name': value, 'filepath': endpoints[i]['view_filepath'], 'line_number': subscript.lineno } params.append(param_dict) elif (type(subscript.value) is _ast3.Attribute and subscript.value.attr in http_methods and type(subscript.value.value) is _ast3.Attribute and subscript.value.value.attr == 'request' and type(subscript.value.value.value) is _ast3.Name and subscript.value.value.value.id == 'self'): # This processes the following: # self.request.<method_in_caps>["id"] try: value = ast3.literal_eval(subscript.slice.value) except ValueError: # Happens when the parameter name is dynamically generated # self.request.<method_in_caps>["id" + i] msg = "Couldn't resolve parameter name. File '%s' line '%d'" logger.warning(msg, endpoints[i]['view_filepath'], subscript.lineno) continue if type(value) is bytes: value = value.decode( "utf-8" ) # Accounting for weird bug in typed-ast library param_dict = { 'name': value, 'filepath': endpoints[i]['view_filepath'], 'line_number': subscript.lineno } params.append(param_dict) # This section processes the following: # <req_name>.<method_in_caps>.get("param_name", None) # self.request.<method_in_caps>.get("param_name", None) calls = self.processor.filter_ast(method, _ast3.Call) for call in calls: if (type(call.func) is _ast3.Attribute and call.func.attr == 'get'): if (type(call.func.value) is _ast3.Attribute and call.func.value.attr in http_methods): if (type(call.func.value.value) is _ast3.Name and call.func.value.value.id == req_name): # This processes the following: # <req_name>.<method_in_caps>.get("param_name", None) args = self.processor.parse_python_method_args( call, ['key', 'default']) if isinstance(args['key'], (bytes, str)): value = args['key'].decode( 'utf-8' ) if type( args['key']) is bytes else args['key'] param_dict = { 'name': value, 'filepath': endpoints[i]['view_filepath'], 'line_number': call.lineno } params.append(param_dict) elif (type( call.func.value.value) is _ast3.Attribute and call.func.value.value.attr == 'request' and type(call.func.value.value.value) is _ast3.Name and call.func.value.value.value.id == 'self'): # This processes the following: # self.request.<method_in_caps>.get("param_name", None) args = self.processor.parse_python_method_args( call, ['key', 'default']) if isinstance(args['key'], (bytes, str)): value = args['key'].decode( 'utf-8' ) if type( args['key']) is bytes else args['key'] param_dict = { 'name': value, 'filepath': endpoints[i]['view_filepath'], 'line_number': call.lineno } params.append(param_dict) # TODO: find the templates and see if they pull params out of the request object within the template endpoints[i]['params'] = params return endpoints
def test_recommend_program(capsys): rec = Recommendations( db=json.loads(Path("examples/dummy/programs_db.json").read_text()), base_path=Path("examples/dummy/"), ) rec.run_pipeline(literal_eval(Path("examples/dummy/pipe.py").read_text())) print(rec.selected_programs) assert rec.selected_programs == { "prg2.py", # "O/N/P", # "Y/T/Q", # "Y", # "X/S/M/L/R/D", # "O", # "O/C/H/B", # "X/S/M", # "X/S/M/L/R", # "Y/T", # "O/C", # "X/G", # "X/S/M/L/V", # "O/C/H/B/I", "prg3.py", # "O/N/P", # "X/K", # "Y/T", # "X/S/M/L/V", # "O/C/H/B", # "X/S/M/L/R", # "O/J", # "X/S/M", # "O/C/F/U", # "O/C/H", # "X/S", # "Y", # "O", # "X/S/M/L", # "Y/E", } print(rec.result) assert rec.result == [ (5, "impart", ["prg8.py"]), (6, "exclude", ["prg7.py", "prg9.py"]), (7, "exclude", ["prg4.py", "prg5.py", "prg6.py"]), (8, "include", ["prg1.py"]), (9, "hide", []), ] costs = { taxon: rec.assess.taxon_cost(taxon) for taxon in rec.db_programs["prg2.py"]["taxa"] } print(costs) assert costs == { "O/N/P": 0, "Y/T/Q": 0.375, "Y": 0, "X/S/M/L/R/D": 0, "O": 0, "O/C/H/B": 0, "X/S/M": 0, "X/S/M/L/R": 0, "Y/T": 0.25, "O/C": 0, "X/G": 0.25, "X/S/M/L/V": 0, "O/C/H/B/I": 0.03125, } text = rec.get_markdown(span_column_width=10) make_snapshot(Path("examples/dummy/programs_recommendations.md"), text, capsys)
def field_to_parameter(field_node: ast3.expr) -> ast3.arg: name_node, annotation_node = field_node.elts return ast3.arg(ast3.literal_eval(name_node), annotation_node)
def test_recommend_program(capsys): rec = Recommendations( commands=literal_eval(Path("tests/data/dummy/pipe.py").read_text()), db=json.loads(Path("tests/data/dummy/db.json").read_text()), base_path=Path("tests/data/dummy/"), ) rec.run_pipeline() print(rec.selected_programs) assert rec.selected_programs == { "prg2.py": [ "O/N/P", "Y/T/Q", "Y", "X/S/M/L/R/D", "O", "O/C/H/B", "X/S/M", "X/S/M/L/R", "Y/T", "O/C", "X/G", "X/S/M/L/V", "O/C/H/B/I", ], "prg3.py": [ "O/N/P", "X/K", "Y/T", "X/S/M/L/V", "O/C/H/B", "X/S/M/L/R", "O/J", "X/S/M", "O/C/F/U", "O/C/H", "X/S", "Y", "O", "X/S/M/L", "Y/E", ], } assert [p["filtered_out"] for p in rec.commands] == [ ["prg8.py"], ["prg7.py", "prg9.py"], ["prg4.py", "prg5.py", "prg6.py"], ["prg1.py"], ] costs = { taxon: rec.taxon_cost(taxon) for taxon in rec.selected_programs["prg2.py"] } print(costs) assert costs == { "O/N/P": 0, "Y/T/Q": 0.375, "Y": 0, "X/S/M/L/R/D": 0, "O": 0, "O/C/H/B": 0, "X/S/M": 0, "X/S/M/L/R": 0, "Y/T": 0.25, "O/C": 0, "X/G": 0.25, "X/S/M/L/V": 0, "O/C/H/B/I": 0.03125, } text = rec.get_markdown(span_column_width=10) make_snapshot(Path("tests/data/dummy/recommendations.md"), text, capsys)