def _find_root_urls_file(self):
# Attempt to find setting folder or settings.py
manage_path = os.path.join(self.project_root_dir, 'manage.py')
manage_ast = self.processor.python_file_asts[manage_path]
settings_module = None
for call in self.processor.filter_ast(manage_ast, _ast3.Call):
if len(call.args) == 2 and ast3.literal_eval(
call.args[0]) == 'DJANGO_SETTINGS_MODULE':
settings_module = ast3.literal_eval(call.args[1])
break
if settings_module:
setting_module = os.path.join(self.project_root_dir,
settings_module.replace('.', '/'))
if setting_module + '.py' in self.processor.python_file_asts:
my_ast = self.processor.python_file_asts[setting_module +
'.py']
for assign in self.processor.filter_ast(my_ast, _ast3.Assign):
# parse through AST of settings.py
if len(assign.targets
) == 1 and assign.targets[0].id == 'ROOT_URLCONF':
root_urls = os.path.join(
self.project_root_dir,
ast3.literal_eval(assign.value).replace(
'.', '/')) + '.py'
if root_urls in self.processor.pythonFileAsts:
return root_urls
elif os.path.isdir(setting_module):
settings_files = list(
filter(lambda x: setting_module in x,
self.processor.pythonFileAsts.keys()))
for sf in settings_files:
my_ast = self.processor.python_file_asts[sf]
for assign in self.processor.filter_ast(
my_ast, _ast3.Assign):
# parse through AST of settings.py
if len(
assign.targets
) == 1 and assign.targets[0].id == 'ROOT_URLCONF':
root_urls = os.path.join(
self.project_root_dir,
ast3.literal_eval(assign.value).replace(
'.', '/')) + '.py'
if root_urls in self.processor.python_file_asts:
return root_urls
url_file = None
url_files = list(
filter(lambda x: 'urls.py' in x,
self.processor.python_file_asts.keys()))
for uf in url_files:
if url_file is None or uf.count('/') < url_file.count('/'):
url_file = uf
return url_file
def cli_wrapper(args):
db_path = Path(args["DB_PATH"])
parent_path = db_path.parent
m = regex.fullmatch(r"(.+)[_-]db\.json", db_path.name)
prefix = m[1] if m else None
pipeline_path = Path(args["--pipe"] or parent_path / f"{prefix}_pipe.py")
if pipeline_path.is_file():
try:
commands = literal_eval(pipeline_path.read_text())
except Exception: # Too many possible exceptions
sys.exit(f"The pipeline '{pipeline_path}' is malformed: aborted.")
elif args["--pipe"]:
sys.exit(f"No pipeline at '{pipeline_path}': aborted.")
else:
commands = []
rec = Recommendations(
commands=commands,
db=json.loads(db_path.read_text()),
base_path=Path(args["--base"] or parent_path),
cost_assessment_strategy=args["--cost"],
)
rec.run_pipeline()
output_path = Path(args["--output"]
or parent_path / f"{prefix}_recommendations.md")
output_path.write_text(rec.get_markdown())
print(f"Dumped: {output_path.resolve()}.\n")
def evaluate_call(node: ast3.Call,
*,
scope: Scope,
module_path: catalog.Path) -> Node:
if not is_named_tuple_definition(node,
scope=scope,
module_path=module_path):
return node
class_name_node, fields_node = node.args
def field_to_parameter(field_node: ast3.expr) -> ast3.arg:
name_node, annotation_node = field_node.elts
return ast3.arg(ast3.literal_eval(name_node), annotation_node)
initializer_node = ast3.FunctionDef(
'__init__',
ast3.arguments([ast3.arg('self', None)]
+ list(map(field_to_parameter,
fields_node.elts)),
None, [], [], None, []),
[ast3.Pass()], [], None)
function_path = evaluate_node(node.func,
scope=scope,
module_path=module_path)
class_def = ast3.ClassDef(ast3.literal_eval(class_name_node),
[ast3.Name(str(function_path), ast3.Load())],
[], [initializer_node], [])
return ast3.fix_missing_locations(ast3.copy_location(class_def, node))
def normalize_strings_to_repr(node):
"""Normalize string leaf nodes to a repr since that is what we generate."""
if isinstance(node, Leaf) and node.type == token.STRING:
node.value = repr(ast3.literal_eval(node.value))
elif isinstance(node, Node):
node.children = [normalize_strings_to_repr(i) for i in node.children]
return node
def cli_wrapper(args):
db_path = Path(args["DB_PATH"])
if not db_path.exists():
print_exit(f"no file or directory at '{db_path.absolute()}': aborted.")
parent_path = db_path.parent
if db_path.is_dir():
for suffix in ("_db", "-db"):
db_path = parent_path / f"{db_path.name}{suffix}.json"
if db_path.is_file():
break
else:
print_exit(
f"unable to locate a tag database in '{parent_path}': aborted."
)
m = regex.fullmatch(r"(.+[_-])db\.json", db_path.name)
prefix = m[1] if m else ""
pipeline_path = Path(args["--pipe"] or parent_path / f"{prefix}pipe.py")
if pipeline_path.is_file():
try:
commands = literal_eval(pipeline_path.read_text())
except Exception: # Too many possible exceptions
print_exit(
f"the pipeline '{pipeline_path}' is malformed: aborted.")
elif args["--pipe"]:
print_exit(f"no pipeline at '{pipeline_path}': aborted.")
else:
commands = []
stdout_backup = sys.stdout
if args["--output"].upper() == "STDOUT":
sys.stdout = sys.stderr
title_format = args["--format"]
if title_format.lower() == "vscode":
title_format = "[`{name}`](vscode://file/{absolute}/{prefix}/{path})"
title_format = title_format.format(
name="{name}",
path="{path}",
prefix=prefix.rstrip("_-"),
absolute=parent_path.resolve(),
relative=parent_path,
)
rec = Recommendations(
db=json.loads(db_path.read_text()),
base_path=Path(args["--base"] or parent_path),
assessment_strategy=args["--cost"],
title_format=title_format,
)
rec.run_pipeline(commands)
if args["--output"].upper() == "STDOUT":
sys.stdout = stdout_backup
return print("\n".join(
sorted(rec.selected_programs - rec.hidden_programs)))
output_path = Path(args["--output"]
or parent_path / f"{prefix}recommendations.md")
output_path.write_text(rec.get_markdown())
print_success(f"Dumped: {output_path.resolve()}\n")
def parse_python_method_args(self, ast_call_object, ordered_args):
"""
This method is used to parse out a python method/function call arguments into a dictionary. It takes an ast call
object and an ordered list of args that it will try to pull out.
:param ast_call_object: The object whose parameters you're trying to parse
:param ordered_args: A list with the names of the arguments. ex. ["request", "parameter", "otherParam"]
:return: A dictionary with the keys being the argument names and the values being the values passed in
"""
results = {}
# loop through all the positional arguments
for i in range(len(ast_call_object.args)):
# Let's make our life easier and resolve the literals
arg = ast_call_object.args[i]
if type(arg) in [
_ast3.Str, _ast3.Bytes, _ast3.Tuple, _ast3.Num, _ast3.List,
_ast3.Set, _ast3.Dict
]:
try:
results[ordered_args[i]] = ast3.literal_eval(arg)
except IndexError:
pass
except ValueError:
results[ordered_args[i]] = arg
else:
results[ordered_args[i]] = arg
# Now let's do that same for the keyword args
for k in ast_call_object.keywords:
if type(k.value) in [
_ast3.Str, _ast3.Bytes, _ast3.Tuple, _ast3.Num, _ast3.List,
_ast3.Set, _ast3.Dict
]:
results[k.arg] = ast3.literal_eval(k.value)
else:
results[k.arg] = k.value
return results
def _find_methods(self, endpoints):
for i in range(len(endpoints)):
view_context = endpoints[i]['view_context']
methods = set()
# Find out if the view is a class or a method/function
if type(view_context) is _ast3.ClassDef:
# find all class functions/methods
functions = list(
filter(lambda x: type(x) is _ast3.FunctionDef,
view_context.body))
for func in functions:
if func.name == 'get':
methods.add('GET')
elif func.name == 'post':
methods.add('POST')
elif func.name == 'put':
methods.add('PUT')
elif func.name == 'patch':
methods.add('PATCH')
elif func.name == 'delete':
methods.add('DELETE')
elif func.name == 'head':
methods.add('HEAD')
elif func.name == 'options':
methods.add('OPTIONS')
elif func.name == 'trace':
methods.add('TRACE')
elif func.name == 'form_valid':
methods.add('POST')
elif type(view_context) is _ast3.FunctionDef:
# Try to find comparators within the function
# ex: if request.method == 'METHOD':
compares = self.processor.filter_ast(view_context,
_ast3.Compare)
for compare in compares:
if (type(compare.left) is _ast3.Attribute
and compare.left.attr == 'method'
and type(compare.left.value) is _ast3.Name
and compare.left.value.id == 'request'
and type(compare.comparators[0]) is _ast3.Str):
methods.add(ast3.literal_eval(compare.comparators[0]))
endpoints[i]['methods'] = methods
return endpoints
def _find_parameters(self, endpoints):
for i in range(len(endpoints)):
# Find out if the view is a class or a method/function
my_ast = self.processor.python_file_asts[endpoints[i]
['view_filepath']]
view_context = None
for x in ast3.walk(my_ast):
if hasattr(x, 'name') and x.name == endpoints[i]['view_name']:
view_context = x
break
# Let's add the view_context to our endpoints dictionary
endpoints[i]['view_context'] = view_context
render_methods = []
method_names = [
'get', 'post', 'put', 'patch', 'delete', 'head', 'options',
'trace', 'form_valid'
]
# Find all the methods/function we have to process
if type(view_context) is _ast3.ClassDef:
for b in view_context.body:
if type(b) is _ast3.FunctionDef and b.name in method_names:
render_methods.append(b)
elif type(view_context) is _ast3.FunctionDef:
render_methods.append(view_context)
params = []
for method in render_methods:
# Find the name of the request object within the method
req_name = None
if method.args.args[0].arg != 'self':
req_name = method.args.args[0].arg
else:
if len(method.args.args) > 1:
req_name = method.args.args[1].arg
else:
pass
http_methods = [
'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS',
'TRACE'
]
# Now lets parse out the params
# This section processes the following:
# <req_name>.cleaned_data['first_name']
# <req_name>.<method_in_caps>["id"]
# self.request.<method_in_caps>["id"]
subscripts = self.processor.filter_ast(method, _ast3.Subscript)
for subscript in subscripts:
if (type(subscript.value) is _ast3.Attribute
and subscript.value.attr == 'cleaned_data'
and type(subscript.value.value) is _ast3.Name
and subscript.value.value.id):
# This processes the following:
# <reqName>.cleaned_data['first_name']
try:
value = ast3.literal_eval(subscript.slice.value)
except ValueError:
# Happens when the parameter name is dynamically generated
# <reqName>.cleaned_data['first_name' + i]
msg = "Couldn't resolve parameter name. File '%s' line '%d'"
logger.warning(msg, endpoints[i]['view_filepath'],
subscript.lineno)
continue
if type(value) is bytes:
value = value.decode(
"utf-8"
) # Accounting for weird bug in typed-ast library
param_dict = {
'name': value,
'filepath': endpoints[i]['view_filepath'],
'line_number': subscript.lineno
}
params.append(param_dict)
elif (type(subscript.value) is _ast3.Attribute
and subscript.value.attr in http_methods
and type(subscript.value.value) is _ast3.Name
and subscript.value.value.id == req_name):
# This processes the following:
# <reqName>.<method_in_caps>["id"]
try:
value = ast3.literal_eval(subscript.slice.value)
except ValueError:
# Happens when the parameter name is dynamically generated
# <reqName>.<method_in_caps>["id" + i]
msg = "Couldn't resolve parameter name. File '%s' line '%d'"
logger.warning(msg, endpoints[i]['view_filepath'],
subscript.lineno)
continue
if type(value) is bytes:
value = value.decode(
"utf-8"
) # Accounting for weird bug in typed-ast library
param_dict = {
'name': value,
'filepath': endpoints[i]['view_filepath'],
'line_number': subscript.lineno
}
params.append(param_dict)
elif (type(subscript.value) is _ast3.Attribute
and subscript.value.attr in http_methods
and type(subscript.value.value) is _ast3.Attribute
and subscript.value.value.attr == 'request'
and type(subscript.value.value.value) is _ast3.Name
and subscript.value.value.value.id == 'self'):
# This processes the following:
# self.request.<method_in_caps>["id"]
try:
value = ast3.literal_eval(subscript.slice.value)
except ValueError:
# Happens when the parameter name is dynamically generated
# self.request.<method_in_caps>["id" + i]
msg = "Couldn't resolve parameter name. File '%s' line '%d'"
logger.warning(msg, endpoints[i]['view_filepath'],
subscript.lineno)
continue
if type(value) is bytes:
value = value.decode(
"utf-8"
) # Accounting for weird bug in typed-ast library
param_dict = {
'name': value,
'filepath': endpoints[i]['view_filepath'],
'line_number': subscript.lineno
}
params.append(param_dict)
# This section processes the following:
# <req_name>.<method_in_caps>.get("param_name", None)
# self.request.<method_in_caps>.get("param_name", None)
calls = self.processor.filter_ast(method, _ast3.Call)
for call in calls:
if (type(call.func) is _ast3.Attribute
and call.func.attr == 'get'):
if (type(call.func.value) is _ast3.Attribute
and call.func.value.attr in http_methods):
if (type(call.func.value.value) is _ast3.Name
and call.func.value.value.id == req_name):
# This processes the following:
# <req_name>.<method_in_caps>.get("param_name", None)
args = self.processor.parse_python_method_args(
call, ['key', 'default'])
if isinstance(args['key'], (bytes, str)):
value = args['key'].decode(
'utf-8'
) if type(
args['key']) is bytes else args['key']
param_dict = {
'name': value,
'filepath':
endpoints[i]['view_filepath'],
'line_number': call.lineno
}
params.append(param_dict)
elif (type(
call.func.value.value) is _ast3.Attribute
and call.func.value.value.attr == 'request'
and type(call.func.value.value.value) is
_ast3.Name and call.func.value.value.value.id
== 'self'):
# This processes the following:
# self.request.<method_in_caps>.get("param_name", None)
args = self.processor.parse_python_method_args(
call, ['key', 'default'])
if isinstance(args['key'], (bytes, str)):
value = args['key'].decode(
'utf-8'
) if type(
args['key']) is bytes else args['key']
param_dict = {
'name': value,
'filepath':
endpoints[i]['view_filepath'],
'line_number': call.lineno
}
params.append(param_dict)
# TODO: find the templates and see if they pull params out of the request object within the template
endpoints[i]['params'] = params
return endpoints
def test_recommend_program(capsys):
rec = Recommendations(
db=json.loads(Path("examples/dummy/programs_db.json").read_text()),
base_path=Path("examples/dummy/"),
)
rec.run_pipeline(literal_eval(Path("examples/dummy/pipe.py").read_text()))
print(rec.selected_programs)
assert rec.selected_programs == {
"prg2.py",
# "O/N/P",
# "Y/T/Q",
# "Y",
# "X/S/M/L/R/D",
# "O",
# "O/C/H/B",
# "X/S/M",
# "X/S/M/L/R",
# "Y/T",
# "O/C",
# "X/G",
# "X/S/M/L/V",
# "O/C/H/B/I",
"prg3.py",
# "O/N/P",
# "X/K",
# "Y/T",
# "X/S/M/L/V",
# "O/C/H/B",
# "X/S/M/L/R",
# "O/J",
# "X/S/M",
# "O/C/F/U",
# "O/C/H",
# "X/S",
# "Y",
# "O",
# "X/S/M/L",
# "Y/E",
}
print(rec.result)
assert rec.result == [
(5, "impart", ["prg8.py"]),
(6, "exclude", ["prg7.py", "prg9.py"]),
(7, "exclude", ["prg4.py", "prg5.py", "prg6.py"]),
(8, "include", ["prg1.py"]),
(9, "hide", []),
]
costs = {
taxon: rec.assess.taxon_cost(taxon)
for taxon in rec.db_programs["prg2.py"]["taxa"]
}
print(costs)
assert costs == {
"O/N/P": 0,
"Y/T/Q": 0.375,
"Y": 0,
"X/S/M/L/R/D": 0,
"O": 0,
"O/C/H/B": 0,
"X/S/M": 0,
"X/S/M/L/R": 0,
"Y/T": 0.25,
"O/C": 0,
"X/G": 0.25,
"X/S/M/L/V": 0,
"O/C/H/B/I": 0.03125,
}
text = rec.get_markdown(span_column_width=10)
make_snapshot(Path("examples/dummy/programs_recommendations.md"), text,
capsys)
def field_to_parameter(field_node: ast3.expr) -> ast3.arg:
name_node, annotation_node = field_node.elts
return ast3.arg(ast3.literal_eval(name_node), annotation_node)
def test_recommend_program(capsys):
rec = Recommendations(
commands=literal_eval(Path("tests/data/dummy/pipe.py").read_text()),
db=json.loads(Path("tests/data/dummy/db.json").read_text()),
base_path=Path("tests/data/dummy/"),
)
rec.run_pipeline()
print(rec.selected_programs)
assert rec.selected_programs == {
"prg2.py": [
"O/N/P",
"Y/T/Q",
"Y",
"X/S/M/L/R/D",
"O",
"O/C/H/B",
"X/S/M",
"X/S/M/L/R",
"Y/T",
"O/C",
"X/G",
"X/S/M/L/V",
"O/C/H/B/I",
],
"prg3.py": [
"O/N/P",
"X/K",
"Y/T",
"X/S/M/L/V",
"O/C/H/B",
"X/S/M/L/R",
"O/J",
"X/S/M",
"O/C/F/U",
"O/C/H",
"X/S",
"Y",
"O",
"X/S/M/L",
"Y/E",
],
}
assert [p["filtered_out"] for p in rec.commands] == [
["prg8.py"],
["prg7.py", "prg9.py"],
["prg4.py", "prg5.py", "prg6.py"],
["prg1.py"],
]
costs = {
taxon: rec.taxon_cost(taxon)
for taxon in rec.selected_programs["prg2.py"]
}
print(costs)
assert costs == {
"O/N/P": 0,
"Y/T/Q": 0.375,
"Y": 0,
"X/S/M/L/R/D": 0,
"O": 0,
"O/C/H/B": 0,
"X/S/M": 0,
"X/S/M/L/R": 0,
"Y/T": 0.25,
"O/C": 0,
"X/G": 0.25,
"X/S/M/L/V": 0,
"O/C/H/B/I": 0.03125,
}
text = rec.get_markdown(span_column_width=10)
make_snapshot(Path("tests/data/dummy/recommendations.md"), text, capsys)
