def _transform(path: str, code: str,
target: CompilationTarget) -> Tuple[str, List[str]]:
"""Applies all transformation for passed target."""
dependencies = [] # type: List[str]
for transformer in transformers:
tree = ast.parse(code, path)
if transformer.target < target:
continue
try:
result = transformer.transform(tree)
except:
raise TransformationError(path, transformer, dump(tree),
format_exc())
if not result.tree_changed:
continue
dependencies.extend(result.dependencies)
try:
code = unparse(tree)
except:
raise TransformationError(path, transformer, dump(tree),
format_exc())
return fix_code(code), dependencies
def dispatch_var_type(self, tree):
code = horast.unparse(tree)
stripped_code = code.strip()
if stripped_code in PYTHON_FORTRAN_TYPE_PAIRS:
type_name, precision = PYTHON_FORTRAN_TYPE_PAIRS[stripped_code]
self.write(type_name)
if precision is not None:
self.write('*')
self.write(str(precision))
elif _match_array(tree):
sli = tree.slice
assert isinstance(sli,
typed_ast3.Index), typed_astunparse.dump(tree)
assert isinstance(sli.value, typed_ast3.Tuple)
assert len(sli.value.elts) in (2, 3), sli.value.elts
elts = sli.value.elts
self.dispatch_var_type(elts[1])
self.write(', ')
self.write('dimension(')
if len(sli.value.elts) == 2:
self.write(':')
else:
if not self._context_input_args:
self.dispatch(elts[2])
else:
assert isinstance(elts[2], typed_ast3.Tuple)
_LOG.warning('coercing indices of %i dims to 0-based',
len(elts[2].elts))
# _LOG.warning('coercing indices of %s in %s to 0-based', arg.arg, t.name)
tup_elts = []
for elt in elts[2].elts:
if isinstance(elt, typed_ast3.Num):
assert isinstance(elt.n, int)
upper = typed_ast3.Num(n=elt.n - 1)
else:
assert isinstance(elt, typed_ast3.Name)
upper = typed_ast3.BinOp(left=elt,
op=typed_ast3.Sub(),
right=typed_ast3.Num(n=1))
tup_elts.append(
typed_ast3.Slice(lower=typed_ast3.Num(n=0),
upper=upper,
step=None))
tup = typed_ast3.Tuple(elts=tup_elts,
ctx=typed_ast3.Load())
self.dispatch(tup)
self.write(')')
elif _match_io(tree):
self.write('integer')
elif isinstance(tree, typed_ast3.Call) and isinstance(tree.func, typed_ast3.Name) \
and tree.func.id == 'type':
self.dispatch(tree)
else:
raise NotImplementedError('not yet implemented: {}'.format(
typed_astunparse.dump(tree)))
def transform(transformer, before):
tree = parse(before)
try:
transformer.transform(tree)
return unparse(tree).strip()
except:
print('Before:')
print(dump(parse(before)))
print('After:')
print(dump(tree))
raise
def _NameConstant(self, t):
if t.value is None:
_LOG.info('possibly invalid Fortran in """%s"""', typed_astunparse.dump(t))
self.write({
None: 'none',
False: '.false.',
True: '.true.'}[t.value])
def _add_kind_info(self):
if len(self.decorator_list) == 0:
if len(self.args.args) == 0:
self._kind = FunctionKind.Function
else:
first_arg = self.args.args[0].arg
if first_arg == 'self':
if self.name == '__init__':
self._kind = FunctionKind.Constructor
else:
self._kind = FunctionKind.InstanceMethod
else:
self._kind = FunctionKind.Function
elif len(self.decorator_list) == 1:
decorator = self.decorator_list[0]
assert isinstance(decorator, ast_module.Name)
self._kind = {
'classmethod': FunctionKind.ClassMethod,
'staticmethod': FunctionKind.StaticMethod}.get(
decorator.id, FunctionKind.Undetermined)
else:
raise NotImplementedError('no support for many decorators:\n{}'
.format(self.decorator_list))
if self._kind is FunctionKind.Undetermined:
raise NotImplementedError('could not determine function kind:\n{}'
.format(typed_astunparse.dump(self)))
def test_dump_files_comparison(self):
"""Print the same data as other existing modules."""
for path in PATHS:
with open(path, 'r', encoding='utf-8') as py_file:
code = py_file.read()
untyped_tree = ast.parse(source=code, filename=path)
untyped_dump = astunparse.dump(untyped_tree)
tested_untyped_dump = typed_astunparse.dump(untyped_tree)
self.assertEqual(untyped_dump.splitlines(),
tested_untyped_dump.splitlines())
typed_tree = typed_ast.ast3.parse(source=code, filename=path)
bad_typed_dump = astunparse.dump(typed_tree)
for annotate_fields in [True, False]:
for include_attributes in [False, True]:
if include_attributes and not annotate_fields:
continue # behaviour differs from typed_ast
with self.assertRaises(TypeError):
_ = typed_ast.ast3.dump(
untyped_tree,
annotate_fields=annotate_fields,
include_attributes=include_attributes)
typed_dump = typed_ast.ast3.dump(
typed_tree,
annotate_fields=annotate_fields,
include_attributes=include_attributes)
tested_typed_dump = _postprocess_dump(
typed_astunparse.dump(
typed_tree,
annotate_fields=annotate_fields,
include_attributes=include_attributes))
if include_attributes:
# because of https://github.com/python/typed_ast/issues/23
self.assertEqual(typed_dump.replace(' ', ''),
tested_typed_dump.replace(' ', ''))
continue
self.assertNotEqual(untyped_dump, bad_typed_dump)
self.assertNotEqual(typed_dump, bad_typed_dump)
self.assertEqual(typed_dump, tested_typed_dump)
def _BinOp(self, t):
if t.op.__class__.__name__ not in self.binop:
raise NotImplementedError('not yet implemented: {}'.format(
typed_astunparse.dump(t)))
self.write('(')
self.dispatch(t.left)
self.write(' ' + self.binop[t.op.__class__.__name__] + ' ')
self.dispatch(t.right)
self.write(')')
def test_dump_examples(self):
"""Print ASTs of examples correctly."""
for description, example in EXAMPLES.items():
for mode in MODES:
if example['trees'][mode] is None:
continue
dump = typed_astunparse.dump(example['trees'][mode])
_LOG.debug('%s', dump)
dump = dump.replace('\n', '').replace(' ', '')
self.assertEqual(dump, example['dumps'][mode], msg=(description, mode))
def test_generalize_examples(self):
code_reader = CodeReader()
parser = C99Parser()
ast_generalizer = CAstGeneralizer()
for path in EXAMPLES_C11_FILES:
code = code_reader.read_file(path)
tree = parser.parse(code, path)
basic_check_c_ast(self, path, tree)
tree = ast_generalizer.generalize(tree)
basic_check_python_ast(self, path, tree)
_LOG.debug('%s', typed_astunparse.dump(tree))
_LOG.debug('%s', typed_astunparse.unparse(tree))
def test_dump_examples(self):
"""Print ASTs of examples correctly."""
for description, example in EXAMPLES.items():
for mode in MODES:
if example['trees'][mode] is None:
continue
dump = typed_astunparse.dump(example['trees'][mode])
_LOG.debug('%s', dump)
dump = dump.replace('\n', '').replace(' ', '')
self.assertEqual(dump,
example['dumps'][mode],
msg=(description, mode))
def test_dump_files_comparison(self):
"""Print the same data as other existing modules."""
for path in PATHS:
with open(path, 'r', encoding='utf-8') as py_file:
code = py_file.read()
untyped_tree = ast.parse(source=code, filename=path)
untyped_dump = astunparse.dump(untyped_tree)
tested_untyped_dump = typed_astunparse.dump(untyped_tree)
self.assertEqual(untyped_dump.splitlines(), tested_untyped_dump.splitlines())
typed_tree = typed_ast.ast3.parse(source=code, filename=path)
bad_typed_dump = astunparse.dump(typed_tree)
for annotate_fields in [True, False]:
for include_attributes in [False, True]:
if include_attributes and not annotate_fields:
continue # behaviour differs from typed_ast
with self.assertRaises(TypeError):
_ = typed_ast.ast3.dump(
untyped_tree, annotate_fields=annotate_fields,
include_attributes=include_attributes)
typed_dump = typed_ast.ast3.dump(
typed_tree, annotate_fields=annotate_fields,
include_attributes=include_attributes)
tested_typed_dump = _postprocess_dump(typed_astunparse.dump(
typed_tree, annotate_fields=annotate_fields,
include_attributes=include_attributes))
if include_attributes:
# because of https://github.com/python/typed_ast/issues/23
self.assertEqual(
typed_dump.replace(' ', ''), tested_typed_dump.replace(' ', ''))
continue
self.assertNotEqual(untyped_dump, bad_typed_dump)
self.assertNotEqual(typed_dump, bad_typed_dump)
self.assertEqual(typed_dump, tested_typed_dump)
def test_generalize_examples(self, input_path):
code_reader = CodeReader()
code = code_reader.read_file(input_path)
parser = CppParser()
cpp_ast = parser.parse(code, input_path)
basic_check_cpp_ast(self, input_path, cpp_ast)
ast_generalizer = CppAstGeneralizer(scope={'path': input_path})
with _TIME.measure('generalize.{}'.format(input_path.name.replace('.', '_'))) as timer:
syntax = ast_generalizer.generalize(cpp_ast)
basic_check_python_ast(self, input_path, syntax)
_LOG.info('generalized "%s" in %fs', input_path, timer.elapsed)
_LOG.debug('%s', typed_astunparse.dump(syntax))
_LOG.debug('%s', typed_astunparse.unparse(syntax))
def test_generalize_examples(self):
code_reader = CodeReader()
parser = CppParser()
for path in EXAMPLES_CPP14_FILES:
ast_generalizer = CppAstGeneralizer(scope={'path': path})
code = code_reader.read_file(path)
tree = parser.parse(code, path)
basic_check_cpp_ast(self, path, tree)
with self.subTest(path=path):
tree = ast_generalizer.generalize(tree)
basic_check_python_ast(self, path, tree)
_LOG.debug('%s', typed_astunparse.dump(tree))
_LOG.debug('%s', typed_astunparse.unparse(tree))
def test_many_dump_roundtrips(self):
"""Preserve ASTs after unparse(parse(...unparse(parse(dump(tree)))...))."""
for description, example in EXAMPLES.items():
for mode in MODES:
if example['trees'][mode] is None:
continue
dump = typed_astunparse.dump(example['trees'][mode])
for _ in range(4):
tree = typed_ast.ast3.parse(source=dump, mode=mode)
dump = typed_astunparse.unparse(tree)
_LOG.debug('%s', dump)
clean_dump = dump.replace('\n', '').replace(' ', '')
self.assertEqual(clean_dump, example['dumps'][mode], msg=(description, mode))
def transform(path: str, code: str, target: CompilationTarget) -> str:
"""Applies all transformation for passed target."""
from ..exceptions import TransformationError
for transformer in transformers:
tree = ast.parse(code, path)
if transformer.target >= target:
transformer().visit(tree)
try:
code = unparse(tree)
except:
raise TransformationError(path, transformer, dump(tree),
format_exc())
return fix_code(code)
def test_many_dump_roundtrips(self):
"""Preserve ASTs after unparse(parse(...unparse(parse(dump(tree)))...))."""
for description, example in EXAMPLES.items():
for mode in MODES:
if example['trees'][mode] is None:
continue
dump = typed_astunparse.dump(example['trees'][mode])
for _ in range(4):
tree = typed_ast.ast3.parse(source=dump, mode=mode)
dump = typed_astunparse.unparse(tree)
_LOG.debug('%s', dump)
clean_dump = dump.replace('\n', '').replace(' ', '')
self.assertEqual(clean_dump,
example['dumps'][mode],
msg=(description, mode))
def _keyword(self, t):
if t.arg is None:
raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
super()._keyword(t)
def test_inline_flash_subset_hydro(self):
app_name = 'FLASH-SUBSET'
if app_name not in _APPS_ROOT_PATHS and app_name in _APPS_OPTIONAL:
self.skipTest('{} directory not found'.format(app_name))
language = Language.find('Fortran')
reader = CodeReader()
parser = Parser.find(language)()
ast_generalizer = AstGeneralizer.find(language)()
f_unparser = Unparser.find(language)()
py_unparser = Unparser.find(Language.find('Python'))()
writer = CodeWriter()
dir_name = app_name.lower()
results_path = pathlib.Path(RESULTS_ROOT, 'transformations', 'inlining', dir_name)
results_path.mkdir(parents=True, exist_ok=True)
path_pairs = [
(pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux_loop.F90'),
pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux.F90'),
(1, 1)),
(pathlib.Path('physics/Eos/EosMain/Eos_getData_loop1.F90'),
pathlib.Path('physics/Eos/EosMain/Eos_getData.F90'),
(1, 2))]
for inlined_path, target_path, (index, extra_lines) in path_pairs:
inlined_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', inlined_path)
target_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', target_path)
output_inlined_path = results_path.joinpath(inlined_path.name)
output_target_path = results_path.joinpath(target_path.name)
output_path = results_path.joinpath(target_path.with_suffix('').name + '_inlined.F90')
inlined_xml = parser.parse('', inlined_path)
inlined_xml = inlined_xml.find('.//subroutine')
writer.write_file(ET.tostring(inlined_xml, 'utf-8').decode(),
output_inlined_path.with_suffix('.xml'))
inlined_syntax = ast_generalizer.generalize(inlined_xml)
writer.write_file(typed_astunparse.dump(inlined_syntax),
output_inlined_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(inlined_syntax),
output_inlined_path.with_suffix('.py'))
writer.write_file(f_unparser.unparse(inlined_syntax),
output_inlined_path.with_suffix('.f95'))
target_code = reader.read_file(target_path)
target_xml = parser.parse(target_code, target_path)
# import ipdb; ipdb.set_trace()
target_xml = target_xml.findall('.//call')[index]
writer.write_file(ET.tostring(target_xml, 'utf-8').decode(),
output_target_path.with_suffix('.xml'))
target_syntax = ast_generalizer.generalize(target_xml)
writer.write_file(typed_astunparse.dump(target_syntax),
output_target_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(target_syntax),
output_target_path.with_suffix('.py'))
writer.write_file(f_unparser.unparse(target_syntax),
output_target_path.with_suffix('.f95'))
mock_function = typed_ast3.FunctionDef(
'f', typed_ast3.arguments([], None, [], None, [], []),
[typed_ast3.Expr(target_syntax)], [], None, None)
output_syntax = inline_syntax(mock_function, inlined_syntax, globals_=globals())
output_syntax = st.augment(typed_ast3.Module(output_syntax.body, []), eval_=False)
writer.write_file(typed_astunparse.dump(output_syntax),
output_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(output_syntax),
output_path.with_suffix('.py'))
output_code = f_unparser.unparse(output_syntax)
writer.write_file(output_code, output_path.with_suffix('.f95'))
_LOG.warning('[%s %s] <- %i', target_xml.attrib['line_begin'],
target_xml.attrib['line_end'], len(output_code))
total_code = replace_scope(
target_code, int(target_xml.attrib['line_begin']),
int(target_xml.attrib['line_end']) + extra_lines, output_code)
writer.write_file(total_code, output_path)
def insert_comment_tokens_approx(
tree: typed_ast.ast3.AST, tokens: t.List[tokenize.TokenInfo]) -> typed_ast.ast3.AST:
assert isinstance(tree, typed_ast.ast3.AST)
assert isinstance(tokens, list)
token_locations = get_token_locations(tokens)
_LOG.debug('token locations: %s', token_locations)
nodes = ast_to_list(tree, only_localizable=True)
if not nodes and tokens:
_LOG.debug('overwriting empty AST with simplest editable tree')
tree = typed_ast.ast3.Module(body=[], type_ignores=[], lineno=1, col_offset=0)
nodes = ast_to_list(tree, only_localizable=True)
node_locations = get_ast_node_locations(nodes)
_LOG.debug('node locations: %s', node_locations)
node_locations_iter = enumerate(node_locations)
token_insertion_indices = []
tokens_eol_status = []
for token_index, token_location in enumerate(token_locations):
eol_comment_here = False
try:
node_index, node_location = next(node_locations_iter)
except StopIteration:
node_index = len(node_locations)
node_location = None
while node_location is not None:
token_line, _ = token_location
node_line, _ = node_location
if node_line > token_line:
break
if node_line == token_line:
eol_comment_here = True
if node_index < len(node_locations) - 1:
next_node_line, _ = node_locations[node_index + 1]
if next_node_line == token_line:
eol_comment_here = False
# if eol_comment_here:
# raise NotImplementedError(
# 'code "{}" and comment "{}" in line {}'
# ' -- only whole line comments are currently supported'
# .format(typed_astunparse.unparse(nodes[node_index]).strip(),
# tokens[token_index].string, node_line))
try:
node_index, node_location = next(node_locations_iter)
except StopIteration:
node_index = len(node_locations)
break
tokens_eol_status.append(eol_comment_here)
token_insertion_indices.append(node_index)
_LOG.debug('token insertion indices: %s', token_insertion_indices)
_LOG.debug('tree before insertion:\n"""\n%s\n"""', typed_astunparse.dump(tree))
_LOG.debug('code before insertion:\n"""\n%s\n"""', typed_astunparse.unparse(tree).strip())
for token_index, token_insertion_index in reversed(list(enumerate(token_insertion_indices))):
token = tokens[token_index]
eol = tokens_eol_status[token_index]
comment = Comment.from_token(token, eol)
if token_insertion_index == 0:
anchor = nodes[token_insertion_index]
before_anchor = True
elif token_insertion_index == len(node_locations):
anchor = nodes[-1]
before_anchor = False
else:
anchor = nodes[token_insertion_index - 1]
before_anchor = False
_LOG.debug('inserting %s %s %s', comment, 'before' if before_anchor else 'after', anchor)
tree = insert_in_tree(tree, comment, anchor=anchor, before_anchor=before_anchor)
_LOG.debug('tree after insertion:\n"""\n%s\n"""', typed_astunparse.dump(tree))
# _LOG.warning('code after insertion:\n"""\n%s\n"""', typed_astunparse.unparse(tree).strip())
return tree
def dump(self, tree) -> str:
return typed_astunparse.dump(tree)
tree from source (above) != example tree (below)
"""
{}
"""'''
for _description, _example in EXAMPLES.items():
for mode in MODES:
if _example['trees'][mode] is None:
try:
tree_from_source = typed_ast.ast3.parse(
source=_example['code'], filename='<string>', mode=mode)
except SyntaxError:
tree_from_source = None
example_tree = None
else:
tree_from_source = typed_astunparse.dump(
typed_ast.ast3.parse(source=_example['code'], filename='<string>', mode=mode)
).replace('\n', '').replace(' ', '')
example_tree = typed_astunparse.dump(
_example['trees'][mode]).replace('\n', '').replace(' ', '')
assert tree_from_source == example_tree, _MSG.format(
_description, mode, tree_from_source, example_tree)
_ROOT_DIRECTORY_PARTS = [getattr(sys, 'real_prefix', sys.prefix), 'lib']
if platform.system() != 'Windows':
_ROOT_DIRECTORY_PARTS.append('python{}.{}'.format(*sys.version_info[:2]))
_ROOT_DIRECTORY = os.path.join(*_ROOT_DIRECTORY_PARTS)
# verify root directory
if __debug__:
assert isinstance(_ROOT_DIRECTORY, str), _ROOT_DIRECTORY
def dump(tree: Module) -> str:
return typed_astunparse.dump(tree)
def _IfExp(self, t):
raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
def _Call(self, t):
if getattr(t, 'fortran_metadata', {}).get('is_procedure_call', False):
self.write('call ')
func_name = horast.unparse(t.func).strip()
if has_annotation(t, 'is_mpi_call'):
self.write('call ')
# val = last_attribute_value(t.func)
# fname = t.func.attr
# assert isinstance(val, typed_ast3.Name), type(val)
func_name = MPI_PYTHON_TO_FORTRAN[t.func.attr]
# attribute = t.func
# while isinstance(attribute.value, typed_ast3.Attribute):
# attribute = attribute.value
components = attribute_chain_components(t.func)[:-1]
t.func = typed_ast3.Name(func_name, typed_ast3.Load())
t.args += components
# print()
# self.write(func_name)
# raise
# raise NotImplementedError(self.f.getvalue())
elif func_name.startswith('Fortran.file_handles['):
t = copy.copy(t)
for suffix in ('read', 'close'):
if func_name.endswith('].{}'.format(suffix)):
t.args.insert(0, t.func.value.slice.value)
t.func = typed_ast3.Name(id=suffix, ctx=typed_ast3.Load())
break
# if func_name.endswith('].read'):
# t.func = typed_ast3.Name(id='read', ctx=typed_ast3.Load())
# elif func_name.endswith('].close'):
# t.func = typed_ast3.Name(id='close', ctx=typed_ast3.Load())
else:
raise NotImplementedError(func_name)
elif func_name.endswith('.format'):
t = copy.copy(t)
prefix, _, label = t.func.value.id.rpartition('_')
assert prefix == 'format_label', prefix
self.write(label)
self.write(' ')
t.func = typed_ast3.Name(id='format', ctx=typed_ast3.Load())
elif func_name.endswith('.rstrip'):
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='trim', ctx=typed_ast3.Load())
elif func_name.endswith('.sum'):
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='count', ctx=typed_ast3.Load())
elif func_name.endswith('.size'):
_LOG.warning('assuming np.size()')
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='size', ctx=typed_ast3.Load())
elif func_name.endswith('.shape'):
_LOG.warning('assuming np.shape()')
t = copy.copy(t)
t.args[0].n += 1
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='size', ctx=typed_ast3.Load())
elif func_name in PYTHON_FORTRAN_INTRINSICS \
and not getattr(t, 'fortran_metadata', {}).get('is_transformed', False):
new_func = PYTHON_FORTRAN_INTRINSICS[func_name]
if isinstance(new_func, collections.abc.Callable):
self.dispatch(new_func(t))
return
t = copy.copy(t)
t.func = typed_ast3.Name(id=new_func, ctx=typed_ast3.Load())
elif func_name.startswith('np.'):
raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
if func_name not in ('print',):
super()._Call(t)
return
self.dispatch(t.func)
self.write(' ')
comma = False
for arg in itertools.chain(t.args, t.keywords):
if comma:
self.write(", ")
else:
comma = True
self.dispatch(arg)
def view(scope):
"""Use this as a decorator for functions with observed values."""
_LOG.debug("parameters.core.view got a scope to handle")
# parse the srouruce and register all type-hinted vars
frame_info = inspect.getouterframes(inspect.currentframe())[1]
caller_frame = frame_info[0]
globals_ = caller_frame.f_globals
locals_ = caller_frame.f_locals
code = inspect.getsource(scope)
code = textwrap.dedent(code)
syntax = typed_ast3.parse(code)
function = syntax.body[0]
# decorators = function.decorator_list
function.decorator_list = []
function = st.augment(function,
locals_=locals_,
globals_={
**globals_,
**globals()
})
# _LOG.warning('%s', function._local_vars)
for var_name, var_types in function._local_vars.items():
for var_type in var_types:
if var_type is Observed or isinstance(var_type, Observed):
raise NotImplementedError(
'local variables currently cannot be observed')
instrumented_targets = {}
# _LOG.warning('%s', function._nonlocal_assignments)
for target, var_types in function._nonlocal_assignments.items():
for var_type in var_types:
if var_type is Observed or isinstance(var_type, Observed):
assert isinstance(target, typed_ast3.Attribute), type(target)
obj = typed_astunparse.unparse(target.value).strip()
attr = target.attr
_LOG.debug(
'assignment to %s of %s within %s will be instrumented',
attr, obj, function.name)
obj_attr = typed_astunparse.unparse(target).strip()
# instrumented_targetstarget)
instrumented_targets[obj_attr] = (obj, attr)
protonn.parameters._parameters[obj_attr] = None
#return scope
class Instrumenter(st.ast_manipulation.RecursiveAstTransformer[ast]):
def visit_node(self, node):
if not isinstance(node, ast.Assign) or len(node.targets) != 1 \
or not isinstance(node.targets[0], ast.Attribute):
return node
obj_attr = typed_astunparse.unparse(node.targets[0]).strip()
if obj_attr not in instrumented_targets:
# _LOG.warning('discarding candidate %s', obj_attr)
return node
obj, attr = instrumented_targets[obj_attr]
instrumentation = ast.parse(
'protonn.parameters.core._observe({}, {}, {})'.format(
repr(obj_attr), obj, repr(attr)),
mode='eval')
return [node, ast.Expr(instrumentation.body)]
syntax = ast.parse(code)
function = syntax.body[0]
function.decorator_list = []
function = Instrumenter(fields_first=True).visit(function)
# transcriber = st.ast_manipulation.AstTranscriber[typed_ast3, ast]()
# syntax = transcriber.visit(syntax)
flatten_syntax(syntax)
_LOG.debug('%s', typed_astunparse.dump(function))
syntax = ast.fix_missing_locations(syntax)
expression = compile(syntax, '<instrumented-function>', 'exec')
empty_locals_ = {}
eval(expression, globals_, empty_locals_)
assert len(empty_locals_) == 1
return next(iter(empty_locals_.values()))
tree from source (above) != example tree (below)
"""
{}
"""'''
for _description, _example in EXAMPLES.items():
for mode in MODES:
if _example['trees'][mode] is None:
try:
tree_from_source = typed_ast.ast3.parse(
source=_example['code'], filename='<string>', mode=mode)
except SyntaxError:
tree_from_source = None
example_tree = None
else:
tree_from_source = typed_astunparse.dump(
typed_ast.ast3.parse(source=_example['code'], filename='<string>', mode=mode)
).replace('\n', '').replace(' ', '')
example_tree = typed_astunparse.dump(
_example['trees'][mode]).replace('\n', '').replace(' ', '')
assert tree_from_source == example_tree, _MSG.format(
_description, mode, tree_from_source, example_tree)
_ROOT_DIRECTORY_PARTS = [getattr(sys, 'real_prefix', sys.prefix), 'lib']
if platform.system() != 'Windows':
_ROOT_DIRECTORY_PARTS.append('python{}.{}'.format(*sys.version_info[:2]))
_ROOT_DIRECTORY = os.path.join(*_ROOT_DIRECTORY_PARTS)
# verify root directory
if __debug__:
assert isinstance(_ROOT_DIRECTORY, str), _ROOT_DIRECTORY
def _Compare(self, t):
if len(t.ops) > 1 or len(t.comparators) > 1 \
or any([o.__class__.__name__ not in self.cmpops for o in t.ops]):
raise NotImplementedError('not yet implemented: {}'.format(
typed_astunparse.dump(t)))
super()._Compare(t)
def _arguments(self, t):
if t.vararg or t.kwonlyargs or t.kw_defaults or t.kwarg or t.defaults:
raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
super()._arguments(t)
