def _emulate_yield_from(self, targets: Optional[List[ast.Name]], node: ast.YieldFrom) -> Iterable[ast.AST]: generator = ast.Name( id='_py_backwards_generator_{}'.format(self._name_suffix)) exception = ast.Name( id='_py_backwards_generator_exception_{}'.format(self._name_suffix)) yield ast.Assign(targets=[generator], value=ast.Call(func=ast.Name(id='iter'), args=[node.value], keywords=[])) assign_to_targets = [ ast.If(test=ast.Call(func=ast.Name(id='hasattr'), args=[ exception, ast.Str(s='value'), ], keywords=[]), body=[ ast.Assign(targets=targets, value=ast.Attribute( value=exception, attr='value')), ], orelse=[]), ast.Break()] if targets else [ast.Break()] yield ast.While(test=ast.NameConstant(value=True), body=[ ast.Try(body=[ ast.Expr(value=ast.Yield(value=ast.Call( func=ast.Name(id='next'), args=[generator], keywords=[]))), ], handlers=[ ast.ExceptHandler( type=ast.Name(id='StopIteration'), name=exception.id, body=assign_to_targets), ], orelse=[], finalbody=[]), ], orelse=[]) self._name_suffix += 1
def assign_target(self, node, value): if isinstance(value.tp, EmptyList): func_node = ast.Expr(value=ast.Call( func=ast.Attribute(value=node, attr="clear"), args=[])) try: self.visit(func_node) except UnknownVariable: raise StaticTypeError( "Must specify a list type when assigning an empty list") return if isinstance(node, ast.Name): left = self.context.assign_type(node.id, value.tp) self.start_line("{} = {};\n".format(left.code, value.as_value())) elif isinstance(node, ast.Subscript): container = self.get_expression_code(node.value) if isinstance(node.slice, ast.Index): index = self.get_expression_code(node.slice.value) setter = container.tp.get_method("set_item") code = setter.get_code(self.context, container, index, value) self.start_line(f"{code.code};\n") else: raise UnimplementedFeature("Slices not yet implemented") elif isinstance(node, ast.Attribute): owner = self.get_expression_code(node.value) owner.tp.set_attr(node.attr, value) left = self.get_expression_code(node) self.start_line(f"{left.code} = {value.code};\n")
def _Class(self, node: ET.Element): # pylint: disable=invalid-name context = node.attrib['context'] assert context in self.namespaces, context cls_name = node.attrib['name'] if '<' in cls_name: _LOG.warning('processing template class %s', cls_name) assert '>' in cls_name cls_name, _, rest = cls_name.partition('<') rest = rest[:-1] generic_args = [_.strip() for _ in rest.split(',')] _LOG.warning('found generic args: %s', generic_args) full_name = '{}::{}'.format(self.namespaces[context], cls_name) is_stl_class = full_name in CPP_STL_CLASSES and generic_args value_type = None body = [] for member_id in node.attrib['members'].split(): if not is_stl_class: # TODO: handle non-STL classes too break if member_id not in self.all_types: continue member_type = self.all_types[member_id] if member_type.tag == 'Typedef' and member_type.attrib[ 'name'] == 'value_type': referenced_id = member_type.attrib['type'] assert referenced_id in self.all_types if referenced_id not in self.relevant_types \ and referenced_id not in self._new_relevant_types: self._new_relevant_types[referenced_id] = self.all_types[ referenced_id] _LOG.debug( 'type marked as relevant due to being container value type %s', ET.tostring( self.all_types[referenced_id]).decode().rstrip()) body.append(typed_ast3.Expr(typed_ast3.Str(referenced_id, ''))) value_type = referenced_id ''' if member_id not in self.relevant_types and member_id not in self._new_relevant_types: self._new_relevant_types[member_id] = member_type _LOG.warning('marked %s as relevant type', ET.tostring(member_type).decode().rstrip()) body.append(typed_ast3.Expr(typed_ast3.Str(member_id, ''))) ''' base_class = typed_ast3.parse(CPP_PYTHON_CLASS_PAIRS[full_name], mode='eval').body if is_stl_class: assert value_type is not None base_class = typed_ast3.Subscript( value=base_class, slice=typed_ast3.Index(typed_ast3.Str(value_type, '')), ctx=typed_ast3.Load()) return base_class
def fix_stmts_in_body(stmts: t.List[typed_ast3.AST]) -> t.List[typed_ast3.AST]: assert isinstance(stmts, list) if not stmts: return [typed_ast3.Pass()] return [ typed_ast3.Expr(value=stmt) if isinstance(stmt, STANDALONE_EXPRESSION_TYPES) else stmt for stmt in stmts ]
def _show_store_contents_expr(self) -> ast3.Expr: """Returns an ast3.Expr which prints the value of the store in the screen. Useful for debugging. """ # print(st) return ast3.Expr(value=ast3.Call( func=ast3.Name(id='print', ctx=ast3.Load()), args=[ast3.Name(id='st', ctx=ast3.Load())], keywords=[], ), )
def visit_Exec(self, n): new_globals = self.maybe_visit(n.globals) if new_globals is None: new_globals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1) new_locals = self.maybe_visit(n.locals) if new_locals is None: new_locals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1) return ast3.Expr(ast3.Call(ast3.Name("exec", ast3.Load(), lineno=n.lineno, col_offset=-1), [self.visit(n.body), new_globals, new_locals], [], lineno=n.lineno, col_offset=-1))
def visit_Print(self, n): keywords = [] if n.dest is not None: keywords.append(ast3.keyword("file", self.visit(n.dest))) if not n.nl: keywords.append(ast3.keyword("end", ast3.Str(" ", lineno=n.lineno, col_offset=-1))) return ast3.Expr(ast3.Call(ast3.Name("print", ast3.Load(), lineno=n.lineno, col_offset=-1), self.visit(n.values), keywords, lineno=n.lineno, col_offset=-1))
def visit_Expr(self, node: ast3.Expr) -> VisitorOutput: """Only the internal parts of an Expr are modified, an Expr keeps being an Expr""" self.generic_visit(node) # In console mode ("single" for Python's compile) any expression statement should # print to console if self.console: return ast3.Expr(value=ast3.Call( func=ast3.Name(id='print_console', ctx=ast3.Load()), args=[node.value], keywords=[], ), ) return node
def visit_ListComp(self, node): from parser.functions import FunctionImplementation # calculate result type if len(node.generators) > 1: raise InvalidOperation( "Only one for statement permitted in comprehensions") comp = node.generators[0] if len(comp.ifs) > 1: raise InvalidOperation( "Only one if statement allowed in List Comprehension") assign_node = ast.Assign(targets=[comp.target], value=ast.Subscript(value=comp.iter, slice=ast.Index( ast.Num(0)))) return_node = ast.Return(value=node.elt) function_node = ast.FunctionDef(name="temp", args=ast.arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[assign_node, return_node]) function_interpreter = FunctionImplementation(function_node, (), self.context) result_type = TypeDB.get_list([function_interpreter.retval.tp]) # create temp list to hold values result = self.context.get_temp_var(result_type) self.prepends.append( f"{result.code} = {result_type.as_literal([])};\n") # create for expression append_node = ast.Expr( ast.Call(func=ast.Attribute(value=ast.Name(id=result.code, ctx=ast.Load()), attr="append", ctx=ast.Load()), args=[node.elt], keywords=[])) if comp.ifs: body = ast.If(test=comp.ifs[0], body=[append_node], orelse=[]) else: body = append_node for_node = ast.For(target=comp.target, iter=comp.iter, body=[body], orelse=[]) self.prepends.append(for_node) return result
def _Function(self, node: ET.Element): # pylint: disable=invalid-name if not self.types.is_relevant(node): raise ContinueIteration() name = node.attrib['name'] arguments = typed_ast3.arguments( args=self.transform_all_subnodes(node), vararg=None, kwonlyargs=[], kwarg=None, defaults=[], kw_defaults=[]) body = [typed_ast3.Expr(value=typed_ast3.Ellipsis())] returns = self.types.resolved_types[node.attrib['returns']] return typed_ast3.FunctionDef(name=name, args=arguments, body=body, decorator_list=[], returns=returns)
def visit_AnnAssign(self, node: ast3.AnnAssign) -> VisitorOutput: """Transforms an assignment with annotation into a pytropos type hint assignment. For example, it converts:: var: ann = expr into:: `var` = pt.type_hint(`ann`, `expr`) """ if node.value is None: raise AstTransformerError( f"{self.filename}:{node.lineno}:{node.col_offset}: Fatal Error: " "Only annotated assignments are allowed (variables with initial values). " "I.e., no full support for PEP 526 yet. Sorry :(") pos = pos_as_tuple(node) # Deleting annotation :S self.generic_visit(node) # new_node = ast3.Assign(targets=[node.target], value=node.value) return ast3.Assign( targets=[node.target], value=ast3.Call( func=ast3.Attribute( value=ast3.Name(id='pt', ctx=ast3.Load()), attr='annotation', ctx=ast3.Load(), ), args=[ node.annotation, node.value, pos if pos else ast3.Expr( value=ast3.NameConstant(value=None)) ], keywords=[], ), )
def test_inline_flash_subset_hydro(self): app_name = 'FLASH-SUBSET' if app_name not in _APPS_ROOT_PATHS and app_name in _APPS_OPTIONAL: self.skipTest('{} directory not found'.format(app_name)) language = Language.find('Fortran') reader = CodeReader() parser = Parser.find(language)() ast_generalizer = AstGeneralizer.find(language)() f_unparser = Unparser.find(language)() py_unparser = Unparser.find(Language.find('Python'))() writer = CodeWriter() dir_name = app_name.lower() results_path = pathlib.Path(RESULTS_ROOT, 'transformations', 'inlining', dir_name) results_path.mkdir(parents=True, exist_ok=True) path_pairs = [ (pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux_loop.F90'), pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux.F90'), (1, 1)), (pathlib.Path('physics/Eos/EosMain/Eos_getData_loop1.F90'), pathlib.Path('physics/Eos/EosMain/Eos_getData.F90'), (1, 2))] for inlined_path, target_path, (index, extra_lines) in path_pairs: inlined_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', inlined_path) target_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', target_path) output_inlined_path = results_path.joinpath(inlined_path.name) output_target_path = results_path.joinpath(target_path.name) output_path = results_path.joinpath(target_path.with_suffix('').name + '_inlined.F90') inlined_xml = parser.parse('', inlined_path) inlined_xml = inlined_xml.find('.//subroutine') writer.write_file(ET.tostring(inlined_xml, 'utf-8').decode(), output_inlined_path.with_suffix('.xml')) inlined_syntax = ast_generalizer.generalize(inlined_xml) writer.write_file(typed_astunparse.dump(inlined_syntax), output_inlined_path.with_suffix('.ast.py')) writer.write_file(py_unparser.unparse(inlined_syntax), output_inlined_path.with_suffix('.py')) writer.write_file(f_unparser.unparse(inlined_syntax), output_inlined_path.with_suffix('.f95')) target_code = reader.read_file(target_path) target_xml = parser.parse(target_code, target_path) # import ipdb; ipdb.set_trace() target_xml = target_xml.findall('.//call')[index] writer.write_file(ET.tostring(target_xml, 'utf-8').decode(), output_target_path.with_suffix('.xml')) target_syntax = ast_generalizer.generalize(target_xml) writer.write_file(typed_astunparse.dump(target_syntax), output_target_path.with_suffix('.ast.py')) writer.write_file(py_unparser.unparse(target_syntax), output_target_path.with_suffix('.py')) writer.write_file(f_unparser.unparse(target_syntax), output_target_path.with_suffix('.f95')) mock_function = typed_ast3.FunctionDef( 'f', typed_ast3.arguments([], None, [], None, [], []), [typed_ast3.Expr(target_syntax)], [], None, None) output_syntax = inline_syntax(mock_function, inlined_syntax, globals_=globals()) output_syntax = st.augment(typed_ast3.Module(output_syntax.body, []), eval_=False) writer.write_file(typed_astunparse.dump(output_syntax), output_path.with_suffix('.ast.py')) writer.write_file(py_unparser.unparse(output_syntax), output_path.with_suffix('.py')) output_code = f_unparser.unparse(output_syntax) writer.write_file(output_code, output_path.with_suffix('.f95')) _LOG.warning('[%s %s] <- %i', target_xml.attrib['line_begin'], target_xml.attrib['line_end'], len(output_code)) total_code = replace_scope( target_code, int(target_xml.attrib['line_begin']), int(target_xml.attrib['line_end']) + extra_lines, output_code) writer.write_file(total_code, output_path)
def visit_ImportFrom(self, node: ast3.ImportFrom) -> VisitorOutput: """Defines how to import (from) modules (supported and nonsupported) For example, it converts:: from numpy import array from numpy import * from somelib import var, othervar as var2 from otherlib import * into:: from pytropos.libs_checking import numpy_module st['array'] = numpy_module.attr['array', pos...] from pytropos.libs_checking import numpy_module st.importStar(numpy_module) st['var'] = pt.Top st['var2'] = pt.Top st.importStar() """ libs: 'List[ast3.AST]' = [] if node.module in self._supported_modules: module_name = self._supported_modules[node.module] # from pytropos.libs_checking import module_name libs.append( ast3.ImportFrom( module='pytropos.libs_checking', names=[ast3.alias(name=module_name, asname=None)], level=0, )) if node.names[0].name == '*': # st.importStar(module_name) libs.append( ast3.Expr(value=ast3.Call( func=ast3.Attribute( value=ast3.Name(id='st', ctx=ast3.Load()), attr='importStar', ctx=ast3.Load(), ), args=[ast3.Name(id=module_name, ctx=ast3.Load())], keywords=[], ), )) else: for alias in node.names: # st['asname'] = modname.attr['name'] pos = pos_as_tuple(node) if pos is not None: attrname = ast3.Tuple( elts=[ast3.Str(s=alias.name), pos], ctx=ast3.Load()) # type: ast3.expr else: attrname = ast3.Str(s=alias.name) libs.append( ast3.Assign( targets=[ ast3.Subscript( value=ast3.Name(id='st', ctx=ast3.Load()), slice=ast3.Index(value=ast3.Str( s=alias.asname if alias. asname else alias.name), ), ctx=ast3.Store(), ), ], value=ast3.Subscript( value=ast3.Attribute( value=ast3.Name(id=module_name, ctx=ast3.Load()), attr='attr', ctx=ast3.Load(), ), slice=ast3.Index(value=attrname, ), ctx=ast3.Load(), ), )) else: if node.names[0].name == '*': # st.importStar() libs.append( ast3.Expr(value=ast3.Call( func=ast3.Attribute( value=ast3.Name(id='st', ctx=ast3.Load()), attr='importStar', ctx=ast3.Load(), ), args=[], keywords=[], ), )) else: libs.extend( ast3.parse( # type: ignore '\n'.join([ "st['{asname}'] = pt.Top".format( asname=alias.asname if alias.asname else alias. name) for alias in node.names ])).body) return libs