def p_error(self, p): p_val = p.value if p else '' err = PythranSyntaxError( "Invalid Pythran spec near '{}'".format(p_val)) err.lineno = self.lexer.lineno if self.input_file: err.filename = self.input_file raise err
def visit_Assign(self, node): """ Create Assign node for final Cxx representation. It tries to handle multi assignment like: >> a = b = c = 2 If only one local variable is assigned, typing is added: >> int a = 2; TODO: Handle case of multi-assignement for some local variables. Finally, process OpenMP clause like #pragma omp atomic """ if not all( isinstance(n, (ast.Name, ast.Subscript)) for n in node.targets): raise PythranSyntaxError( "Must assign to an identifier or a subscript", node) value = self.visit(node.value) targets = [self.visit(t) for t in node.targets] alltargets = "= ".join(targets) islocal = (len(targets) == 1 and isinstance(node.targets[0], ast.Name) and node.targets[0].id in self.scope[node] and node.targets[0].id not in self.openmp_deps) if islocal: # remove this decls from local decls self.ldecls.difference_update(t.id for t in node.targets) # add a local declaration if self.types[node.targets[0]].iscombined(): alltargets = '{} {}'.format(self.typeof(node.targets[0]), alltargets) elif isinstance(self.types[node.targets[0]], self.types.builder.Assignable): alltargets = '{} {}'.format( self.types.builder.AssignableNoEscape( self.types.builder.NamedType( 'decltype({})'.format(value))), alltargets) else: assert isinstance(self.types[node.targets[0]], self.types.builder.Lazy) alltargets = '{} {}'.format( self.types.builder.Lazy( self.types.builder.NamedType( 'decltype({})'.format(value))), alltargets) stmt = Assign(alltargets, value) return self.process_omp_attachements(node, stmt)
def access_path(node): if isinstance(node, ast.Name): return MODULES.get(demangle(node.id), node.id) elif isinstance(node, ast.Attribute): attr_key = demangle(node.attr) value_dict = Aliases.access_path(node.value) if attr_key not in value_dict: raise PythranSyntaxError( "Unsupported attribute '{}' for this object".format( attr_key), node.value) return value_dict[attr_key] elif isinstance(node, ast.FunctionDef): return node.name else: return node
def check_global(self, node, arg): if not isinstance(arg, ast.Call): return try: aliases = self.strict_aliases[arg.func] except KeyError: return for alias in aliases: if not isinstance(alias, ast.FunctionDef): continue if metadata.get(alias.body[0], metadata.StaticReturn): raise PythranSyntaxError( ("Cannot modify '{}': global variables are constant " "in pythran.").format(alias.name), node)
def visit_AugAssign(self, node): md.visit(self, node) self.visit(node.value) self.visit(node.target) var = node.target while isinstance(var, ast.Subscript): var = var.value if isinstance(var, ast.Name): var = var.id else: err = "AugAssign can't be used on {0}" raise PythranSyntaxError(err.format(var), node) last_node = self.current_node[var].pop() self.result[var].node[last_node]['action'] = "UD" self.current_node[var] = set([last_node])
def visit_Name(self, node): s = node.id if(isinstance(node.ctx, ast.Load) and s not in self.locals[node] and s not in self.globals and s in MODULES['__builtin__']): if s == 'getattr': raise PythranSyntaxError("You fool! Trying a getattr?", node) self.update = True return ast.Attribute( ast.Name('__builtin__', ast.Load()), s, node.ctx) else: return node
def visit_Assign(self, node): if not isinstance(node.value, ast.Name): return self.visit_assign(node) renaming = self.lookup(node.value.id) if not renaming: return self.visit_assign(node) if not is_mangled_module(renaming): return self.visit_assign(node) if any(not isinstance(target, ast.Name) for target in node.targets): raise PythranSyntaxError("Invalid module assignment", node) return node
def getsource(name, module_dir, level): # Try to load py file module_base = name.replace('.', os.path.sep) + '.py' if module_dir is None: assert level <= 0, "Cannot use relative path without module_dir" module_file = module_base else: module_file = os.path.sep.join( ([module_dir] + ['..'] * (level - 1) + [module_base])) try: with open(module_file, 'r') as fp: return fp.read() except IOError: raise PythranSyntaxError( "Module '{}' unknown and not found.".format(name))
def visit_AugAssign(self, node): md.visit(self, node) # augassigned variable can't be lazy self.visit(node.value) if isinstance(node.target, ast.Name): # variable is modified so other variables that use it dies self.modify(node.target.id) # and this variable can't be lazy self.result[node.target.id] = LazynessAnalysis.INF elif isinstance(node.target, ast.Subscript) or isattr(node.target): var_name = get_variable(node.target) # variable is modified so other variables that use it dies self.modify(var_name.id) # and this variable can't be lazy self.result[var_name.id] = LazynessAnalysis.INF else: raise PythranSyntaxError("AugAssign to unknown node", node)
def getsource(name, module_dir, level): # Try to load py file module_base = name.replace('.', os.path.sep) + '.py' if module_dir is None: assert level <= 0, "Cannot use relative path without module_dir" module_file = module_base else: module_file = os.path.sep.join( ([module_dir] + ['..'] * (level - 1) + [module_base])) try: with open(module_file, 'r') as fp: from pythran.frontend import raw_parse node = raw_parse(fp.read()) add_filename_field(node, name + ".py") return node except IOError: raise PythranSyntaxError("Module '{}' not found.".format(name))
def func_args_lazyness(self, func_name, args, node): for fun in self.aliases[func_name].aliases: if isinstance(fun, ast.Call): # call to partial functions self.func_args_lazyness(fun.args[0], fun.args[1:] + args, node) elif fun in self.argument_effects: # when there is an argument effet, we apply "modify" to the arg for i, arg in enumerate(self.argument_effects[fun]): # check len of args as default is 11 args if arg and len(args) > i: if isinstance(args[i], ast.Name): self.modify(args[i].id, node) elif isinstance(fun, ast.Name): # it may be a variable to a function. Lazyness will be compute # correctly thanks to aliasing continue else: raise PythranSyntaxError("Bad call in LazynessAnalysis", node)
def visit_For(self, node): md.visit(self, node) ids = self.passmanager.gather(Identifiers, node.iter, self.ctx) for id in ids: # iterate value can't be lazy self.result[id] = LazynessAnalysis.INF if isinstance(node.target, ast.Name): self.assign_to(node.target, ids, node.iter) self.result[node.target.id] = LazynessAnalysis.INF else: err = "Assignation in for loop not to a Name" raise PythranSyntaxError(err, node) self.in_loop = True map(self.visit, node.body) self.in_loop = False map(self.visit, node.orelse)
def generic_visit(self, node): if node in self.constant_expressions: try: fake_node = ast.Expression( node.value if isinstance(node, ast.Index) else node) code = compile(ast.gast_to_ast(fake_node), '<constant folding>', 'eval') value = eval(code, self.env) new_node = to_ast(value) if(isinstance(node, ast.Index) and not isinstance(new_node, ast.Index)): new_node = ast.Index(new_node) try: if not ASTMatcher(node).search(new_node): self.update = True return new_node except DamnTooLongPattern as e: print("W: ", e, " Assume no update happened.") return Transformation.generic_visit(self, node) except ConversionError as e: print('error in constant folding: ', e) raise except ToNotEval: return Transformation.generic_visit(self, node) except AttributeError as e: # FIXME union_ function is not handle by constant folding if "union_" in e.args[0]: return Transformation.generic_visit(self, node) elif "pythran" in e.args[0]: # FIXME: Can be fix giving a Python implementation for # these functions. return Transformation.generic_visit(self, node) raise except NameError as e: # FIXME dispatched function are not processed by constant # folding if "__dispatch__" in e.args[0]: return Transformation.generic_visit(self, node) raise except Exception as e: raise PythranSyntaxError(str(e), node) else: return Transformation.generic_visit(self, node)
def visit_Assign(self, node): md.visit(self, node) self.visit(node.value) ids = self.passmanager.gather(Identifiers, node.value, self.ctx) for target in node.targets: if isinstance(target, ast.Name): self.assign_to(target, ids, node.value) if node.value not in self.pure_expressions: self.result[target.id] = LazynessAnalysis.INF elif isinstance(target, ast.Subscript): # if we modify just a part of a variable, it can't be lazy var_name = get_variable(target) if isinstance(var_name, ast.Name): # variable is modified so other variables that use it dies self.modify(var_name.id, node.value) # and this variable can't be lazy self.result[var_name.id] = LazynessAnalysis.INF else: raise PythranSyntaxError("Assign to unknown node", node)
def is_is_not_none(expr): if not isinstance(expr, ast.Compare): return None if len(expr.ops) != 1: exprs = [expr.left] + expr.comparators if any(is_none(expr) for expr in exprs): raise PythranSyntaxError("is None in complex condition", expr) return None if not isinstance(expr.ops[0], (ast.NotEq, ast.IsNot)): return None if is_none(expr.left): return expr.comparators[0] if is_none(expr.comparators[0]): return expr.left return None
def visit_Call(self, node): self.generic_visit(node) func = node.func for alias in self.strict_aliases[func]: if not isinstance(alias, ast.FunctionDef): continue ubound = len(alias.args.args) lbound = ubound - len(alias.args.defaults) call_args_count = len(node.args) + len(node.keywords) if lbound <= call_args_count <= ubound: continue if lbound == ubound: msg = 'Invalid call to {}: expected {} arguments, got {}' msg = msg.format(alias.name, len(alias.args.args), len(node.args)) else: msg = ('Invalid {} call: ' 'expected between {} and {} arguments, got {}') msg = msg.format(alias.name, lbound, ubound, len(node.args)) raise PythranSyntaxError(msg, node)
def visit_Call(self, node): if node.keywords: self.update = True aliases = self.aliases[node.func] assert aliases, "at least one alias" # all aliases should have the same structural type... # call to self.handle_keywords raises an exception otherwise try: replacements = {} for func_alias in aliases: handle_special_calls(func_alias, node) if func_alias is None: # aliasing computation failed pass elif isinstance(func_alias, ast.Call): # nested function # func_alias looks like functools.partial(foo, a) # so we reorder using alias for 'foo' offset = len(func_alias.args) - 1 call = func_alias.args[0] for func_alias in self.aliases[call]: replacements = self.handle_keywords(func_alias, node, offset) else: replacements = self.handle_keywords(func_alias, node) # if we reach this point, we should have a replacement # candidate, or nothing structural typing issues would have # raised an exception in handle_keywords if replacements: for index, value in replacements.items(): node.args[index] = value node.keywords = [] except KeyError as ve: err = ("function uses an unknown (or unsupported) keyword " "argument `{}`".format(ve.args[0])) raise PythranSyntaxError(err, node) return self.generic_visit(node)
def generic_visit(self, node): if isinstance(node, ast.expr) and node in self.constant_expressions: fake_node = ast.Expression(node) code = compile(ast.gast_to_ast(fake_node), '<constant folding>', 'eval') try: value = eval(code, self.env) new_node = to_ast(value) try: if not ASTMatcher(node).search(new_node): self.update = True return new_node except DamnTooLongPattern as e: print("W: ", e, " Assume no update happened.") return Transformation.generic_visit(self, node) except ConversionError as e: print('error in constant folding: ', e) raise except ToNotEval: return Transformation.generic_visit(self, node) except AttributeError as e: # this may miss a few optimization logger.info('During constant folding, bailing out due to: ' + e.args[0]) return Transformation.generic_visit(self, node) except NameError as e: # FIXME dispatched function are not processed by constant # folding if "__dispatch__" in e.args[0]: return Transformation.generic_visit(self, node) # this may miss a few optimization logger.info('During constant folding, bailing out due to: ' + e.args[0]) return Transformation.generic_visit(self, node) except Exception as e: raise PythranSyntaxError(str(e), node) else: return Transformation.generic_visit(self, node)
def p_type(self, p): '''type : term | array_type | pointer_type | type LIST | type SET | type LPAREN opt_types RPAREN | type COLUMN type DICT | LPAREN types RPAREN | LARRAY type RARRAY | type OR type ''' if len(p) == 2: if isinstance(p[1], tuple): p[0] = p[1] else: p[0] = p[1], elif len(p) == 3 and p[2] == 'list': p[0] = tuple(List[t] for t in p[1]) elif len(p) == 3 and p[2] == 'set': p[0] = tuple(Set[t] for t in p[1]) elif len(p) == 5 and p[4] == ')': p[0] = tuple(Fun[args, r] for r in p[1] for args in (product(*p[3]) if len(p[3]) > 1 else p[3])) elif len(p) == 5: p[0] = tuple(Dict[k, v] for k in p[1] for v in p[3]) elif len(p) == 4 and p[2] == 'or': p[0] = p[1] + p[3] elif len(p) == 4 and p[3] == ')': p[0] = tuple(Tuple[t] for t in p[2]) elif len(p) == 4 and p[3] == ']': p[0] = p[2] else: raise PythranSyntaxError("Invalid Pythran spec. " "Unknown text '{0}'".format(p.value))
def rec(path, cur_module): """ Recursively rename path content looking in matching module. Prefers __module__ to module if it exists. This recursion is done as modules are visited top->bottom while attributes have to be visited bottom->top. """ err = "Function path is chained attributes and name" assert isinstance(path, (ast.Name, ast.Attribute)), err if isinstance(path, ast.Attribute): new_node, cur_module = rec(path.value, cur_module) new_id, mname = self.renamer(path.attr, cur_module) return (ast.Attribute(new_node, new_id, ast.Load()), cur_module[mname]) else: new_id, mname = self.renamer(path.id, cur_module) if mname not in cur_module: raise PythranSyntaxError( "Unbound identifier '{}'".format(mname), node) return (ast.Name(new_id, ast.Load(), None), cur_module[mname])
def visit_Call(self, node): if node.keywords: aliases = self.aliases[node.func].aliases assert aliases, "at least one alias" # all aliases should have the same structural type... # call to self.handle_keywords raises an exception otherwise try: replacements = {} for func_alias in aliases: if func_alias is None: # aliasing computation failed pass elif type(func_alias) is ast.Call: # nested function # func_alias looks like functools.partial(foo, a) # so we reorder using alias for 'foo' offset = len(func_alias.args) - 1 call = func_alias.args[0] for func_alias in self.aliases[call].aliases: replacements = self.handle_keywords(func_alias, node, offset) else: replacements = self.handle_keywords(func_alias, node) # if we reach this point, we should have a replacement # candidate, or nothing structural typing issues would have # raised an exception in handle_keywords if replacements: for index, value in replacements.iteritems(): node.args[index] = value node.keywords = [] except: err = "function aliases to incompatible types" raise PythranSyntaxError(err, node) return self.generic_visit(node)
def visit_Assign(self, node): """ Create Assign node for final Cxx representation. It tries to handle multi assignment like: >> a = b = c = 2 If only one local variable is assigned, typing is added: >> int a = 2; TODO: Handle case of multi-assignement for some local variables. Finally, process OpenMP clause like #pragma omp atomic """ if not all(isinstance(n, (ast.Name, ast.Subscript)) for n in node.targets): raise PythranSyntaxError( "Must assign to an identifier or a subscript", node) value = self.visit(node.value) targets = [self.visit(t) for t in node.targets] alltargets = "= ".join(targets) islocal = (len(targets) == 1 and isinstance(node.targets[0], ast.Name) and node.targets[0].id in self.scope[node]) if islocal and not self.yields: # remove this decl from local decls tdecls = {t.id for t in node.targets} self.ldecls = {d for d in self.ldecls if d.id not in tdecls} # add a local declaration alltargets = '{} {}'.format(self.local_types[node.targets[0]], alltargets) stmt = Assign(alltargets, value) return self.process_omp_attachements(node, stmt)
def visit_For(self, node): """ Create For representation for Cxx generation. Examples -------- >> for i in xrange(10): >> ... work ... Becomes >> typename returnable<decltype(__builtin__.xrange(10))>::type __iterX = __builtin__.xrange(10); >> ... possible container size reservation ... >> for (typename decltype(__iterX)::iterator::reference i: __iterX) >> ... the work ... This function also handle assignment for local variables. We can notice that three kind of loop are possible: - Normal for loop on iterator - Autofor loop. - Normal for loop using integer variable iteration Kind of loop used depend on OpenMP, yield use and variable scope. """ if not isinstance(node.target, ast.Name): raise PythranSyntaxError( "Using something other than an identifier as loop target", node.target) target = self.visit(node.target) # Handle the body of the for loop loop_body = Block(map(self.visit, node.body)) # Declare local variables at the top of the loop body loop_body = self.process_locals(node, loop_body, node.target.id) iterable = self.visit(node.iter) if self.can_use_c_for(node): header, loop = self.gen_c_for(node, target, loop_body) else: # Iterator declaration local_iter = "__iter{0}".format(len(self.break_handlers)) local_iter_decl = Assignable(DeclType(iterable)) self.handle_omp_for(node, local_iter) # For yield function, iterable is globals. if self.yields: self.extra_declarations.append(( local_iter, local_iter_decl, )) local_iter_decl = "" # Assign iterable # For C loop, it avoid issue if upper bound is reassign in the loop header = [ Statement("{0} {1} = {2}".format(local_iter_decl, local_iter, iterable)) ] if self.can_use_autofor(node): self.ldecls = { d for d in self.ldecls if d.id != node.target.id } autofor = AutoFor(target, local_iter, loop_body) loop = [self.process_omp_attachements(node, autofor)] else: loop = self.gen_for(node, target, local_iter, local_iter_decl, loop_body) # For xxxComprehension, it is replaced by a for loop. In this case, # pre-allocate size of container. for comp in metadata.get(node, metadata.Comprehension): header.append( Statement("pythonic::utils::reserve({0},{1})".format( comp.target, iterable))) return Block(header + loop)
def visit_TryFinally(self, node): err = ("This node should have been removed in previous passes") raise PythranSyntaxError(err, node)
def visit_For(self, node): """ Create For representation for Cxx generation. Examples -------- >> for i in xrange(10): >> ... work ... Becomes >> typename returnable<decltype(__builtin__.xrange(10))>::type __iterX = __builtin__.xrange(10); >> ... possible container size reservation ... >> for (auto&& i: __iterX) >> ... the work ... This function also handle assignment for local variables. We can notice that three kind of loop are possible: - Normal for loop on iterator - Autofor loop. - Normal for loop using integer variable iteration Kind of loop used depend on OpenMP, yield use and variable scope. """ if not isinstance(node.target, ast.Name): raise PythranSyntaxError( "Using something other than an identifier as loop target", node.target) target = self.visit(node.target) # Handle the body of the for loop loop_body = Block([self.visit(stmt) for stmt in node.body]) # Declare local variables at the top of the loop body loop_body = self.process_locals(node, loop_body, node.target.id) iterable = self.visit(node.iter) if self.can_use_c_for(node): header, loop = self.gen_c_for(node, target, loop_body) else: if self.can_use_autofor(node): header = [] self.ldecls.remove(node.target.id) autofor = AutoFor(target, iterable, loop_body) loop = [self.process_omp_attachements(node, autofor)] else: # Iterator declaration local_iter = "__iter{0}".format(id(node)) local_iter_decl = self.types.builder.Assignable( self.types[node.iter]) self.handle_omp_for(node, local_iter) # Assign iterable # For C loop, it avoids issues # if the upper bound is assigned in the loop asgnt = self.make_assign(local_iter_decl, local_iter, iterable) header = [Statement(asgnt)] loop = self.gen_for(node, target, local_iter, local_iter_decl, loop_body) # For xxxComprehension, it is replaced by a for loop. In this case, # pre-allocate size of container. for comp in metadata.get(node, metadata.Comprehension): header.append( Statement("pythonic::utils::reserve({0},{1})".format( comp.target, iterable))) return Block(header + loop)
def visit_Call(self, node): if isinstance(node.func, ast.Name): renaming = self.lookup(node.func.id) if renaming and is_mangled_module(renaming): raise PythranSyntaxError("Invalid module call", node) return self.generic_visit(node)
def visit_Cond(self, node): ''' generic expression splitting algorithm. Should work for ifexp and if using W(rap) and U(n)W(rap) to manage difference between expr and stmt The idea is to split a BinOp in three expressions: 1. a (possibly empty) non-static expr 2. an expr containing a static expr 3. a (possibly empty) non-static expr Once split, the if body is refactored to keep the semantic, and then recursively split again, until all static expr are alone in a test condition ''' NodeTy = type(node) if NodeTy is ast.IfExp: def W(x): return x def UW(x): return x else: def W(x): return [x] def UW(x): return x[0] has_static_expr = self.gather(HasStaticExpression, node.test) if not has_static_expr: return self.generic_visit(node) if node.test in self.static_expressions: return self.generic_visit(node) if not isinstance(node.test, ast.BinOp): return self.generic_visit(node) before, static = [], [] values = [node.test.right, node.test.left] def has_static_expression(n): return self.gather(HasStaticExpression, n) while values and not has_static_expression(values[-1]): before.append(values.pop()) while values and has_static_expression(values[-1]): static.append(values.pop()) after = list(reversed(values)) test_before = NodeTy(None, None, None) if before: assert len(before) == 1 test_before.test = before[0] test_static = NodeTy(None, None, None) if static: test_static.test = static[0] if len(static) > 1: if after: assert len(after) == 1 after = [ast.BinOp(static[1], node.test.op, after[0])] else: after = static[1:] test_after = NodeTy(None, None, None) if after: assert len(after) == 1 test_after.test = after[0] if isinstance(node.test.op, ast.BitAnd): if after: test_after.body = deepcopy(node.body) test_after.orelse = deepcopy(node.orelse) test_after = W(test_after) else: test_after = deepcopy(node.body) if static: test_static.body = test_after test_static.orelse = deepcopy(node.orelse) test_static = W(test_static) else: test_static = test_after if before: test_before.body = test_static test_before.orelse = node.orelse node = test_before else: node = UW(test_static) elif isinstance(node.test.op, ast.BitOr): if after: test_after.body = deepcopy(node.body) test_after.orelse = deepcopy(node.orelse) test_after = W(test_after) else: test_after = deepcopy(node.orelse) if static: test_static.body = deepcopy(node.body) test_static.orelse = test_after test_static = W(test_static) else: test_static = test_after if before: test_before.body = deepcopy(node.body) test_before.orelse = test_static node = test_before else: node = UW(test_static) else: raise PythranSyntaxError("operator not supported in a static if", node) self.update = True return self.generic_visit(node)
def visit_Module(self, node): """Turn globals assignment to functionDef and visit function defs. """ module_body = list() symbols = set() # Gather top level assigned variables. for stmt in node.body: if isinstance(stmt, (ast.Import, ast.ImportFrom)): for alias in stmt.names: name = alias.asname or alias.name symbols.add(name) # no warning here elif isinstance(stmt, ast.FunctionDef): if stmt.name in symbols: raise PythranSyntaxError( "Multiple top-level definition of %s." % stmt.name, stmt) else: symbols.add(stmt.name) if not isinstance(stmt, ast.Assign): continue for target in stmt.targets: if not isinstance(target, ast.Name): raise PythranSyntaxError( "Top-level assignment to an expression.", target) if target.id in self.to_expand: raise PythranSyntaxError( "Multiple top-level definition of %s." % target.id, target) if isinstance(stmt.value, ast.Name): if stmt.value.id in symbols: continue # create aliasing between top level symbols self.to_expand.add(target.id) for stmt in node.body: if isinstance(stmt, ast.Assign): # that's not a global var, but a module/function aliasing if all( isinstance(t, ast.Name) and t.id not in self.to_expand for t in stmt.targets): module_body.append(stmt) continue self.local_decl = set() cst_value = GlobalTransformer().visit(self.visit(stmt.value)) for target in stmt.targets: assert isinstance(target, ast.Name) module_body.append( ast.FunctionDef( target.id, ast.arguments([], [], None, [], [], None, []), [ast.Return(value=cst_value)], [], None, None)) metadata.add(module_body[-1].body[0], metadata.StaticReturn()) else: self.local_decl = self.gather(LocalNameDeclarations, stmt) module_body.append(self.visit(stmt)) self.update |= bool(self.to_expand) node.body = module_body return node
def visit_TryFinally(self, node): """ Assert TryFinally node are already removed before use_def_chain.""" err = ("This node should have been removed in previous passes") raise PythranSyntaxError(err, node)
def check_assert_with_side_effect(self, node, arg): if self.inassert: raise PythranSyntaxError( "Cannot call a function with side effect " "in an assert", node)