def autodiff_tree(func, wrt, motion, mode, preserve_result, check_dims, verbose): """Perform AD on all functions in a call tree. This function walks the call tree and differentiates each function in it. It also ensures that the global namespaces that each function in the call tree was in are merged. The `tangent` and `numpy` packages are added to the namespace here, so that the gradient templates can assume that they are present. Args: See `grad`. Returns: final: A single module which contains the primals and adjoints of all the functions in the call tree. namespace: A merged dictionary with all the variables in the global namespaces of each function. The primals and adjoints need access to these in order to execute. """ # Imported here to avoid circular imports import tangent namespace = {'tangent': tangent, 'numpy': numpy} done = set() final = gast.Module(body=[]) namespace.update(six.get_function_globals(func)) node, required = autodiff_ast(func, wrt, motion, mode, preserve_result, check_dims, verbose) final.body.extend(node.body) to_do = set(required) if motion == 'split' and mode == 'reverse': done.add((func, wrt)) to_do -= done while to_do: func, wrt = to_do.pop() namespace.update(six.get_function_globals(func)) node, required = autodiff_ast( func=func, wrt=wrt, motion='split', mode=mode, preserve_result=True, check_dims=False, verbose=verbose) final.body.extend(node.body) done.add((func, wrt)) to_do.update(required) to_do -= done return final, namespace
def _update_function(oldfunc, newfunc): """Update a function object.""" if _closure_changed(six.get_function_closure(oldfunc), six.get_function_closure(newfunc)): raise ClosureChanged() setattr(oldfunc, six._func_code, six.get_function_code(newfunc)) setattr(oldfunc, six._func_defaults, six.get_function_defaults(newfunc)) _update_scope(six.get_function_globals(oldfunc), six.get_function_globals(newfunc)) # XXX What else? return oldfunc
def _new_func_from_source(source, func): """ Create new function defined in source but maintain context from func @param func: The function whose global + local context we will use @param source: Python source code containing def statement """ src_str = ''.join(source) frames = inspect.getouterframes(inspect.currentframe()) calling_frame = frames[2][0] context = {} # My initial instict was: exec src_str in func.func_globals.items(), calling_frame.f_locals # however this seems to break the function closure so caveat here is that we create a new # function with the locals merged into the globals. # # Possible consequences I can think of: # - If a global exists that already has the same name as the local, it will be overwritten in # in the context of this function. This shouldnt matter though as the global should have already # been hidden by the new name? # # This functionality should be considered experimental as no idea what other consequences there # could be. # # relevant: http://stackoverflow.com/questions/2749655/why-are-closures-broken-within-exec globals = six.get_function_globals(func) locals = calling_frame.f_locals combined = globals.copy() combined.update(locals) Logger.debug('New src_str:\n %s' % src_str) six.exec_(src_str, combined, context) new_func = context[func.__name__] return new_func
def function_to_graph(f, conversion_map, param_value_hints, owner_type=None): """Specialization of `object_to_graph` for callable functions.""" node = parser.parse_object(f).body[0] node_globals = six.get_function_globals(f) # This is needed for non-global functions. closure = six.get_function_closure(f) if closure: for e in closure: if callable(e.cell_contents): fn = e.cell_contents node_globals[fn.__name__] = fn namer = conversion_map.new_namer(node_globals) node = node_to_graph(node, namer, node_globals, param_value_hints) # Simulate a rename to ensure the top level is in the name map. This is needed # for top level functions, and it also helps the consistency verification made # by update_name_map. if owner_type is not None: new_name = namer.compiled_function_name(f.__name__, f, owner_type) else: new_name = namer.compiled_function_name(f.__name__, f) node.name = new_name conversion_map.update_name_map(namer) return node, conversion_map.name_map[f]
def to_graph(o, arg_value_hints=None): """Compile a Python entity into equivalent TensorFlow code. Currently supported entities: * functions * classes Classes are handled by converting all their methods into a new class. Args: o: A Python function or class. arg_value_hints: A dict mapping parameter names to objects that can hint at the type of those parameters. Returns: A function with a signature identical to `o`, but which when executed it creates TF a graph that has the same functionality as the original entity. """ conversion_map = conversion.ConversionMap() _, name = conversion.object_to_graph(o, conversion_map, arg_value_hints) module = gast.Module([]) for import_line in config.COMPILED_IMPORT_STATEMENTS: module.body.append(parser.parse_str(import_line)) for dep in conversion_map.dependency_cache.values(): module.body.append(dep) compiled_node = compiler.ast_to_object(module) # The compiled code should see everything the entry function saw. # TODO(mdan): This might not work well if the call tree spans modules? if tf_inspect.isfunction(o): compiled_node.__dict__.update(six.get_function_globals(o)) compiled_fn = getattr(compiled_node, name) return compiled_fn
def wrapper(*args, **kwargs): func_name = func.__name__ caller_module_name = six.get_function_globals(func)['__name__'] logger.info("STARTED: {}.{}".format( caller_module_name, func_name)) args_name = getargspec(func)[0] args_dict = dict(zip(args_name, args)) for key, value in viewitems(args_dict): if key == "self": continue logger.debug(" {} == {}".format(key, value)) if len(args) > len(args_name): for i in range(len(args_name), len(args)): logger.debug(" {}".format(args[i])) for key, value in viewitems(kwargs): logger.debug(" {} == {}".format(key, value)) start = time.time() result = func(*args, **kwargs) end = time.time() logger.info( "COMPLETED: {}.{} in {}s".format( caller_module_name, func_name, round(end - start, 2))) return result
def __init__(self, func, local, variables, param, dtype=None, **kwargs): self.func = func self.code = get_function_code(func) self.dtype = 'float' if dtype == np.float32 else 'double' self.local = local self.param = param self.variables = variables self.float_char = 'f' if dtype == np.float32 else '' self.func_globals = get_function_globals(self.func) self.used_variables = set() CodeGenerator.__init__(self, self.func, newline=';\n', offset=4, \ ostream=StringIO()) self.generate() self.generate_signature() fname = self.func.__name__ self.definition = template_device_func.render( dtype = self.dtype, name = fname, local = self.local, signature = self.signature, used = self.used_variables, src = self.ostream.getvalue() ) self.invocation = "{}({});".format(fname, ", ".join(self.args))
def to_graph(f, arg_value_hints=None): """Compile a Python function into equivalent TensorFlow code. Args: f: A Python function with arbitrary arguments and return values. arg_value_hints: A dict mapping parameter names to objects that can hint at the type of those parameters. Returns: A function with a signature identical to `f`, but which when executed it creates TF a graph that has the same functionality as the original function. """ conversion_map = conversion.ConversionMap() _, name = conversion.object_to_graph(f, conversion_map, arg_value_hints) module = gast.Module([]) for import_line in config.COMPILED_IMPORT_STATEMENTS: module.body.append(parser.parse_str(import_line)) for dep in conversion_map.dependency_cache.values(): module.body.append(dep) compiled_node = compiler.ast_to_object(module) # The compiled code should see everything the entry function saw. # TODO(mdan): This might not work well if the call tree spans modules? compiled_node.__dict__.update(six.get_function_globals(f)) compiled_fn = getattr(compiled_node, name) return compiled_fn
def class_to_graph(c, conversion_map): """Specialization of `entity_to_graph` for classes.""" converted_members = {} members = tf_inspect.getmembers(c, predicate=tf_inspect.ismethod) if not members: raise ValueError('Cannot convert %s: it has no member methods.') class_globals = None for _, m in members: node, _ = function_to_graph( m, conversion_map=conversion_map, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) # TODO(mdan): Do not assume all members have the same view of globals. if class_globals is None: class_globals = six.get_function_globals(m) converted_members[m] = node namer = conversion_map.new_namer(class_globals) class_name = namer.compiled_class_name(c.__name__, c) node = gast.ClassDef( class_name, bases=[], keywords=[], body=converted_members.values(), decorator_list=[]) return node, class_name
def function_to_graph(f, conversion_map, arg_values, arg_types, owner_type=None): """Specialization of `entity_to_graph` for callable functions.""" node = parser.parse_object(f).body[0] namespace = six.get_function_globals(f) # This is needed for non-global functions. closure = six.get_function_closure(f) if closure: for e in closure: if callable(e.cell_contents): fn = e.cell_contents namespace[fn.__name__] = fn namer = conversion_map.new_namer(namespace) ctx = context.EntityContext( namer=namer, source_code=tf_inspect.getsource(f), source_file=tf_inspect.getfile(f), namespace=namespace, arg_values=arg_values, arg_types=arg_types) node = node_to_graph(node, ctx, conversion_map.nocompile_decorators) # Simulate a rename to ensure the top level is in the name map. This is needed # for top level functions, and it also helps the consistency verification made # by update_name_map. if owner_type is not None: new_name = namer.compiled_function_name(f.__name__, f, owner_type) else: new_name = namer.compiled_function_name(f.__name__, f) node.name = new_name conversion_map.update_name_map(namer) return node, conversion_map.name_map[f]
def construct_new_test_function(original_func, name, build_params): """Builds a new test function based on parameterized data. :param original_func: The original test function that is used as a template :param name: The fullname of the new test function :param build_params: A dictionary or list containing args or kwargs for the new test :return: A new function object """ new_func = types.FunctionType( six.get_function_code(original_func), six.get_function_globals(original_func), name=name, argdefs=six.get_function_defaults(original_func) ) # Support either an arg list or kwarg dict for our data build_args = build_params if isinstance(build_params, list) else [] build_kwargs = build_params if isinstance(build_params, dict) else {} # Build a test wrapper to execute with our kwargs def test_wrapper(func, test_args, test_kwargs): @functools.wraps(func) def wrapper(self): return func(self, *test_args, **test_kwargs) return wrapper return test_wrapper(new_func, build_args, build_kwargs)
def _build_new_function(func, name): code = six.get_function_code(func) func_globals = six.get_function_globals(func) func_defaults = six.get_function_defaults(func) func_closure = six.get_function_closure(func) return types.FunctionType(code, func_globals, name, func_defaults, func_closure)
def parse_args(args, path, query, specials): def one_or_many(fn_, dict_, key): result = [fn_(value) for value in dict_[key]] return result[0] if len(result) == 1 else result kwargs = {} for arg, parse_fn in six.iteritems(args): if arg in specials: kwargs[arg] = specials[arg]() elif parse_fn is None: kwargs[arg] = one_or_many(lambda x: x, query, arg) elif isinstance(parse_fn, tuple): kwargs[arg] = parse_fn[DEFAULT] if arg not in query else one_or_many(parse_fn[CALLABLE], query, arg) elif isalambda(parse_fn): _code = six.get_function_code(parse_fn) closures = six.get_function_closure(parse_fn) if closures: assert len(closures) <= 1 fn = closures[0].cell_contents else: fn = eval(".".join(_code.co_names), six.get_function_globals(parse_fn)) kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path, query, specials)) else: kwargs[arg] = one_or_many(parse_fn, query, arg) return kwargs
def parse_args(args, path, query, specials): def one_or_many(fn_, dict_, key): result = [fn_(value) for value in dict_[key]] return result[0] if len(result) == 1 else result kwargs = {} for arg, parse_fn in six.iteritems(args): if arg in specials: kwargs[arg] = specials[arg]() elif parse_fn is None: kwargs[arg] = one_or_many(lambda x: x, query, arg) elif isinstance(parse_fn, tuple): kwargs[ arg] = parse_fn[DEFAULT] if arg not in query else one_or_many( parse_fn[CALLABLE], query, arg) elif isalambda(parse_fn): _code = six.get_function_code(parse_fn) closures = six.get_function_closure(parse_fn) if closures: assert len(closures) <= 1 fn = closures[0].cell_contents else: fn = eval(".".join(_code.co_names), six.get_function_globals(parse_fn)) kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path, query, specials)) else: kwargs[arg] = one_or_many(parse_fn, query, arg) return kwargs
def __init__(self, model, **kwargs): self.num = kwargs.pop('num', None) ostream = StringIO() code_gen = NumpyKernelGenerator(model, model.ode, offset=4, ostream=ostream) code_gen.generate() post = model.__class__.post for cls in model.__class__.__bases__: if cls.__name__ == 'Model' and post != cls.post: code_gen = NumpyKernelGenerator(model, post, offset=4, ostream=ostream) code_gen.generate() break self.source = ostream.getvalue() self.func_globals = get_function_globals(model.ode) self.name = "Numpy{}".format(model.__class__.__name__) self.compile() self.ode = MethodType(self.module.ode, model) if 'post' in self.module.__dict__: self.post = MethodType(self.module.post, model)
def function_to_graph(f, conversion_map, param_value_hints): """Specialization of `object_to_graph` for callable functions.""" node = parser.parse_object(f).body[0] node_globals = six.get_function_globals(f) # This is needed for non-global functions. closure = six.get_function_closure(f) if closure: for e in closure: if callable(e.cell_contents): fn = e.cell_contents node_globals[fn.__name__] = fn namer = conversion_map.new_namer(node_globals) node = node_to_graph(node, namer, node_globals, param_value_hints) # Simulate a rename to ensure the top level is in the name map. This is needed # for top level functions, and it also helps the consistency verification made # by update_name_map. namer.compiled_function_name(f.__name__, f) conversion_map.add_to_cache(f, node) conversion_map.update_name_map(namer) # Recursively convert any remaining dependencies. for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: object_to_graph(obj, conversion_map, None) return node, conversion_map.name_map[f]
def class_to_graph(c, conversion_map, param_value_hints): """Specialization of `object_to_graph` for classes.""" converted_members = {} members = tf_inspect.getmembers(c, predicate=tf_inspect.ismethod) if not members: raise ValueError('Cannot convert %s: it has no member methods.') if 'self' in param_value_hints: raise ValueError('Hints may not be provided for reserved name "self".') param_value_hints['self'] = (c.__name__, c) class_globals = None for _, m in members: node, _ = function_to_graph(m, conversion_map, param_value_hints, c) # TODO(mdan): Do not assume all members have the same view of globals. if class_globals is None: class_globals = six.get_function_globals(m) converted_members[m] = node namer = conversion_map.new_namer(class_globals) class_name = namer.compiled_class_name(c.__name__, c) node = gast.ClassDef( class_name, bases=[], keywords=[], body=converted_members.values(), decorator_list=[]) return node, class_name
def __init__(self, model, func, dtype, **kwargs): self.dtype = dtype self.model = model self.func = func self.float_char = 'f' if self.dtype == 'float' else '' self.params_gdata = kwargs.pop('params_gdata', []) self.inputs = kwargs.pop('inputs', dict()) self.variables = [] self.has_random = False self.func_globals = get_function_globals(self.func) CodeGenerator.__init__(self, self.func, newline=';\n', offset=4, ostream=StringIO(), **kwargs) _, self.signature, self.kwargs = self.extract_signature(self.func) self.generate() self.args = self.process_signature() self.src = self.ostream.getvalue()
def getnamespace(f): """Returns the complete namespace of a function. Namespace is defined here as the mapping of all non-local variables to values. This includes the globals and the closure variables. Note that this captures the entire globals collection of the function, and may contain extra symbols that it does not actually use. Args: f: User defined function. Returns: A dict mapping symbol names to values. """ namespace = dict(six.get_function_globals(f)) closure = six.get_function_closure(f) freevars = six.get_function_code(f).co_freevars if freevars and closure: for name, cell in zip(freevars, closure): try: namespace[name] = cell.cell_contents except ValueError: # Cell contains undefined variable, omit it from the namespace. pass return namespace
def DataDrivenFixture(cls): """Generates new unittest test methods from methods defined in the decorated class""" if not issubclass(cls, TestCase): raise DataDrivenFixtureError test_case_attrs = dir(cls) for attr_name in test_case_attrs: if attr_name.startswith(DATA_DRIVEN_TEST_PREFIX) is False: # Not a data driven test, skip it continue original_test = getattr(cls, attr_name, None).__func__ test_data = getattr(original_test, DATA_DRIVEN_TEST_ATTR, None) if test_data is None: # no data was provided to the datasource decorator or this is not a # data driven test, skip it. continue for dataset in test_data: # Name the new test based on original and dataset names base_test_name = str( original_test.__name__)[int(len(DATA_DRIVEN_TEST_PREFIX)):] new_test_name = "test_{0}_{1}".format(base_test_name, dataset.name) # Create a new test from the old test new_test = FunctionType(six.get_function_code(original_test), six.get_function_globals(original_test), name=new_test_name) # Copy over any other attributes the original test had (mainly to # support test tag decorator) for attr in list(set(dir(original_test)) - set(dir(new_test))): setattr(new_test, attr, getattr(original_test, attr)) # Change the new test's default keyword values to the appropriate # new data as defined by the datasource decorator args, _, _, defaults = inspect.getargspec(original_test) # Self doesn't have a default, so we need to remove it args.remove('self') # Make sure we take into account required arguments kwargs = dict( zip_longest(args[::-1], list(defaults or ())[::-1], fillvalue=None)) kwargs.update(dataset.data) # Make sure the updated values are in the correct order new_default_values = [kwargs[arg] for arg in args] setattr(new_test, "func_defaults", tuple(new_default_values)) # Add the new test to the decorated TestCase setattr(cls, new_test_name, new_test) return cls
def __init__(self, func): self.func = func self.namespace = six.get_function_globals(func) if six.get_function_closure(func): self.namespace.update( dict( zip(func.__code__.co_freevars, (cell.cell_contents for cell in six.get_function_closure(func)))))
def to_graph(e, recursive=True, verbose=False, arg_values=None, arg_types=None, partial_types=None): """Compile a Python entity into equivalent TensorFlow code. Currently supported entities: * functions * classes Classes are handled by converting all their methods into a new class. Args: e: A Python entity. recursive: Whether to recusrively convert any functions that the decorator function may call. verbose: Whether to output the compiled code in the logs. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. partial_types: A set of types (e.g. classes) that will not be converted entirely. Calls to member functions for these types will be renamed independently. Returns: A function with a signature identical to `o`, but which when executed it creates TF a graph that has the same functionality as the original entity. """ conversion_map = conversion.ConversionMap( recursive=recursive, nocompile_decorators=(convert, graph_ready, convert_inline), partial_types=partial_types, api_module=tf_inspect.getmodule(to_graph)) _, name = conversion.entity_to_graph(e, conversion_map, arg_values, arg_types) module = gast.Module([]) for import_line in config.COMPILED_IMPORT_STATEMENTS: module.body.append(parser.parse_str(import_line)) for dep in conversion_map.dependency_cache.values(): module.body.append(dep) compiled_node, compiled_src = compiler.ast_to_object(module) # The compiled code should see everything the entry function saw. # TODO(mdan): This might not work well if the call tree spans modules? if tf_inspect.isfunction(e): compiled_node.__dict__.update(six.get_function_globals(e)) compiled_fn = getattr(compiled_node, name) if verbose: logging.info('Compiled output of %s:\n\n%s\n', e, compiled_src) return compiled_fn
def fix_js_args(func): '''Use this function when unsure whether func takes this and arguments as its last 2 args. It will append 2 args if it does not.''' fcode = six.get_function_code(func) fargs = fcode.co_varnames[fcode.co_argcount-2:fcode.co_argcount] if fargs==('this', 'arguments') or fargs==('arguments', 'var'): return func code = append_arguments(six.get_function_code(func), ('this','arguments')) return types.FunctionType(code, six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
def DataDrivenFixture(cls): """Generates new unittest test methods from methods defined in the decorated class""" if not issubclass(cls, TestCase): raise DataDrivenFixtureError test_case_attrs = dir(cls) for attr_name in test_case_attrs: if attr_name.startswith(DATA_DRIVEN_TEST_PREFIX) is False: # Not a data driven test, skip it continue original_test = getattr(cls, attr_name, None).__func__ test_data = getattr(original_test, DATA_DRIVEN_TEST_ATTR, None) if test_data is None: # no data was provided to the datasource decorator or this is not a # data driven test, skip it. continue for dataset in test_data: # Name the new test based on original and dataset names base_test_name = str(original_test.__name__)[int(len(DATA_DRIVEN_TEST_PREFIX)) :] new_test_name = "test_{0}_{1}".format(base_test_name, dataset.name) # Create a new test from the old test new_test = FunctionType( six.get_function_code(original_test), six.get_function_globals(original_test), name=new_test_name ) # Copy over any other attributes the original test had (mainly to # support test tag decorator) for attr in list(set(dir(original_test)) - set(dir(new_test))): setattr(new_test, attr, getattr(original_test, attr)) # Change the new test's default keyword values to the appropriate # new data as defined by the datasource decorator args, _, _, defaults = inspect.getargspec(original_test) # Self doesn't have a default, so we need to remove it args.remove("self") # Make sure we take into account required arguments kwargs = dict(zip_longest(args[::-1], list(defaults or ())[::-1], fillvalue=None)) kwargs.update(dataset.data) # Make sure the updated values are in the correct order new_default_values = [kwargs[arg] for arg in args] setattr(new_test, "func_defaults", tuple(new_default_values)) # Add the new test to the decorated TestCase setattr(cls, new_test_name, new_test) return cls
def execute(*args, **kwargs): ''' 修饰器代理函数 ''' start = time.time() ret = func(*args, **kwargs) logging.info("{module}.{func} execute sec:{total_sec}".format( date=datetime.datetime.now().strftime("%m-%d %H:%M"), module=six.get_function_globals(func).get('__name__', ''), func=func.__name__, total_sec=str(round(time.time() - start, 2)) )) return ret
def _environment(function, names_to_omit=()): """Yields the names and values visible from the function's scope.""" str_names_to_omit = set(map(str, names_to_omit)) for name, val in six.iteritems(six.get_function_globals(function)): if str(name) not in str_names_to_omit: yield name, val closure = six.get_function_closure(function) if closure is not None: freevars = six.get_function_code(function).co_freevars for name, cell in zip(freevars, closure): if str(name) not in str_names_to_omit: yield name, cell.cell_contents
def compile_func(func, variables, backend): codegen = FuncGenerator(func, variables=variables, backend=backend) src = codegen.generate() co = compile(src, '<string>', 'exec') locs = dict() globals = dict.copy(get_function_globals(func)) eval(co, globals, locs) ode = locs[func.__name__] del locs return ode, src
def test_getnamespace_hermetic(self): # Intentionally hiding the global function to make sure we don't overwrite # it in the global namespace. free_function = object() # pylint:disable=redefined-outer-name def test_fn(): return free_function ns = inspect_utils.getnamespace(test_fn) globs = six.get_function_globals(test_fn) self.assertTrue(ns['free_function'] is free_function) self.assertFalse(globs['free_function'] is free_function)
def decorator(caller, func=None): """ decorator(caller) converts a caller function into a decorator; decorator(caller, func) decorates a function using a caller. """ if func is not None: # returns a decorated function evaldict = six.get_function_globals(func).copy() evaldict['_call_'] = caller evaldict['_func_'] = func return FunctionMaker.create( func, "return _call_(_func_, %(shortsignature)s)", evaldict, undecorated=func, __wrapped__=func) else: # returns a decorator if inspect.isclass(caller): name = caller.__name__.lower() callerfunc = get_init(caller) doc = 'decorator(%s) converts functions/generators into ' \ 'factories of %s objects' % (caller.__name__, caller.__name__) fun = getfullargspec(callerfunc).args[1] # second arg elif inspect.isfunction(caller): name = '_lambda_' if caller.__name__ == '<lambda>' \ else caller.__name__ callerfunc = caller doc = caller.__doc__ fun = getfullargspec(callerfunc).args[0] # first arg else: # assume caller is an object with a __call__ method name = caller.__class__.__name__.lower() callerfunc = caller.__call__.im_func doc = caller.__call__.__doc__ fun = getfullargspec(callerfunc).args[1] # second arg evaldict = six.get_function_globals(callerfunc).copy() evaldict['_call_'] = caller evaldict['decorator'] = decorator return FunctionMaker.create( '%s(%s)' % (name, fun), 'return decorator(_call_, %s)' % fun, evaldict, undecorated=caller, __wrapped__=caller, doc=doc, module=caller.__module__)
def copy_func(f, name=None): """Create a copy of a function. Parameters ---------- f : function Function to copy. name : str, optional Name of new function. """ return types.FunctionType(six.get_function_code(f), six.get_function_globals(f), name or f.__name__, six.get_function_defaults(f), six.get_function_closure(f))
def execute(*args, **kwargs): ''' 修饰器代理函数 ''' try: return func(*args, **kwargs) except Exception: arg_str = ",".join([get_arg_str(arg) for arg in args]) kwargs_str = ",".join(["%s=%s" % (k, get_arg_str(v)) for k, v in kwargs.items()]) func_args_str = ",".join([item for item in [arg_str, kwargs_str] if item]) logging.error('{module}.{func}({func_args_str}): {tb}'.format( module=six.get_function_globals(func).get('__name__', ''), func=func.__name__, tb=traceback.format_exc(), func_args_str=func_args_str, ))
def __init__(self, func, fallback=None, autojit_kw=None): self.func = func # This covers a Python 2/3 change not covered by six try: self.func_name = func.__name__ except AttributeError: self.func_name = func.func_name module_name = inspect.getmoduleinfo( six.get_function_globals(func)['__file__']).name module_name = '.'.join(['trackpy', module_name]) self.module_name = module_name self.autojit_kw = autojit_kw if fallback is not None: self.ordinary = fallback else: self.ordinary = func
def function_to_graph(f, conversion_map, arg_values, arg_types, owner_type=None): """Specialization of `entity_to_graph` for callable functions.""" node, source = parser.parse_entity(f) node = node.body[0] namespace = six.get_function_globals(f) # This is needed for non-global functions. closure = six.get_function_closure(f) if closure: for e in closure: if callable(e.cell_contents): fn = e.cell_contents namespace[fn.__name__] = fn # Manually add the utils namespace which may be used from generated code. if 'py2tf_util' not in namespace: namespace['py2tf_utils'] = utils elif namespace['py2tf_utils'] != utils: raise ValueError( 'The module name py2tf_utils is reserved and may not be used.') namer = conversion_map.new_namer(namespace) ctx = context.EntityContext( namer=namer, source_code=source, source_file='<fragment>', namespace=namespace, arg_values=arg_values, arg_types=arg_types, recursive=conversion_map.recursive) node = node_to_graph(node, ctx, conversion_map.nocompile_decorators) # TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type) if not did_rename: new_name = f.__name__ if node.name != f.__name__: raise NotImplementedError('Strange corner case. Send us offending code!') node.name = new_name conversion_map.update_name_map(namer) return node, new_name
def getnamespace(f): """Returns the complete namespace of a function. Namespace is defined here as the mapping of all non-local variables to values. This includes the globals and the closure variables. Note that this captures the entire globals collection of the function, and may contain extra symbols that it does not actually use. Args: f: User defined function. Returns: A dict mapping symbol names to values. """ namespace = dict(six.get_function_globals(f)) closure = six.get_function_closure(f) freevars = six.get_function_code(f).co_freevars if freevars and closure: for name, cell in zip(freevars, closure): namespace[name] = cell.cell_contents return namespace
def save_function(pickler, obj): if not _locate_function(obj, pickler): log.info("F1: %s" % obj) globs = get_function_globals(obj) mod_name = obj.__module__ pickler.save_reduce(_create_function, (get_function_code(obj), {}, obj.__name__, get_function_defaults(obj), get_function_closure(obj), obj.__dict__, mod_name), obj=obj, func_globals=globs) log.info("# F1 %s" % obj) else: log.info("F2: %s" % obj) StockPickler.save_global(pickler, obj) log.info("# F2 %s" % obj) return
def test_get_function_globals(): def f(): pass assert six.get_function_globals(f) is globals()
def translate(env, func, *args, **kwargs): """ Given a shellcode environment, a function and its parameters, translate the function to a list of shellcode operations ready to be compiled or assembled using :meth:`~pwnypack.shellcode.base.BaseEnvironment.compile` or :meth:`~pwnypack.shellcode.base.BaseEnvironment.assemble`. Arguments: env(~pwnypack.shellcode.base.Base): An instance of a shellcode environment. func(callable): The function to translate to shellcode. args(...): The positional arguments for the function. kwargs(...): The keyword arguments for the function. Returns: list: The high-level shellcode operations. """ func_code = six.get_function_code(func) func_globals = dict(__builtins__) func_globals.update(six.get_function_globals(func)) ops = bc.disassemble(func_code.co_code) program = [] f_args = inspect.getcallargs(func, *args, **kwargs) variables = dict( (func_code.co_varnames.index(arg_name), arg_value) for arg_name, arg_value in six.iteritems(f_args) ) stack = [] for op in ops: if op.name == 'LOAD_CONST': stack.append(func_code.co_consts[op.arg]) elif op.name == 'LOAD_GLOBAL': global_name = func_code.co_names[op.arg] stack.append(getattr(env, global_name, func_globals.get(global_name))) elif op.name == 'LOAD_FAST': var_name = func_code.co_varnames[op.arg] stack.append(getattr(env, var_name, variables.get(op.arg))) elif op.name == 'BUILD_LIST': items = stack[-op.arg:] del stack[-op.arg:] stack.append(items) elif op.name == 'LOAD_ATTR': obj = stack.pop() stack.append(getattr(obj, func_code.co_names[op.arg])) elif op.name == 'CALL_FUNCTION': nargs = op.arg & 0xff nkwargs = op.arg >> 8 if nkwargs: f_kwargs = dict(zip(stack[-nkwargs * 2::2], stack[-nkwargs * 2 + 1::2])) del stack[-nkwargs * 2:] else: f_kwargs = {} if nargs: f_args = stack[-nargs:] del stack[-nargs:] else: f_args = [] f = stack.pop() if isinstance(f, Fragment): stack.append(f(env, *f_args, **f_kwargs)) else: stack.append(f(*f_args, **f_kwargs)) elif op.name == 'STORE_FAST': value = stack.pop() var_name = func_code.co_varnames[op.arg] var = getattr(env, var_name, variables.get(op.arg, None)) if isinstance(var, Register): program.append(LoadRegister(var, value)) else: variables[op.arg] = value elif op.name == 'POP_TOP': value = stack.pop() if isinstance(value, SyscallInvoke): program.append(value) elif isinstance(value, list): program.extend(value) else: raise ValueError('No idea how to compile %s' % (value,)) elif op.name == 'RETURN_VALUE': stack.pop() elif op.name == 'DUP_TOP': value = stack[-1] if isinstance(value, SyscallInvoke): stack.insert(-1, env.SYSCALL_RET_REG) else: stack.append(value) elif op.name == 'BINARY_SUBSCR': index = stack.pop() value = stack.pop() stack.append(value[index]) elif op.name == 'STORE_SUBSCR': index = stack.pop() value = stack.pop() new_value = stack.pop() var = value[index] if isinstance(var, Register): program.append(LoadRegister(var, new_value)) else: value[index] = new_value elif op.name == 'INPLACE_ADD': value = stack.pop() reg = stack.pop() if not isinstance(reg, Register): raise TypeError('In-place addition is only supported on registers') program.extend(env.reg_add(reg, value)) stack.append(reg) elif op.name == 'INPLACE_SUBTRACT': value = stack.pop() reg = stack.pop() if not isinstance(reg, Register): raise TypeError('In-place subtraction is only supported on registers') program.extend(env.reg_sub(reg, value)) stack.append(reg) else: raise RuntimeError('Unsupported opcode: %s' % op.name) return program
print(pos_to_inst[i]) print(pos_to_inst[i - 2]) print(list(map(chr, old_bytecode))[i - 4:i + 8]) print(bytelist[i - 4:i + 8]) break raise RuntimeError( 'Your python version made changes to the bytecode') check(six.get_function_code(check)) if __name__ == '__main__': x = 'Wrong' dick = 3000 def func(a): print(x, y, z, a) print(dick) d = (x, ) for e in (e for e in x): print(e) return x, y, z func2 = types.FunctionType( append_arguments(six.get_function_code(func), ('x', 'y', 'z')), six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func)) args = (2, 2, 3, 4), 3, 4 assert func2(1, *args) == args
def get_function_module(func): return get_function_globals(func).get('__name__')
from __future__ import absolute_import, print_function import inspect import logging import traceback import types from tornado.httpserver import HTTPRequest from six import iteritems, get_method_self, get_function_closure, get_function_code, get_function_globals function_module = lambda func: get_function_globals(func).get('__name__') function_closure_dict = lambda func: dict(zip(get_function_code(func).co_freevars, (c.cell_contents for c in get_function_closure(func)))) class TornadoContextInspector(object): """ Tool for inspect and found HTTRequest from callback functions or stack frames. This is useful for finding and logging the HTTPRequest object when an exception occur in async callback. This class also can be used to generate traceback-like string for calls withing `tornado.gen` framework Finding the actual request object in async callbacks is a pain, and this is can be done only by inspecting function closures, objects that owning methods, and ... The usage of `TornadoContextInspector` is simple >>> import sys >>> inspector = TornadoContextInspector() >>> inspector.inspect_frame(sys.exc_info()[2].tb_frame) >>> print(inspector.found_req) # may be None HTTPRequest(protocol='http', ...) >>> print(''.join(inspector.format_async_frames())) File "file.py", line 32, in func_1
def _getparams_rtype(cls, function): """Get function params from input function and rtype. :return: OrderedDict, rtype, vargs and kwargs. :rtype: tuple """ try: args, vargs, kwargs, default = getargspec(function) except TypeError: args, vargs, kwargs, default = (), (), (), () indexlen = len(args) - (0 if default is None else len(default)) params = OrderedDict() for index, arg in enumerate(args): pkwargs = { 'name': arg, 'mandatory': True } # param kwargs if index >= indexlen: # has default value value = default[index - indexlen] pkwargs['default'] = value pkwargs['ref'] = None if value is None else data2schema(value) pkwargs['mandatory'] = False params[arg] = pkwargs rtype = None # parse docstring if function.__doc__ is not None and not isbuiltin(function): scope = get_function_globals(function) for match in cls._REC.findall(function.__doc__): if rtype is None: rrtype = match[4].strip() or None if rrtype: rtypes = rrtype.split(',') schemas = [] for rtype_ in rtypes: rtype_ = rtype_.strip() islist = False try: lkrtype = lookup(rtype_, scope=scope) except ImportError: islist = True try: if rtype_[-1] == 's': lkrtype = lookup( rtype_[:-1], scope=scope ) elif rtype_.startswith('list of '): lkrtype = lookup( rtype_[8:], scope=scope ) else: raise except ImportError: msg = 'rtype "{0}" ({1}) from {2} not found.' raise ImportError( msg.format(rtype_, rrtype, function) ) try: schemacls = datatype2schemacls(lkrtype) except TypeError: schemacls = ParamTypeSchema(type=lkrtype) rschema = schemacls() if islist: rschema = ArraySchema(itemtype=rschema) schemas.append(rschema) if len(rtypes) > 1: rtype = OneOfSchema(schemas=schemas, nullable=True) else: rtype = schemas[0] continue pname = (match[1] or match[2]).strip() if pname and pname in params: ptype = (match[0] or match[3]).strip() ptypes = ptype.split(',') schemas = [] for ptype in ptypes: ptype = ptype.strip() islist = False try: lkptype = lookup(ptype, scope=scope) except ImportError: islist = True try: if ptype[-1] == 's': lkptype = lookup(ptype[:-1], scope=scope) elif ptype.startswith('list of '): lkptype = lookup(ptype[8:], scope=scope) else: raise except ImportError: msg = 'Error on ptype "{0}" ({1}) from {2} not found.' raise ImportError( msg.format(pname, ptype, function) ) try: schemacls = datatype2schemacls(lkptype) except TypeError: schemacls = ParamTypeSchema(type=lkptype) pschema = schemacls() if islist: pschema = ArraySchema(itemtype=pschema) schemas.append(pschema) if len(ptypes) > 1: pschema = OneOfSchema(schemas=schemas, nullable=True) else: pschema = schemas[0] params[pname]['ref'] = pschema return params, rtype, vargs, kwargs
return [chr(op), chr(oparg & 255), chr((oparg >> 8) & 255)] elif oparg <= 4294967296: return [chr(opcode.EXTENDED_ARG), chr((oparg >> 16) & 255), chr((oparg >> 24) & 255), chr(op), chr(oparg & 255), chr((oparg >> 8) & 255)] else: raise ValueError("Invalid oparg: {0} is too large".format(oparg)) if __name__=='__main__': x = 'Wrong' dick = 3000 def func(a): print(x,y,z, a) print(dick) d = (x,) for e in (e for e in x): print(e) return x, y, z func2 =types.FunctionType(append_arguments(six.get_function_code(func), ('x', 'y', 'z')), six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func)) args = (2,2,3,4),3,4 assert func2(1, *args) == args
def pytest_runtest_setup(self, item): # Check to see if we need to benchmark any invocations. bench = item.keywords.get('bench') if bench is None: # Nope; nothing to see here. # Check to see if we can skip this test (requested to /only/ run # benchmarks). if self.config.option.bench_only: raise pytest.skip('no associated benchmark') # Just continue to the test. return # Get the first argument to indicate what method to benchmark. expression = bench.args[0] iterations = bench.kwargs.get('iterations', 100) # Create a wrapper for the test case that applies the benchmark. item_function = self._item_function = item.function item_function_globals = six.get_function_globals(item_function) item_function_argspec = inspect.getargspec(item.function) @wraps(item.function) def item_function_wrapper(*args, **kwargs): # Extract the function from the expression. locals_, globals_ = locals(), item_function_globals locals_.update(dict(zip(item_function_argspec.args, args))) locals_.update(kwargs) six.exec_('_function = %s' % expression, globals_, locals_) _function = locals_['_function'] # Initialize benchmark process. props = {'times': list()} # Create a wrapper for the method to benchmark. @wraps(_function) def benchmark(*args, **kwargs): # nonlocal elapsed, real_iterations gc.collect() gc.disable() start = timer() result = _function(*args, **kwargs) finish = timer() gc.enable() props['times'].append(finish - start) return result # Replace the function with the wrapped function. locals_['benchmark'] = benchmark six.exec_('%s = benchmark' % expression, globals_, locals_) # Attempt to replace it in global scope as well. globals_.update(locals_) # Get the (unbound) function. try: locals_['function'] = six.get_method_function(item_function) except AttributeError: locals_['function'] = item_function # Iterate the set number of iterations. item.teardown() for _ in range(iterations): item.setup() locals_['args'] = args locals_['kwargs'] = kwargs six.exec_('function(*args, **kwargs)', globals_, locals_) item.teardown() # Restore the benchmarked function. six.exec_('%s = _function' % expression, globals_, locals_) # Construct a Benchmark instance to store the result. self._benchmarks.append(Benchmark(item, **props)) if item.cls is not None: setattr(item.cls, item.function.__name__, item_function_wrapper) else: item.obj = item_function_wrapper
def _compilecode(function, name, impl, args, varargs, kwargs): """Get generated code. :return: function proxy generated code. :rtype: str """ newcodestr, generatedname, impl_name = _generatecode( function=function, name=name, impl=impl, args=args, varargs=varargs, kwargs=kwargs ) try: __file__ = getfile(function) except TypeError: __file__ = '<string>' # compile newcodestr code = compile(newcodestr, __file__, 'single') # define the code with the new function _globals = {} exec_(code, _globals) # get new code _var = _globals[generatedname] newco = get_function_code(_var) # get new consts list newconsts = list(newco.co_consts) if PY3: newcode = list(newco.co_code) else: newcode = [ord(co) for co in newco.co_code] consts_values = {impl_name: impl} # change LOAD_GLOBAL to LOAD_CONST index = 0 newcodelen = len(newcode) while index < newcodelen: if newcode[index] == LOAD_GLOBAL: oparg = newcode[index + 1] + (newcode[index + 2] << 8) name = newco.co_names[oparg] if name in consts_values: const_value = consts_values[name] if const_value in newconsts: pos = newconsts.index(const_value) else: pos = len(newconsts) newconsts.append(consts_values[name]) newcode[index] = LOAD_CONST newcode[index + 1] = pos & 0xFF newcode[index + 2] = pos >> 8 index += 1 codeobj = getcodeobj(newconsts, newcode, newco, get_function_code(function)) # instanciate a new function if function is None or isbuiltin(function): result = FunctionType(codeobj, {}) else: result = type(function)( codeobj, get_function_globals(function), function.__name__, get_function_defaults(function), get_function_closure(function) ) return result