def test_lambdify(): try: from sympy import symbols, lambdify except ImportError: return settings['recurse'] = True x = symbols("x") y = x**2 f = lambdify([x], y) z = min d = globals() globalvars(f, recurse=True, builtin=True) assert z is min assert d is globals()
def importable(obj, alias='', source=True, builtin=True): """get an importable string (i.e. source code or the import string) for the given object, including any required objects from the enclosing and global scope This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, otherwise an import will be sought. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If builtin=True, then force an import for builtins where possible. """ #NOTE: we always 'force', and 'lstrip' as necessary #NOTE: for 'enclosing', use importable(outermost(obj)) if builtin and isbuiltin(obj): source = False tried_source = tried_import = False while True: if not source: # we want an import try: if _isinstance(obj): # for instances, punt to _importable return _importable(obj, alias, source=False, builtin=builtin) src = _closuredimport(obj, alias=alias, builtin=builtin) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] except: if tried_source: raise tried_import = True # we want the source try: src = _closuredsource(obj, alias=alias) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') src = list(src.values())[0] if src[0] and src[-1]: src = '\n'.join(src) elif src[0]: src = src[0] elif src[-1]: src = src[-1] else: src = '' # get source code of objects referred to by obj in global scope from dill.detect import globalvars obj = globalvars(obj) #XXX: don't worry about alias? obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items()) obj = '\n'.join(obj) if obj else '' # combine all referred-to source (global then enclosing) if not obj: return src if not src: return obj return obj + src except: if tried_import: raise tried_source = True source = not source # should never get here return
def _matchlambda(func, line): """check if lambda object 'func' matches raw line of code 'line'""" from dill.detect import code as getcode from dill.detect import freevars, globalvars, varnames dummy = lambda: '__this_is_a_big_dummy_function__' # process the line (removing leading whitespace, etc) lhs, rhs = line.split('lambda ', 1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs, rhs), globals(), locals()) except: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure _f = [line.count(i) for i in freevars(func).keys()] if not _f: # not in closure # check if code matches if _ == code: return True return False # weak check on freevars if not all(_f): return False #XXX: VERY WEAK # weak check on varnames and globalvars _f = varnames(func) _f = [line.count(i) for i in _f[0] + _f[1]] if _f and not all(_f): return False #XXX: VERY WEAK _f = [line.count(i) for i in globalvars(func).keys()] if _f and not all(_f): return False #XXX: VERY WEAK # check if func is a double lambda if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()): _lhs, _rhs = rhs.split('lambda ', 1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs, _rhs), globals(), locals()) except: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False #NOTE: should be same code same order, but except for 't' and '\x88' _ = set((i, j) for (i, j) in zip(_, code) if i != j) if len(_) != 1: return False #('t','\x88') return True # check indentsize if not indentsize(line): return False #FIXME: is this a good check??? # check if code 'pattern' matches #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?) _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same? _f = code.split(code[0]) # '\x88' #NOTE: should be same code different order, with different first element _ = dict( re.match('([\W\D\S])(.*)', _[i]).groups() for i in range(1, len(_))) _f = dict( re.match('([\W\D\S])(.*)', _f[i]).groups() for i in range(1, len(_f))) if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())): return True return False
def _matchlambda(func, line): """check if lambda object 'func' matches raw line of code 'line'""" from dill.detect import code as getcode from dill.detect import freevars, globalvars, varnames dummy = lambda : '__this_is_a_big_dummy_function__' # process the line (removing leading whitespace, etc) lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) except: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure _f = [line.count(i) for i in freevars(func).keys()] if not _f: # not in closure # check if code matches if _ == code: return True return False # weak check on freevars if not all(_f): return False #XXX: VERY WEAK # weak check on varnames and globalvars _f = varnames(func) _f = [line.count(i) for i in _f[0]+_f[1]] if _f and not all(_f): return False #XXX: VERY WEAK _f = [line.count(i) for i in globalvars(func).keys()] if _f and not all(_f): return False #XXX: VERY WEAK # check if func is a double lambda if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) except: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False #NOTE: should be same code same order, but except for 't' and '\x88' _ = set((i,j) for (i,j) in zip(_,code) if i != j) if len(_) != 1: return False #('t','\x88') return True # check indentsize if not indentsize(line): return False #FIXME: is this a good check??? # check if code 'pattern' matches #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?) _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same? _f = code.split(code[0]) # '\x88' #NOTE: should be same code different order, with different first element _ = dict(re.match('([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_))) _f = dict(re.match('([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f))) if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())): return True return False
def _write_globals(func, path: Path) -> Path: """Fetches, serialized and writes current globals required for `func` to `path`. :param func: A callable that may or may not use global variables :param path: Where to write the serialized globals :return Path object to the serialized globals file """ globs = globalvars( func) # Globals that are relevant for the function to run globs.update({'__name__': '__main__'}) # Include the entry point for the script globs_serialized = Serializer.serialize(globs) globs_file = path.joinpath("globs.msk") with globs_file.open('w') as globs_fd: globs_fd.write(globs_serialized) globs_fd.flush() return globs_file
def test_globals(): def f(): a def g(): b def h(): c assert globalvars(f) == dict(a=1, b=2, c=3) res = globalvars(foo, recurse=True) assert set(res) == set(['squared', 'a']) res = globalvars(foo, recurse=False) assert res == {} zap = foo(2) res = globalvars(zap, recurse=True) assert set(res) == set(['squared', 'a']) res = globalvars(zap, recurse=False) assert set(res) == set(['squared']) del zap res = globalvars(squared) assert set(res) == set(['a'])
else: assert parent(obj, int) is x[-1] # python oddly? finds last int assert at(id(at)) is at def f(): a def g(): b def h(): c a, b, c = 1, 2, 3 assert globalvars(f) == dict(a=1, b=2, c=3) def squared(x): return a + x**2 def foo(x): def bar(y): return squared(x) + y return bar class _class: def _method(self):
x = [4,5,6,7] listiter = iter(x) obj = parent(listiter, list) assert obj is x assert parent(obj, int) is x[-1] assert at(id(at)) is at def f(): a def g(): b def h(): c a, b, c = 1, 2, 3 assert globalvars(f) == dict(a=1, b=2, c=3) def squared(x): return a+x**2 def foo(x): def bar(y): return squared(x)+y return bar class _class: def _method(self): pass def ok(self): return True
def importable(obj, alias='', source=None, builtin=True): """get an importable string (i.e. source code or the import string) for the given object, including any required objects from the enclosing and global scope This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, to attempt to force the use of an import, use source=False; otherwise an import will be sought for objects not defined in __main__. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If builtin=True, then force an import for builtins where possible. """ #NOTE: we always 'force', and 'lstrip' as necessary #NOTE: for 'enclosing', use importable(outermost(obj)) if source is None: source = True if isfrommain(obj) else False elif builtin and isbuiltin(obj): source = False tried_source = tried_import = False while True: if not source: # we want an import try: if _isinstance(obj): # for instances, punt to _importable return _importable(obj, alias, source=False, builtin=builtin) src = _closuredimport(obj, alias=alias, builtin=builtin) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] except: if tried_source: raise tried_import = True # we want the source try: src = _closuredsource(obj, alias=alias) if len(src) == 0: raise NotImplementedError('not implemented') # groan... an inline code stitcher def _code_stitcher(block): "stitch together the strings in tuple 'block'" if block[0] and block[-1]: block = '\n'.join(block) elif block[0]: block = block[0] elif block[-1]: block = block[-1] else: block = '' return block # get free_vars first _src = _code_stitcher(src.pop(None)) _src = [_src] if _src else [] # get func_vars for xxx in src.values(): xxx = _code_stitcher(xxx) if xxx: _src.append(xxx) # make a single source string if not len(_src): src = '' elif len(_src) == 1: src = _src[0] else: src = '\n'.join(_src) # get source code of objects referred to by obj in global scope from dill.detect import globalvars obj = globalvars(obj) #XXX: don't worry about alias? obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items()) obj = '\n'.join(obj) if obj else '' # combine all referred-to source (global then enclosing) if not obj: return src if not src: return obj return obj + src except: if tried_import: raise tried_source = True source = not source # should never get here return
def importable(obj, alias='', source=None, builtin=True): """get an importable string (i.e. source code or the import string) for the given object, including any required objects from the enclosing and global scope This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, to attempt to force the use of an import, use source=False; otherwise an import will be sought for objects not defined in __main__. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If builtin=True, then force an import for builtins where possible. """ #NOTE: we always 'force', and 'lstrip' as necessary #NOTE: for 'enclosing', use importable(outermost(obj)) if source is None: source = True if isfrommain(obj) else False elif builtin and isbuiltin(obj): source = False tried_source = tried_import = False while True: if not source: # we want an import try: if _isinstance(obj): # for instances, punt to _importable return _importable(obj, alias, source=False, builtin=builtin) src = _closuredimport(obj, alias=alias, builtin=builtin) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] except: if tried_source: raise tried_import = True # we want the source try: src = _closuredsource(obj, alias=alias) if len(src) == 0: raise NotImplementedError('not implemented') # groan... an inline code stitcher def _code_stitcher(block): "stitch together the strings in tuple 'block'" if block[0] and block[-1]: block = '\n'.join(block) elif block[0]: block = block[0] elif block[-1]: block = block[-1] else: block = '' return block # get free_vars first _src = _code_stitcher(src.pop(None)) _src = [_src] if _src else [] # get func_vars for xxx in src.values(): xxx = _code_stitcher(xxx) if xxx: _src.append(xxx) # make a single source string if not len(_src): src = '' elif len(_src) == 1: src = _src[0] else: src = '\n'.join(_src) # get source code of objects referred to by obj in global scope from dill.detect import globalvars obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc? obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj)) obj = '\n'.join(obj) if obj else '' # combine all referred-to source (global then enclosing) if not obj: return src if not src: return obj return obj + src except: if tried_import: raise tried_source = True source = not source # should never get here return
def importable(obj, alias='', source=True, builtin=True): """get an importable string (i.e. source code or the import string) for the given object, including any required objects from the enclosing and global scope This function will attempt to discover the name of the object, or the repr of the object, or the source code for the object. To attempt to force discovery of the source code, use source=True, otherwise an import will be sought. The intent is to build a string that can be imported from a python file. obj is the object to inspect. If alias is provided, then rename the object with the given alias. If builtin=True, then force an import for builtins where possible. """ #NOTE: we always 'force', and 'lstrip' as necessary #NOTE: for 'enclosing', use importable(outermost(obj)) if builtin and isbuiltin(obj): source = False tried_source = tried_import = False while True: if not source: # we want an import try: if _isinstance(obj): # for instances, punt to _importable return _importable(obj, alias, source=False, builtin=builtin) src = _closuredimport(obj, alias=alias, builtin=builtin) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] except: if tried_source: raise tried_import = True # we want the source try: src = _closuredsource(obj, alias=alias) if len(src) == 0: raise NotImplementedError('not implemented') if len(src) > 1: raise NotImplementedError('not implemented') src = list(src.values())[0] if src[0] and src[-1]: src = '\n'.join(src) elif src[0]: src = src[0] elif src[-1]: src = src[-1] else: src = '' # get source code of objects referred to by obj in global scope from dill.detect import globalvars obj = globalvars(obj) #XXX: don't worry about alias? obj = list( getsource(_obj, name, force=True) for (name, _obj) in obj.items()) obj = '\n'.join(obj) if obj else '' # combine all referred-to source (global then enclosing) if not obj: return src if not src: return obj return obj + src except: if tried_import: raise tried_source = True source = not source # should never get here return
def save_function(pickler, obj): if not dill._dill._locate_function(obj, pickler): dill._dill.log.info("F1: %s" % obj) _recurse = getattr(pickler, "_recurse", None) # _byref = getattr(pickler, "_byref", None) # TODO: not used _postproc = getattr(pickler, "_postproc", None) _main_modified = getattr(pickler, "_main_modified", None) _original_main = getattr(pickler, "_original_main", dill._dill.__builtin__) # 'None' postproc_list = [] if _recurse: # recurse to get all globals referred to by obj from dill.detect import globalvars globs_copy = globalvars(obj, recurse=True, builtin=True) # Add the name of the module to the globs dictionary to prevent # the duplication of the dictionary. Pickle the unpopulated # globals dictionary and set the remaining items after the function # is created to correctly handle recursion. globs = {"__name__": obj.__module__} else: globs_copy = obj.__globals__ if dill._dill.PY3 else obj.func_globals # If the globals is the __dict__ from the module being saved as a # session, substitute it by the dictionary being actually saved. if _main_modified and globs_copy is _original_main.__dict__: globs_copy = getattr(pickler, "_main", _original_main).__dict__ globs = globs_copy # If the globals is a module __dict__, do not save it in the pickle. elif (globs_copy is not None and obj.__module__ is not None and getattr(dill._dill._import_module(obj.__module__, True), "__dict__", None) is globs_copy): globs = globs_copy else: globs = {"__name__": obj.__module__} if globs_copy is not None and globs is not globs_copy: # In the case that the globals are copied, we need to ensure that # the globals dictionary is updated when all objects in the # dictionary are already created. if dill._dill.PY3: glob_ids = {id(g) for g in globs_copy.values()} else: glob_ids = {id(g) for g in globs_copy.itervalues()} for stack_element in _postproc: if stack_element in glob_ids: _postproc[stack_element].append( (dill._dill._setitems, (globs, globs_copy))) break else: postproc_list.append( (dill._dill._setitems, (globs, globs_copy))) # DONE: globs is a dictionary with keys = var names (str) and values = python objects # however the dictionary is not always loaded in the same order # therefore we have to sort the keys to make deterministic. # This is important to make `dump` deterministic. # Only this line is different from the original implementation: globs = {k: globs[k] for k in sorted(globs.keys())} if dill._dill.PY3: closure = obj.__closure__ state_dict = {} for fattrname in ("__doc__", "__kwdefaults__", "__annotations__"): fattr = getattr(obj, fattrname, None) if fattr is not None: state_dict[fattrname] = fattr if obj.__qualname__ != obj.__name__: state_dict["__qualname__"] = obj.__qualname__ if "__name__" not in globs or obj.__module__ != globs[ "__name__"]: state_dict["__module__"] = obj.__module__ state = obj.__dict__ if type(state) is not dict: state_dict["__dict__"] = state state = None if state_dict: state = state, state_dict dill._dill._save_with_postproc( pickler, ( dill._dill._create_function, (obj.__code__, globs, obj.__name__, obj.__defaults__, closure), state, ), obj=obj, postproc_list=postproc_list, ) else: closure = obj.func_closure if obj.__doc__ is not None: postproc_list.append( (setattr, (obj, "__doc__", obj.__doc__))) if "__name__" not in globs or obj.__module__ != globs[ "__name__"]: postproc_list.append( (setattr, (obj, "__module__", obj.__module__))) if obj.__dict__: postproc_list.append( (setattr, (obj, "__dict__", obj.__dict__))) dill._dill._save_with_postproc( pickler, (dill._dill._create_function, (obj.func_code, globs, obj.func_name, obj.func_defaults, closure)), obj=obj, postproc_list=postproc_list, ) # Lift closure cell update to earliest function (#458) if _postproc: topmost_postproc = next(iter(_postproc.values()), None) if closure and topmost_postproc: for cell in closure: possible_postproc = (setattr, (cell, "cell_contents", obj)) try: topmost_postproc.remove(possible_postproc) except ValueError: continue # Change the value of the cell pickler.save_reduce(*possible_postproc) # pop None created by calling preprocessing step off stack if dill._dill.PY3: pickler.write(bytes("0", "UTF-8")) else: pickler.write("0") dill._dill.log.info("# F1") else: dill._dill.log.info("F2: %s" % obj) name = getattr(obj, "__qualname__", getattr(obj, "__name__", None)) dill._dill.StockPickler.save_global(pickler, obj, name=name) dill._dill.log.info("# F2") return
def save_function(pickler, obj): """ From dill._dill.save_function This is a modified version that make globs deterministic since the order of the keys in the output dictionary of globalvars can change. """ if not dill._dill._locate_function(obj): dill._dill.log.info(f"F1: {obj}") if getattr(pickler, "_recurse", False): # recurse to get all globals referred to by obj globalvars = dill.detect.globalvars globs = globalvars(obj, recurse=True, builtin=True) if id(obj) in dill._dill.stack: globs = obj.__globals__ if dill._dill.PY3 else obj.func_globals else: globs = obj.__globals__ if dill._dill.PY3 else obj.func_globals # globs is a dictionary with keys = var names (str) and values = python objects # however the dictionary is not always loaded in the same order # therefore we have to sort the keys to make deterministic. # This is important to make `dump` deterministic. # Only this line is different from the original implementation: globs = {k: globs[k] for k in sorted(globs.keys())} # The rest is the same as in the original dill implementation _byref = getattr(pickler, "_byref", None) _recurse = getattr(pickler, "_recurse", None) _memo = (id(obj) in dill._dill.stack) and (_recurse is not None) dill._dill.stack[id(obj)] = len(dill._dill.stack), obj if dill._dill.PY3: _super = ("super" in getattr(obj.__code__, "co_names", ())) and (_byref is not None) if _super: pickler._byref = True if _memo: pickler._recurse = False fkwdefaults = getattr(obj, "__kwdefaults__", None) pickler.save_reduce( dill._dill._create_function, (obj.__code__, globs, obj.__name__, obj.__defaults__, obj.__closure__, obj.__dict__, fkwdefaults), obj=obj, ) else: _super = (("super" in getattr(obj.func_code, "co_names", ())) and (_byref is not None) and getattr(pickler, "_recurse", False)) if _super: pickler._byref = True if _memo: pickler._recurse = False pickler.save_reduce( dill._dill._create_function, (obj.func_code, globs, obj.func_name, obj.func_defaults, obj.func_closure, obj.__dict__), obj=obj, ) if _super: pickler._byref = _byref if _memo: pickler._recurse = _recurse if (dill._dill.OLDER and not _byref and (_super or (not _super and _memo) or (not _super and not _memo and _recurse))): pickler.clear_memo() dill._dill.log.info("# F1") else: dill._dill.log.info(f"F2: {obj}") name = getattr(obj, "__qualname__", getattr(obj, "__name__", None)) dill._dill.StockPickler.save_global(pickler, obj, name=name) dill._dill.log.info("# F2") return