def test_lambdify():
try:
from sympy import symbols, lambdify
except ImportError:
return
settings['recurse'] = True
x = symbols("x")
y = x**2
f = lambdify([x], y)
z = min
d = globals()
globalvars(f, recurse=True, builtin=True)
assert z is min
assert d is globals()
def test_lambdify():
try:
from sympy import symbols, lambdify
except ImportError:
return
settings['recurse'] = True
x = symbols("x")
y = x**2
f = lambdify([x], y)
z = min
d = globals()
globalvars(f, recurse=True, builtin=True)
assert z is min
assert d is globals()
def importable(obj, alias='', source=True, builtin=True):
"""get an importable string (i.e. source code or the import string)
for the given object, including any required objects from the enclosing
and global scope
This function will attempt to discover the name of the object, or the repr
of the object, or the source code for the object. To attempt to force
discovery of the source code, use source=True, otherwise an import will be
sought. The intent is to build a string that can be imported from a python
file. obj is the object to inspect. If alias is provided, then rename the
object with the given alias. If builtin=True, then force an import for
builtins where possible.
"""
#NOTE: we always 'force', and 'lstrip' as necessary
#NOTE: for 'enclosing', use importable(outermost(obj))
if builtin and isbuiltin(obj): source = False
tried_source = tried_import = False
while True:
if not source: # we want an import
try:
if _isinstance(obj): # for instances, punt to _importable
return _importable(obj, alias, source=False, builtin=builtin)
src = _closuredimport(obj, alias=alias, builtin=builtin)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
return list(src.values())[0]
except:
if tried_source: raise
tried_import = True
# we want the source
try:
src = _closuredsource(obj, alias=alias)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
src = list(src.values())[0]
if src[0] and src[-1]: src = '\n'.join(src)
elif src[0]: src = src[0]
elif src[-1]: src = src[-1]
else: src = ''
# get source code of objects referred to by obj in global scope
from dill.detect import globalvars
obj = globalvars(obj) #XXX: don't worry about alias?
obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items())
obj = '\n'.join(obj) if obj else ''
# combine all referred-to source (global then enclosing)
if not obj: return src
if not src: return obj
return obj + src
except:
if tried_import: raise
tried_source = True
source = not source
# should never get here
return
def _matchlambda(func, line):
"""check if lambda object 'func' matches raw line of code 'line'"""
from dill.detect import code as getcode
from dill.detect import freevars, globalvars, varnames
dummy = lambda: '__this_is_a_big_dummy_function__'
# process the line (removing leading whitespace, etc)
lhs, rhs = line.split('lambda ', 1)[-1].split(":", 1) #FIXME: if !1 inputs
try: #FIXME: unsafe
_ = eval("lambda %s : %s" % (lhs, rhs), globals(), locals())
except:
_ = dummy
# get code objects, for comparison
_, code = getcode(_).co_code, getcode(func).co_code
# check if func is in closure
_f = [line.count(i) for i in freevars(func).keys()]
if not _f: # not in closure
# check if code matches
if _ == code: return True
return False
# weak check on freevars
if not all(_f): return False #XXX: VERY WEAK
# weak check on varnames and globalvars
_f = varnames(func)
_f = [line.count(i) for i in _f[0] + _f[1]]
if _f and not all(_f): return False #XXX: VERY WEAK
_f = [line.count(i) for i in globalvars(func).keys()]
if _f and not all(_f): return False #XXX: VERY WEAK
# check if func is a double lambda
if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()):
_lhs, _rhs = rhs.split('lambda ', 1)[-1].split(":",
1) #FIXME: if !1 inputs
try: #FIXME: unsafe
_f = eval("lambda %s : %s" % (_lhs, _rhs), globals(), locals())
except:
_f = dummy
# get code objects, for comparison
_, code = getcode(_f).co_code, getcode(func).co_code
if len(_) != len(code): return False
#NOTE: should be same code same order, but except for 't' and '\x88'
_ = set((i, j) for (i, j) in zip(_, code) if i != j)
if len(_) != 1: return False #('t','\x88')
return True
# check indentsize
if not indentsize(line): return False #FIXME: is this a good check???
# check if code 'pattern' matches
#XXX: or pattern match against dis.dis(code)? (or use uncompyle2?)
_ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same?
_f = code.split(code[0]) # '\x88'
#NOTE: should be same code different order, with different first element
_ = dict(
re.match('([\W\D\S])(.*)', _[i]).groups() for i in range(1, len(_)))
_f = dict(
re.match('([\W\D\S])(.*)', _f[i]).groups() for i in range(1, len(_f)))
if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())):
return True
return False
def _matchlambda(func, line):
"""check if lambda object 'func' matches raw line of code 'line'"""
from dill.detect import code as getcode
from dill.detect import freevars, globalvars, varnames
dummy = lambda : '__this_is_a_big_dummy_function__'
# process the line (removing leading whitespace, etc)
lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs
try: #FIXME: unsafe
_ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals())
except: _ = dummy
# get code objects, for comparison
_, code = getcode(_).co_code, getcode(func).co_code
# check if func is in closure
_f = [line.count(i) for i in freevars(func).keys()]
if not _f: # not in closure
# check if code matches
if _ == code: return True
return False
# weak check on freevars
if not all(_f): return False #XXX: VERY WEAK
# weak check on varnames and globalvars
_f = varnames(func)
_f = [line.count(i) for i in _f[0]+_f[1]]
if _f and not all(_f): return False #XXX: VERY WEAK
_f = [line.count(i) for i in globalvars(func).keys()]
if _f and not all(_f): return False #XXX: VERY WEAK
# check if func is a double lambda
if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()):
_lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs
try: #FIXME: unsafe
_f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals())
except: _f = dummy
# get code objects, for comparison
_, code = getcode(_f).co_code, getcode(func).co_code
if len(_) != len(code): return False
#NOTE: should be same code same order, but except for 't' and '\x88'
_ = set((i,j) for (i,j) in zip(_,code) if i != j)
if len(_) != 1: return False #('t','\x88')
return True
# check indentsize
if not indentsize(line): return False #FIXME: is this a good check???
# check if code 'pattern' matches
#XXX: or pattern match against dis.dis(code)? (or use uncompyle2?)
_ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same?
_f = code.split(code[0]) # '\x88'
#NOTE: should be same code different order, with different first element
_ = dict(re.match('([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_)))
_f = dict(re.match('([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f)))
if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())):
return True
return False
def _write_globals(func, path: Path) -> Path:
"""Fetches, serialized and writes current globals required for `func` to `path`.
:param func: A callable that may or may not use global variables
:param path: Where to write the serialized globals
:return Path object to the serialized globals file
"""
globs = globalvars(
func) # Globals that are relevant for the function to run
globs.update({'__name__':
'__main__'}) # Include the entry point for the script
globs_serialized = Serializer.serialize(globs)
globs_file = path.joinpath("globs.msk")
with globs_file.open('w') as globs_fd:
globs_fd.write(globs_serialized)
globs_fd.flush()
return globs_file
def test_globals():
def f():
a
def g():
b
def h():
c
assert globalvars(f) == dict(a=1, b=2, c=3)
res = globalvars(foo, recurse=True)
assert set(res) == set(['squared', 'a'])
res = globalvars(foo, recurse=False)
assert res == {}
zap = foo(2)
res = globalvars(zap, recurse=True)
assert set(res) == set(['squared', 'a'])
res = globalvars(zap, recurse=False)
assert set(res) == set(['squared'])
del zap
res = globalvars(squared)
assert set(res) == set(['a'])
def test_globals():
def f():
a
def g():
b
def h():
c
assert globalvars(f) == dict(a=1, b=2, c=3)
res = globalvars(foo, recurse=True)
assert set(res) == set(['squared', 'a'])
res = globalvars(foo, recurse=False)
assert res == {}
zap = foo(2)
res = globalvars(zap, recurse=True)
assert set(res) == set(['squared', 'a'])
res = globalvars(zap, recurse=False)
assert set(res) == set(['squared'])
del zap
res = globalvars(squared)
assert set(res) == set(['a'])
else: assert parent(obj, int) is x[-1] # python oddly? finds last int
assert at(id(at)) is at
def f():
a
def g():
b
def h():
c
a, b, c = 1, 2, 3
assert globalvars(f) == dict(a=1, b=2, c=3)
def squared(x):
return a + x**2
def foo(x):
def bar(y):
return squared(x) + y
return bar
class _class:
def _method(self):
x = [4,5,6,7]
listiter = iter(x)
obj = parent(listiter, list)
assert obj is x
assert parent(obj, int) is x[-1]
assert at(id(at)) is at
def f():
a
def g():
b
def h():
c
a, b, c = 1, 2, 3
assert globalvars(f) == dict(a=1, b=2, c=3)
def squared(x):
return a+x**2
def foo(x):
def bar(y):
return squared(x)+y
return bar
class _class:
def _method(self):
pass
def ok(self):
return True
def importable(obj, alias='', source=None, builtin=True):
"""get an importable string (i.e. source code or the import string)
for the given object, including any required objects from the enclosing
and global scope
This function will attempt to discover the name of the object, or the repr
of the object, or the source code for the object. To attempt to force
discovery of the source code, use source=True, to attempt to force the
use of an import, use source=False; otherwise an import will be sought
for objects not defined in __main__. The intent is to build a string
that can be imported from a python file.
obj is the object to inspect. If alias is provided, then rename the
object with the given alias. If builtin=True, then force an import for
builtins where possible.
"""
#NOTE: we always 'force', and 'lstrip' as necessary
#NOTE: for 'enclosing', use importable(outermost(obj))
if source is None:
source = True if isfrommain(obj) else False
elif builtin and isbuiltin(obj):
source = False
tried_source = tried_import = False
while True:
if not source: # we want an import
try:
if _isinstance(obj): # for instances, punt to _importable
return _importable(obj, alias, source=False, builtin=builtin)
src = _closuredimport(obj, alias=alias, builtin=builtin)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
return list(src.values())[0]
except:
if tried_source: raise
tried_import = True
# we want the source
try:
src = _closuredsource(obj, alias=alias)
if len(src) == 0:
raise NotImplementedError('not implemented')
# groan... an inline code stitcher
def _code_stitcher(block):
"stitch together the strings in tuple 'block'"
if block[0] and block[-1]: block = '\n'.join(block)
elif block[0]: block = block[0]
elif block[-1]: block = block[-1]
else: block = ''
return block
# get free_vars first
_src = _code_stitcher(src.pop(None))
_src = [_src] if _src else []
# get func_vars
for xxx in src.values():
xxx = _code_stitcher(xxx)
if xxx: _src.append(xxx)
# make a single source string
if not len(_src):
src = ''
elif len(_src) == 1:
src = _src[0]
else:
src = '\n'.join(_src)
# get source code of objects referred to by obj in global scope
from dill.detect import globalvars
obj = globalvars(obj) #XXX: don't worry about alias?
obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items())
obj = '\n'.join(obj) if obj else ''
# combine all referred-to source (global then enclosing)
if not obj: return src
if not src: return obj
return obj + src
except:
if tried_import: raise
tried_source = True
source = not source
# should never get here
return
def importable(obj, alias='', source=None, builtin=True):
"""get an importable string (i.e. source code or the import string)
for the given object, including any required objects from the enclosing
and global scope
This function will attempt to discover the name of the object, or the repr
of the object, or the source code for the object. To attempt to force
discovery of the source code, use source=True, to attempt to force the
use of an import, use source=False; otherwise an import will be sought
for objects not defined in __main__. The intent is to build a string
that can be imported from a python file.
obj is the object to inspect. If alias is provided, then rename the
object with the given alias. If builtin=True, then force an import for
builtins where possible.
"""
#NOTE: we always 'force', and 'lstrip' as necessary
#NOTE: for 'enclosing', use importable(outermost(obj))
if source is None:
source = True if isfrommain(obj) else False
elif builtin and isbuiltin(obj):
source = False
tried_source = tried_import = False
while True:
if not source: # we want an import
try:
if _isinstance(obj): # for instances, punt to _importable
return _importable(obj, alias, source=False, builtin=builtin)
src = _closuredimport(obj, alias=alias, builtin=builtin)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
return list(src.values())[0]
except:
if tried_source: raise
tried_import = True
# we want the source
try:
src = _closuredsource(obj, alias=alias)
if len(src) == 0:
raise NotImplementedError('not implemented')
# groan... an inline code stitcher
def _code_stitcher(block):
"stitch together the strings in tuple 'block'"
if block[0] and block[-1]: block = '\n'.join(block)
elif block[0]: block = block[0]
elif block[-1]: block = block[-1]
else: block = ''
return block
# get free_vars first
_src = _code_stitcher(src.pop(None))
_src = [_src] if _src else []
# get func_vars
for xxx in src.values():
xxx = _code_stitcher(xxx)
if xxx: _src.append(xxx)
# make a single source string
if not len(_src):
src = ''
elif len(_src) == 1:
src = _src[0]
else:
src = '\n'.join(_src)
# get source code of objects referred to by obj in global scope
from dill.detect import globalvars
obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc?
obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj))
obj = '\n'.join(obj) if obj else ''
# combine all referred-to source (global then enclosing)
if not obj: return src
if not src: return obj
return obj + src
except:
if tried_import: raise
tried_source = True
source = not source
# should never get here
return
def importable(obj, alias='', source=True, builtin=True):
"""get an importable string (i.e. source code or the import string)
for the given object, including any required objects from the enclosing
and global scope
This function will attempt to discover the name of the object, or the repr
of the object, or the source code for the object. To attempt to force
discovery of the source code, use source=True, otherwise an import will be
sought. The intent is to build a string that can be imported from a python
file. obj is the object to inspect. If alias is provided, then rename the
object with the given alias. If builtin=True, then force an import for
builtins where possible.
"""
#NOTE: we always 'force', and 'lstrip' as necessary
#NOTE: for 'enclosing', use importable(outermost(obj))
if builtin and isbuiltin(obj): source = False
tried_source = tried_import = False
while True:
if not source: # we want an import
try:
if _isinstance(obj): # for instances, punt to _importable
return _importable(obj,
alias,
source=False,
builtin=builtin)
src = _closuredimport(obj, alias=alias, builtin=builtin)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
return list(src.values())[0]
except:
if tried_source: raise
tried_import = True
# we want the source
try:
src = _closuredsource(obj, alias=alias)
if len(src) == 0:
raise NotImplementedError('not implemented')
if len(src) > 1:
raise NotImplementedError('not implemented')
src = list(src.values())[0]
if src[0] and src[-1]: src = '\n'.join(src)
elif src[0]: src = src[0]
elif src[-1]: src = src[-1]
else: src = ''
# get source code of objects referred to by obj in global scope
from dill.detect import globalvars
obj = globalvars(obj) #XXX: don't worry about alias?
obj = list(
getsource(_obj, name, force=True)
for (name, _obj) in obj.items())
obj = '\n'.join(obj) if obj else ''
# combine all referred-to source (global then enclosing)
if not obj: return src
if not src: return obj
return obj + src
except:
if tried_import: raise
tried_source = True
source = not source
# should never get here
return
def save_function(pickler, obj):
if not dill._dill._locate_function(obj, pickler):
dill._dill.log.info("F1: %s" % obj)
_recurse = getattr(pickler, "_recurse", None)
# _byref = getattr(pickler, "_byref", None) # TODO: not used
_postproc = getattr(pickler, "_postproc", None)
_main_modified = getattr(pickler, "_main_modified", None)
_original_main = getattr(pickler, "_original_main",
dill._dill.__builtin__) # 'None'
postproc_list = []
if _recurse:
# recurse to get all globals referred to by obj
from dill.detect import globalvars
globs_copy = globalvars(obj, recurse=True, builtin=True)
# Add the name of the module to the globs dictionary to prevent
# the duplication of the dictionary. Pickle the unpopulated
# globals dictionary and set the remaining items after the function
# is created to correctly handle recursion.
globs = {"__name__": obj.__module__}
else:
globs_copy = obj.__globals__ if dill._dill.PY3 else obj.func_globals
# If the globals is the __dict__ from the module being saved as a
# session, substitute it by the dictionary being actually saved.
if _main_modified and globs_copy is _original_main.__dict__:
globs_copy = getattr(pickler, "_main",
_original_main).__dict__
globs = globs_copy
# If the globals is a module __dict__, do not save it in the pickle.
elif (globs_copy is not None and obj.__module__ is not None and
getattr(dill._dill._import_module(obj.__module__, True),
"__dict__", None) is globs_copy):
globs = globs_copy
else:
globs = {"__name__": obj.__module__}
if globs_copy is not None and globs is not globs_copy:
# In the case that the globals are copied, we need to ensure that
# the globals dictionary is updated when all objects in the
# dictionary are already created.
if dill._dill.PY3:
glob_ids = {id(g) for g in globs_copy.values()}
else:
glob_ids = {id(g) for g in globs_copy.itervalues()}
for stack_element in _postproc:
if stack_element in glob_ids:
_postproc[stack_element].append(
(dill._dill._setitems, (globs, globs_copy)))
break
else:
postproc_list.append(
(dill._dill._setitems, (globs, globs_copy)))
# DONE: globs is a dictionary with keys = var names (str) and values = python objects
# however the dictionary is not always loaded in the same order
# therefore we have to sort the keys to make deterministic.
# This is important to make `dump` deterministic.
# Only this line is different from the original implementation:
globs = {k: globs[k] for k in sorted(globs.keys())}
if dill._dill.PY3:
closure = obj.__closure__
state_dict = {}
for fattrname in ("__doc__", "__kwdefaults__",
"__annotations__"):
fattr = getattr(obj, fattrname, None)
if fattr is not None:
state_dict[fattrname] = fattr
if obj.__qualname__ != obj.__name__:
state_dict["__qualname__"] = obj.__qualname__
if "__name__" not in globs or obj.__module__ != globs[
"__name__"]:
state_dict["__module__"] = obj.__module__
state = obj.__dict__
if type(state) is not dict:
state_dict["__dict__"] = state
state = None
if state_dict:
state = state, state_dict
dill._dill._save_with_postproc(
pickler,
(
dill._dill._create_function,
(obj.__code__, globs, obj.__name__, obj.__defaults__,
closure),
state,
),
obj=obj,
postproc_list=postproc_list,
)
else:
closure = obj.func_closure
if obj.__doc__ is not None:
postproc_list.append(
(setattr, (obj, "__doc__", obj.__doc__)))
if "__name__" not in globs or obj.__module__ != globs[
"__name__"]:
postproc_list.append(
(setattr, (obj, "__module__", obj.__module__)))
if obj.__dict__:
postproc_list.append(
(setattr, (obj, "__dict__", obj.__dict__)))
dill._dill._save_with_postproc(
pickler,
(dill._dill._create_function,
(obj.func_code, globs, obj.func_name, obj.func_defaults,
closure)),
obj=obj,
postproc_list=postproc_list,
)
# Lift closure cell update to earliest function (#458)
if _postproc:
topmost_postproc = next(iter(_postproc.values()), None)
if closure and topmost_postproc:
for cell in closure:
possible_postproc = (setattr, (cell, "cell_contents",
obj))
try:
topmost_postproc.remove(possible_postproc)
except ValueError:
continue
# Change the value of the cell
pickler.save_reduce(*possible_postproc)
# pop None created by calling preprocessing step off stack
if dill._dill.PY3:
pickler.write(bytes("0", "UTF-8"))
else:
pickler.write("0")
dill._dill.log.info("# F1")
else:
dill._dill.log.info("F2: %s" % obj)
name = getattr(obj, "__qualname__", getattr(obj, "__name__", None))
dill._dill.StockPickler.save_global(pickler, obj, name=name)
dill._dill.log.info("# F2")
return
def save_function(pickler, obj):
"""
From dill._dill.save_function
This is a modified version that make globs deterministic since the order of
the keys in the output dictionary of globalvars can change.
"""
if not dill._dill._locate_function(obj):
dill._dill.log.info(f"F1: {obj}")
if getattr(pickler, "_recurse", False):
# recurse to get all globals referred to by obj
globalvars = dill.detect.globalvars
globs = globalvars(obj, recurse=True, builtin=True)
if id(obj) in dill._dill.stack:
globs = obj.__globals__ if dill._dill.PY3 else obj.func_globals
else:
globs = obj.__globals__ if dill._dill.PY3 else obj.func_globals
# globs is a dictionary with keys = var names (str) and values = python objects
# however the dictionary is not always loaded in the same order
# therefore we have to sort the keys to make deterministic.
# This is important to make `dump` deterministic.
# Only this line is different from the original implementation:
globs = {k: globs[k] for k in sorted(globs.keys())}
# The rest is the same as in the original dill implementation
_byref = getattr(pickler, "_byref", None)
_recurse = getattr(pickler, "_recurse", None)
_memo = (id(obj) in dill._dill.stack) and (_recurse is not None)
dill._dill.stack[id(obj)] = len(dill._dill.stack), obj
if dill._dill.PY3:
_super = ("super" in getattr(obj.__code__, "co_names",
())) and (_byref is not None)
if _super:
pickler._byref = True
if _memo:
pickler._recurse = False
fkwdefaults = getattr(obj, "__kwdefaults__", None)
pickler.save_reduce(
dill._dill._create_function,
(obj.__code__, globs, obj.__name__, obj.__defaults__,
obj.__closure__, obj.__dict__, fkwdefaults),
obj=obj,
)
else:
_super = (("super" in getattr(obj.func_code, "co_names",
())) and (_byref is not None)
and getattr(pickler, "_recurse", False))
if _super:
pickler._byref = True
if _memo:
pickler._recurse = False
pickler.save_reduce(
dill._dill._create_function,
(obj.func_code, globs, obj.func_name, obj.func_defaults,
obj.func_closure, obj.__dict__),
obj=obj,
)
if _super:
pickler._byref = _byref
if _memo:
pickler._recurse = _recurse
if (dill._dill.OLDER and not _byref
and (_super or (not _super and _memo) or
(not _super and not _memo and _recurse))):
pickler.clear_memo()
dill._dill.log.info("# F1")
else:
dill._dill.log.info(f"F2: {obj}")
name = getattr(obj, "__qualname__", getattr(obj, "__name__", None))
dill._dill.StockPickler.save_global(pickler, obj, name=name)
dill._dill.log.info("# F2")
return
