def rename_function(fn, newname):
"""Create a copy of the function with a different name."""
co = fn.__code__
extra_args = []
if hasattr(co, "co_posonlyargcount"): # pragma: no cover
extra_args.append(co.co_posonlyargcount)
newcode = type(co)(
co.co_argcount,
*extra_args,
co.co_kwonlyargcount,
co.co_nlocals,
co.co_stacksize,
co.co_flags,
co.co_code,
co.co_consts,
co.co_names,
co.co_varnames,
co.co_filename,
newname,
co.co_firstlineno,
co.co_lnotab,
co.co_freevars,
co.co_cellvars,
)
new_fn = FunctionType(newcode, fn.__globals__, newname, fn.__defaults__,
fn.__closure__)
new_fn.__annotations__ = fn.__annotations__
return new_fn
def __init__(self, addr, **kwargs): self.thread = Thread(target=self.serve_forever, daemon=True) self.__Request.selfmain = self self.setup = FunctionType(self.setup.__code__, self.setup.__globals__) self.handle = FunctionType(self.handle.__code__, self.handle.__globals__) self.finish = FunctionType(self.finish.__code__, self.finish.__globals__) super().__init__(addr, self.__Request, True)
def FactoryBindingSpec(name, prototype):
r"""
Return a BindingSpec that calls ``prototype`` as a provider for ``name``.
EXAMPLES::
>>> def create(dependency): return 1337
>>> binding = FactoryBindingSpec("object", create)
>>> inspect.signature(binding.provide_object)
<Signature (dependency)>
"""
from types import FunctionType
signature = inspect.signature(prototype)
args = [param for param in signature.parameters.keys()]
provider = f"def provide_{name}(self, {', '.join(args)}): return prototype({', '.join(args)})"
provider = FunctionType(
compile(provider, "<string>", "exec").co_consts[0],
{"prototype": prototype},
f"provide_{name}",
)
provider.__module__ = "__main__"
binding = type(
f"{name}FactoryBinding",
(pinject.BindingSpec, ),
{
f"provide_{name}": provider,
"__repr__": lambda self: f"{name}->{prototype}"
},
)()
binding.name = name
return binding
def _deserialize_func(funcs, globalDict):
items = pickle.loads(funcs)
res = None
for objType, name, data in items:
if objType == 'func':
codeArgs, funcArgs, updatedGlobals = pickle.loads(data)
code = CodeType(*codeArgs)
globalDict.update(**updatedGlobals)
value = FunctionType(code, globalDict, *funcArgs)
elif objType == 'mod':
value = __import__(data)
elif objType == 'oldclass':
class_name, module, bases, class_dict = data
value = typesmod.ClassType(class_name, bases, {k:_deserialize_func(v, globalDict) for k, v in class_dict.items()})
value.__module__ = module
elif objType == 'type':
raise Exception('deserialize type')
else:
raise Exception('Unknown serialization type')
globalDict[name] = value
if res is None:
res = value
return res
def wrap(**kwargs):
injected = [
param.name for param in signature.parameters.values()
if param.name not in kwargs and param.default is inspect._empty
]
args = [f"{arg}={arg}" for arg in injected + list(kwargs.keys())]
provider = f"def provide_{name}(self, {', '.join(injected)}): return prototype({', '.join(args)})"
provider = FunctionType(
compile(provider, "<string>", "exec").co_consts[0],
{
**kwargs, "prototype": prototype
},
f"provide_{name}",
)
provider.__module__ = "__main__"
binding = type(
f"Partial{prototype.__name__}Binding",
(pinject.BindingSpec, ),
{
f"provide_{name}": provider,
"__repr__": lambda self: f"{name}->{prototype.__name__}",
},
)()
binding.name = name
return binding
def hax(target: T) -> T:
if isinstance(target, CodeType):
return cast(T, _hax(target))
if isinstance(target, FunctionType):
new = FunctionType(
_hax(target.__code__),
target.__globals__,
target.__name__,
target.__defaults__,
target.__closure__,
)
if target.__annotations__:
new.__annotations__ = target.__annotations__.copy()
if target.__kwdefaults__ is not None:
new.__kwdefaults__ = target.__kwdefaults__.copy()
if target.__dict__:
new.__dict__ = target.__dict__.copy()
return cast(T, new)
raise TypeError(f"HAX doesn't support this! Got type {type(target)!r}.")
def chained_function(meta, func, mod):
d = ModuleChainedDict(mod.__dict__, func.__globals__)
newfunc = FunctionType(func.__code, d)
newfunc.__doc__ = func.__doc__
newfunc.__defaults__ = newfunc.__defaults__
newfunc.__kwdefaults__ = func.__kwdefaults__
return newfunc
def copy_func(f: FunctionType, name, defaults, signature: Signature):
"""
Makes exact copy of a function object with given name and defaults
"""
new_defaults = []
kw_only_defaults = f.__kwdefaults__.copy() if f.__kwdefaults__ else {}
for key, param in signature.parameters.items():
if param.kind is param.KEYWORD_ONLY:
if key in defaults:
kw_only_defaults[key] = defaults.pop(key)
elif key in defaults:
new_defaults.append(defaults.pop(key))
elif param.default is not param.empty:
new_defaults.append(param.default)
new_func = FunctionType(
code=copy_code(f.__code__, co_name=name),
globals=f.__globals__,
name=name,
argdefs=tuple(new_defaults),
closure=f.__closure__,
)
new_func.__kwdefaults__ = kw_only_defaults
new_func.__dict__.update(f.__dict__)
return new_func
def newObjCpt(self, elt):
argNames = self.__code__.co_varnames[:self.__code__.co_argcount]
argValues = [elt]
wfunc = FunctionType(self.__code__, self.__globals__)
class Process(object):
__call__ = staticmethod(wfunc)
def __setattr__(s, name, value):
argValues[argNames.index(name)] = value
wfunc.__defaults__ = tuple(argValues)
def __getattr__(s, name):
return argValues[argNames.index(name)]
@classmethod
def __cptDefs__(cls):
cpts= {name : cpt for (name, cpt) in zip( argNames, self.__defaults__) if isinstance(cpt, _cptDef)}
return sorted(cpts.items(),key =lambda item:item[1].id)
process = Process()
argValues += [p.newObjCpt(process) for p in self.__defaults__[1:]]
wfunc.__defaults__ = tuple(argValues)
return process
def _deserialize_func(funcs, globalDict):
items = pickle.loads(funcs)
res = None
for objType, name, data in items:
if objType == 'func':
codeArgs, funcArgs, updatedGlobals = pickle.loads(data)
code = CodeType(*codeArgs)
globalDict.update(**updatedGlobals)
value = FunctionType(code, globalDict, *funcArgs)
elif objType == 'mod':
value = __import__(data)
elif objType == 'oldclass':
class_name, module, bases, class_dict = data
value = typesmod.ClassType(class_name, bases, {k:_deserialize_func(v, globalDict) for k, v in class_dict.items()})
value.__module__ = module
elif objType == 'type':
raise Exception('deserialize type')
else:
raise Exception('Unknown serialization type')
globalDict[name] = value
if res is None:
res = value
return res
def dict_to_function(obj):
is_recursive = False
function_globals = obj['__globals__']
for name, out_obj in function_globals.items():
if name == obj['__name__']:
is_recursive = True
function_globals[name] = deserialize(out_obj)
function_globals['__builtins__'] = __builtins__
code = obj['__code__']
for i in range(len(code)):
if i == 13 and code[i] is None:
code[i] = b''
if code[i] is None:
code[i] = ()
elif isinstance(code[i], list):
code[i] = bytes(code[i])
func = FunctionType(CodeType(*code), function_globals, obj['__name__'],
obj['__defaults__'], None)
if is_recursive:
func.__getattribute__('__globals__')[obj['__name__']] = func
return func
def reader(name=None, doc=None, mp_flattener=False):
"""Construct a new unified input/output reader.
This method is required to create a new copy of the
:func:`astropy.io.registry.read` with a dynamic docstring.
Returns
-------
read : `function`
a copy of the :func:`astropy.io.registry.read` function
doc : `str`
custom docstring for this reader
mp_flattener : `function`
the function to flatten multiple instances of the parent object,
enabling multiprocessed reading via the `nproc` argument
"""
func = FunctionType(read.func_code, read.func_globals,
name or read.func_name, read.func_defaults,
read.func_closure)
if doc is not None:
func.__doc__ = doc.strip('\n ')
if mp_flattener:
return with_nproc(func, mp_flattener)
else:
return func
def deserialize_func(func):
code_fields = func[CODE_FIELD][VALUE_FIELD]
code_args = []
for field in CODE_ARGS:
arg = code_fields[field]
if type(arg) == dict:
if arg[TYPE_FIELD] == "bytes":
code_args.append(bytes(arg[VALUE_FIELD]))
else:
code_args.append(tuple(arg[VALUE_FIELD]))
else:
code_args.append(arg)
details = [CodeType(*code_args)]
glob = {"__builtins__": __builtins__}
for name, o in func[GLOBALS].items():
glob[name] = deserialize_obj(o)
details.append(glob)
for attr in FUNCTION_ATTRIBUTES:
if attr == CODE_FIELD:
continue
details.append(deserialize_obj(func[attr]))
result_func = FunctionType(*details)
if result_func.__name__ in result_func.__getattribute__(GLOBALS):
result_func.__getattribute__(GLOBALS)[
result_func.__name__] = result_func
return result_func
def execute(self, function_code: str, params: List):
try:
entry_function_name, program_code = self._get_entry_function_name(
function_code)
if entry_function_name is None or program_code is None:
return None
machine_code = compile(program_code, '', 'exec')
if machine_code is None:
return None
entry_point = None
lib_functions = []
for fcode in machine_code.co_consts:
if isinstance(fcode, CodeType):
if fcode.co_name == entry_function_name:
entry_point = FunctionType(fcode, self._sandbox)
else:
lib_functions.append(
(fcode.co_name, FunctionType(fcode,
self._sandbox)))
if entry_point is None:
return None
for lf in lib_functions: #enable calling of all functions but not the entry point
if lf[1] is None:
continue
self._sandbox[lf[0]] = lf[1]
return entry_point(*params)
except:
#raise
return None
def get_object(self, g=None):
# try to load function back into its module:
if not self.module.startswith('__'):
__import__(self.module)
g = sys.modules[self.module].__dict__
if g is None:
g = {}
if self.defaults:
defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
else:
defaults = None
if self.kwdefaults:
kwdefaults = uncan_dict(self.kwdefaults)
else:
kwdefaults = None
if self.annotations:
annotations = uncan_dict(self.annotations)
else:
annotations = {}
if self.closure:
closure = tuple(uncan(cell, g) for cell in self.closure)
else:
closure = None
newFunc = FunctionType(self.code, g, self.__name__, defaults, closure)
if kwdefaults:
newFunc.__kwdefaults__ = kwdefaults
if annotations:
newFunc.__annotations__ = annotations
return newFunc
def _update_function(oldfunc: FunctionType, newfunc: FunctionType):
"""Update a function object."""
logger.info(f"Patch function {oldfunc.__qualname__}")
oldfunc.__doc__ = newfunc.__doc__
oldfunc.__dict__.update(newfunc.__dict__)
oldfunc.__annotations__ = newfunc.__annotations__
oldfunc.__code__ = newfunc.__code__
oldfunc.__defaults__ = newfunc.__defaults__
def deserialize(d):
d = dict((a, b) for a, b in d)
object_type = d["type"]
ans = None
if object_type == "list":
ans = [Serializer.deserialize(i) for i in d["value"]]
elif object_type == "dict":
ans = {}
if "value" not in d:
raise InvalidTypeSourceException()
for i in d["value"]:
val = Serializer.deserialize(i[1])
ans[Serializer.deserialize(i[0])] = val
elif object_type == "tuple":
ans = tuple([Serializer.deserialize(i) for i in d["value"]])
elif object_type == "function":
func = [0] * 4
code = [0] * 16
glob = {"__builtins__": __builtins__}
for i in d["value"]:
key = Serializer.deserialize(i[0])
if key == "__globals__":
globdict = Serializer.deserialize(i[1])
for globkey in globdict:
glob[globkey] = globdict[globkey]
elif key == "__code__":
val = i[1][1][1]
for arg in val:
codeArgKey = Serializer.deserialize(arg[0])
if codeArgKey != "__doc__":
codeArgVal = Serializer.deserialize(arg[1])
index = CODE_OBJECT_ARGS.index(codeArgKey)
code[index] = codeArgVal
code = CodeType(*code)
else:
index = FUNCTION_ATTRIBUTES.index(key)
func[index] = (Serializer.deserialize(i[1]))
func[0] = code
func.insert(1, glob)
ans = FunctionType(*func)
if ans.__name__ in ans.__getattribute__("__globals__"):
ans.__getattribute__("__globals__")[ans.__name__] = ans
elif object_type == "NoneType":
ans = None
elif object_type == "bytes":
ans = bytes([Serializer.deserialize(i) for i in d["value"]])
else:
if object_type == "bool":
ans = d["value"] == "True"
else:
ans = locate(object_type)(d["value"])
return ans
def make_inner_f_cores(f, *args):
f_list = []
for co_const in f.__code__.co_consts:
if iscode(co_const) and co_const.co_freevars:
f_list.append(FunctionType(co_const, f.__globals__, name=co_const.co_name, closure=make_closure(f, co_const, *args)))
elif iscode(co_const) and not co_const.co_freevars:
f_list.append(FunctionType(co_const, f.__globals__, name=co_const.co_name))
return f_list
def build_get_single_ctrl(query, args_names, field_names):
"""
Create optimized controller.
"""
codes = [
opcode.opmap['LOAD_GLOBAL'], 0, 0,
opcode.opmap['LOAD_CONST'], 1, 0
]
for i in range(len(args_names)):
codes.extend((opcode.opmap['LOAD_FAST'], i, 0))
codes.extend((
opcode.opmap['BUILD_TUPLE'], len(args_names), 0,
opcode.opmap['CALL_FUNCTION'], 2, 0,
opcode.opmap['LOAD_ATTR'], 1, 0,
opcode.opmap['CALL_FUNCTION'], 0, 0,
opcode.opmap['STORE_FAST'], len(args_names), 0, # store to 'values'
opcode.opmap['LOAD_FAST'], len(args_names), 0, # load from 'values'
opcode.opmap['LOAD_CONST'], 0, 0,
opcode.opmap['COMPARE_OP'], 8, 0,
opcode.opmap['POP_JUMP_IF_FALSE'], len(args_names) * 3 + 37, 0,
opcode.opmap['LOAD_FAST'], len(args_names), 0,
opcode.opmap['RETURN_VALUE'],
opcode.opmap['LOAD_GLOBAL'], 2, 0, # (dict)
opcode.opmap['LOAD_GLOBAL'], 3, 0, # (zip)
opcode.opmap['LOAD_CONST'], 2, 0,
opcode.opmap['LOAD_FAST'], len(args_names), 0, # load from 'values'
opcode.opmap['CALL_FUNCTION'], 2, 0,
opcode.opmap['CALL_FUNCTION'], 1, 0,
opcode.opmap['RETURN_VALUE'],
))
ctrl = FunctionType(
CodeType(
len(args_names), # argcount
0, # kwonlyargcount
len(args_names) + 1, # + nb var used
7, # stacksize
67, # flags
bytes(codes), # codestring
(None, query, field_names), # constants
('sql_execute', 'fetchone', 'dict', 'zip'), # names
tuple(args_names) + ('values',), # varnames + var used
__file__, # filename
'ctrl', # name
0, # firstlineno
b'\x00\x01\x1b\x01\x0c\x01\x04\x01' # lnotab
), {'dict': dict, 'zip': zip, 'sql_execute': MetaResource.db.execute}
)
ctrl.single = True
ctrl.optimizable = True
return ctrl
def _save_generator_impl(self, frame, gen, filler):
if frame is None:
# frame is None when the generator is fully consumed; take a fast path
self.save_reduce(
_restore_spent_generator,
(gen.__name__, getattr(gen, '__qualname__', None)),
obj=gen,
)
return
f_locals = frame.f_locals
f_code = frame.f_code
# Create a copy of generator function without the closure to serve as a box
# to serialize the code, globals, name, and closure. Cloudpickle already
# handles things like closures and complicated globals so just rely on
# cloudpickle to serialize this function.
gen_func = FunctionType(
f_code,
frame.f_globals,
gen.__name__,
(),
(_empty_cell(), ) * len(f_code.co_freevars),
)
try:
gen_func.__qualname__ = gen.__qualname__
except AttributeError:
# there is no __qualname__ on generators in Python < 3.5
pass
save = self.save
write = self.write
# push a function onto the stack to fill up our skeleton generator
# or coroutine
save(filler)
# the start of the tuple to pass to ``_fill_generator`` (or
# ``_fill_coroutine``, ``_fill_async_generator``)
write(pickle.MARK)
save(_create_skeleton_generator)
save((gen_func, ))
write(pickle.REDUCE)
self.memoize(gen)
# push the rest of the arguments to ``_fill_generator`` (or
# ``_fill_coroutine``, ``_fill_async_generator``)
save(frame.f_lasti)
save(f_locals)
save(private_frame_data(frame))
# call ``_fill_generator`` (or ``_fill_coroutine``,
# _fill_async_generator``)
write(pickle.TUPLE)
write(pickle.REDUCE)
def _resolve_binding_globals(self, node):
try:
operator_func = self._operators[node.operator]
except KeyError:
self._raise_error(OperatorLookupError, node.operator, node)
node.operator_func = operator_func
f_globals = self._f_globals
node.func = FunctionType(node.code, f_globals, node.name)
if node.auxcode is not None:
node.auxfunc = FunctionType(node.auxcode, f_globals, node.name)
def create_function(name, first_arg, args, defaults, function_code):
formatted_args = ", ".join("{}".format(arg) for arg in args)
formatted_args = (formatted_args +
", **kwargs" if args else formatted_args + " **kwargs")
function_code = "def {}({}, {}): {}".format(name, first_arg,
formatted_args, function_code)
compiled_func = compile(function_code, name, "exec")
function = FunctionType(compiled_func.co_consts[0], globals(), name)
function.__defaults__ = tuple(defaults)
return function
def handle_deffun(self, func, fdict, fdoc, remote_globals):
func = self.unpack(func)
g = globals()
glbls = {k:g[k] for k in remote_globals if k in g} if remote_globals is not None else g.copy()
glbls.update(func[1])
func[1].update(glbls)
f = FunctionType(*func)
f.__dict__ = self.unpack(fdict)
f.__doc__ = self.unpack(fdoc)
return self.pack(f)
def _create_function(fcode, fglobals, fname=None, fdefaults=None, fclosure=None, fdict=None, mod_name=None):
# same as FunctionType, but enable passing __dict__ to new function,
# __dict__ is the storehouse for attributes added after function creation
log.info('loading function: ' + fname)
fdict = fdict or dict()
fglobals = fglobals or {}
func = FunctionType(fcode, fglobals, fname, fdefaults, fclosure)
func.__dict__.update(fdict)
func.__module__ = mod_name
return func
def _clone_function(f):
'''Make a copy of a function'''
# based on http://stackoverflow.com/a/13503277/2289509
new_f = FunctionType(f.__code__, f.__globals__,
name=f.__name__,
argdefs=f.__defaults__,
closure=f.__closure__)
new_f = update_wrapper(new_f, f)
new_f.__kwdefaults__ = f.__kwdefaults__
return new_f
def copy_func(f: Callable) -> Callable:
"""Duplicate a function object"""
g = FunctionType(f.__code__,
f.__globals__,
name=f.__name__,
argdefs=f.__defaults__,
closure=f.__closure__)
g = update_wrapper(g, f)
g.__kwdefaults__ = f.__kwdefaults__
return g
def _make_typedarg_rule(f, name):
"""Copy a rule and allow for annotations.
"""
rule = FunctionType(f.__code__, f.__globals__, name, f.__defaults__,
f.__closure__)
new_doc = f.__doc__.replace('fpdef', 'tfpdef')
rule.__doc__ = new_doc.replace('varargslist', 'typedargslist')
return rule
def copy_func(func: FunctionType, deep: bool = False) -> FunctionType:
"""Copy a function as a different object.
Args:
func: Function object to be copied.
deep: If ``True``, mutable attributes of ``func`` are deep-copied.
Returns:
Function as a different object from the original one.
"""
copied = FunctionType(
func.__code__,
func.__globals__,
func.__name__,
func.__defaults__,
func.__closure__,
)
# mutable attributes are copied by the given method
copier = deepcopy if deep else copy
copied.__annotations__ = copier(func.__annotations__)
copied.__dict__ = copier(func.__dict__)
copied.__kwdefaults__ = copier(func.__kwdefaults__)
# immutable attributes are not copied (just assigned)
copied.__doc__ = func.__doc__
copied.__module__ = func.__module__
copied.__name__ = func.__name__
copied.__qualname__ = func.__qualname__
return copied
def __new__(cls, mplayer=MPLAYER_PATH, pipe=PIPE_PATH,
stdout=STDOUT_PATH, pid=PID_PATH, debug=False):
def _doc_creator(item):
## Doc creator for the command
doc_info = item['comment']
py_command = item['pycommand']
doc = '%s\n%s' % (py_command, doc_info)
return doc
## Creating new class methods from mplayer cmdlist_dict
cmdlist_dict = CmdDictGenerator(mplayer).get_cmdlist()
for item in cmdlist_dict.keys():
if item == 'get_property': continue
if item == 'set_property': continue
#if item == 'set_property_osd': continue
doc = _doc_creator(cmdlist_dict[item])
# Creating a dictionary that would include variables from
# item and globals() (excluding locals()).
# This is necessary for passing it to a new method.
method_dict = {'item': cmdlist_dict[item]}
for i in globals().keys():
if i in locals().keys(): continue
method_dict[i] = globals()[i]
# Creating a function
if 'get' not in item:
if len(cmdlist_dict[item]['types']) != 0:
# If list of types contains some types
new_method = FunctionType(cls._new_args_method.func_code,
method_dict,
item)
else:
# If list of types is empty
new_method = FunctionType(cls._new_simple_method.func_code,
method_dict,
item)
else:
new_method = FunctionType(cls._new_get_method.func_code,
method_dict,
item)
# Adding doc, editing name
new_method.__doc__ = doc
new_method.__name__ = item
# Adding function to this class as a method
setattr(cls, item, new_method)
# Create 'properties' property and
# making it use the doc from Properties class
properties_class = Properties()
def get_properties(self):
return properties_class
properties = property(fget=get_properties,
doc=Properties.__doc__)
setattr(cls, 'properties', properties)
return super(Player, cls).__new__(cls)
def fake_import(fake_module):
module_name = 'martiantest.fake.' + fake_module.__name__
module = ModuleType(module_name)
module_name_parts = module_name.split('.')
module.__file__ = '/' + '/'.join(module_name_parts)
glob = {}
for name in dir(fake_module):
if name.startswith('__') and '.' not in name:
continue
obj = getattr(fake_module, name)
glob[name] = obj
try:
obj = obj.im_func
except AttributeError:
pass
__module__ = None
try:
__module__ == obj.__dict__.get('__module__')
except AttributeError:
try:
__module__ = obj.__module__
except AttributeError:
pass
if __module__ is None or __module__ == '__builtin__':
try:
obj.__module__ = module.__name__
except AttributeError:
pass
setattr(module, name, obj)
# provide correct globals for functions
for name in dir(module):
if name.startswith('__'):
continue
obj = getattr(module, name)
try:
code = obj.__code__
new_func = FunctionType(code, glob, name)
new_func.__module__ = module.__name__
setattr(module, name, new_func)
glob[name] = new_func
except AttributeError:
pass
if not 'martiantest' in sys.modules:
sys.modules['martiantest'] = ModuleType('martiantest')
sys.modules['martiantest.fake'] = ModuleType('martiantest.fake')
sys.modules['martiantest'].fake = sys.modules['martiantest.fake']
sys.modules[module_name] = module
setattr(sys.modules['martiantest.fake'],
module_name.split('.')[-1], module)
return module
def handle_deffun(self, func, fdict, fdoc, remote_globals):
func = self.unpack(func)
g = globals()
glbls = {k: g[k]
for k in remote_globals
if k in g} if remote_globals is not None else g.copy()
glbls.update(func[1])
func[1].update(glbls)
f = FunctionType(*func)
f.__dict__ = self.unpack(fdict)
f.__doc__ = self.unpack(fdoc)
return self.pack(f)
def interpolate(self, obj, name):
"""Inject the formatted listing in the second blank line of `name`."""
f = getattr(obj, name)
f2 = FunctionType(f.__code__, f.__globals__, name=f.__name__, argdefs=f.__defaults__, closure=f.__closure__)
# Conveniently the original docstring is on f2, not the new ones if
# inheritence is happening. I have no idea why.
t = f2.__doc__.split("\n\n")
t.insert(2, self.formatted_listing())
f2.__doc__ = "\n\n".join(t)
setattr(obj, name, f2)
def alias(name: str, doc: str, fun: Callable[..., Any]) -> Callable[..., Any]:
# Adapted from https://stackoverflow.com/questions/13503079/how-to-create-a-copy-of-a-python-function#
# See also help(type(lambda: 0))
alias = FunctionType(fun.__code__,
fun.__globals__,
name=name,
argdefs=fun.__defaults__,
closure=fun.__closure__)
alias = update_wrapper(alias, fun)
alias.__kwdefaults__ = fun.__kwdefaults__
alias.__doc__ = doc
return alias
def interpolate(self, obj, name):
"""Inject the formatted listing in the second blank line of `name`."""
f = getattr(obj, name)
f2 = FunctionType(f.__code__, f.__globals__, name=f.__name__,
argdefs=f.__defaults__, closure=f.__closure__)
# Conveniently the original docstring is on f2, not the new ones if
# inheritence is happening. I have no idea why.
t = f2.__doc__.split("\n\n")
t.insert(2, self.formatted_listing())
f2.__doc__ = "\n\n".join(t)
setattr(obj, name, f2)
def _copy_handler(self, f, kwargs):
fname = '_'.join(
getattr(v, 'name', v.__class__.__name__) for v in kwargs.values())
"""Creates a copy of the given handler and injects the kwargs as func variables"""
fn = FunctionType(f.__code__, f.__globals__, f'{f.__name__}_{fname}',
f.__defaults__, f.__closure__)
# in case f was given attrs (note this dict is a shallow copy):
fn.__dict__.update(f.__dict__)
# Inject the route kwargs
fn.__dict__.update(kwargs)
# Copy the type hints so Vibora can optimize stuff
fn.__annotations__ = f.__annotations__
return fn
def rebindFunction(f, rebindDir=None, **rebinds):
'''return *f* with some globals rebound.'''
d = {}
if rebindDir: d.update(rebindDir)
if rebinds: d.update(rebinds)
if not d: return f
f = getattr(f, 'im_func', f)
fd = f.func_globals.copy()
fd.update(d)
nf = FunctionType(f.func_code, fd, f.func_name, f.func_defaults or ())
nf.__doc__ = f.__doc__
if f.__dict__ is not None: nf.__dict__ = f.__dict__.copy()
return nf
def deserialize(obj):
d = dict((a, b) for a, b in obj)
obj_type = d['type']
res = None
if obj_type == 'list':
res = [Serializer.deserialize(i) for i in d['value']]
elif obj_type == 'tuple':
res = tuple([Serializer.deserialize(i) for i in d['value']])
elif obj_type == 'dict':
res = {}
for i in d['value']:
value = Serializer.deserialize(i[1])
res[Serializer.deserialize(i[0])] = value
elif obj_type == 'function':
func = [0] * 4
code = [0] * 16
glob = {'__builtins__': __builtins__}
for i in d['value']:
key = Serializer.deserialize(i[0])
if key == '__globals__':
global_d = Serializer.deserialize(i[1])
for global_key in global_d:
glob[global_key] = global_d[global_key]
elif key == '__code__':
value = i[1][1][1]
for arg in value:
codeArgKey = Serializer.deserialize(arg[0])
if codeArgKey != '__doc__':
codeArgVal = Serializer.deserialize(arg[1])
index = CODE_OBJECT_ARGS.index(codeArgKey)
code[index] = codeArgVal
code = CodeType(*code)
else:
index = FUNC_ATTRS.index(key)
func[index] = (Serializer.deserialize(i[1]))
func[0] = code
func.insert(1, glob)
res = FunctionType(*func)
if res.__name__ in res.__getattribute__('__globals__'):
res.__getattribute__('__globals__')[res.__name__] = res
elif obj_type == 'NoneType':
res = None
elif obj_type == 'bytes':
res = bytes([Serializer.deserialize(i) for i in d['value']])
else:
if obj_type == 'bool':
res = d['value'] == 'True'
else:
res = locate(obj_type)(d['value'])
return res
def interactive(f):
"""decorator for making functions appear as interactively defined.
This results in the function being linked to the user_ns as globals()
instead of the module globals().
"""
# build new FunctionType, so it can have the right globals
# interactive functions never have closures, that's kind of the point
if isinstance(f, FunctionType):
mainmod = __import__("__main__")
f = FunctionType(f.__code__, mainmod.__dict__, f.__name__, f.__defaults__)
# associate with __main__ for uncanning
f.__module__ = "__main__"
return f
def conform(self, obj1: types.FunctionType, obj2: types.FunctionType):
fv1 = obj1.__code__.co_freevars
fv2 = obj2.__code__.co_freevars
if fv1 != fv2:
msg = (
f"Cannot replace closure `{obj1.__name__}` because the free "
f"variables changed. Before: {fv1}; after: {fv2}."
)
if ("__class__" in (fv1 or ())) ^ ("__class__" in (fv2 or ())):
msg += " Note: The use of `super` entails the `__class__` free variable."
raise ConformException(msg)
obj1.__code__ = obj2.__code__
obj1.__defaults__ = obj2.__defaults__
obj1.__kwdefaults__ = obj2.__kwdefaults__
def buildFunction(baseFunc, code=None, glbls=None,
name=None, defaults=None,
kwdefaults=None, closure=None,
annotations=None, doc=None, dct=None):
resf = None
def _f():
pass
if hasattr(_f, 'func_code'):
# Python 2.x
resf = FunctionType(code or baseFunc.func_code,
glbls or baseFunc.func_globals,
name or baseFunc.func_name,
defaults or baseFunc.func_defaults,
closure or baseFunc.func_closure)
resf.func_dict = dct or baseFunc.func_dict
resf.func_doc = doc or baseFunc.func_doc
else:
# Python 3.x
resf = FunctionType(code or baseFunc.__code__,
glbls or baseFunc.__globals__,
name or baseFunc.__name__,
defaults or baseFunc.__defaults__,
closure or baseFunc.__closure__)
resf.__kwdefaults__ = kwdefaults or baseFunc.__kwdefaults__
resf.__annotations__ = annotations or baseFunc.__annotations__
resf.__dict__ = dct or baseFunc.__dict__
resf.__doc__ = doc or baseFunc.__doc__
return resf
def loads_function(s):
'''Restores a function serialized with :func:`dumps_function`.'''
name, code, globals_, defaults, closure, func_dict = loads(s)
code = marshal.loads(code)
for k, v in globals_.iteritems():
if isinstance(v, Module):
globals_[k] = v.mod
if closure is not None:
import ctypes
ctypes.pythonapi.PyCell_New.restype = ctypes.py_object
ctypes.pythonapi.PyCell_New.argtypes = [ctypes.py_object]
closure = tuple(ctypes.pythonapi.PyCell_New(c) for c in closure)
r = FunctionType(code, globals_, name, defaults, closure)
r.func_dict = func_dict
return r
def rust_bind(fn: FunctionType) -> FunctionType:
if hasattr(fn, '_bind_to_rust'):
raise AttributeError(
"the attribute name `_bind_to_rust` is reserved for binding "
"functions of RustyPy")
fn._bind_to_rust = True
return fn
def reader(name=None, doc=None):
"""Construct a new unified input/output reader.
This method is required to create a new copy of the
:func:`astropy.io.registry.read` with a dynamic docstring.
Returns
-------
read : `function`
A copy of the :func:`astropy.io.registry.read` function
"""
func = FunctionType(read.func_code, read.func_globals,
name or read.func_name, read.func_defaults,
read.func_closure)
if doc is not None:
func.__doc__ = doc.strip('\n ')
return func
def writer(doc=None):
"""Construct a new unified input/output writeer.
This method is required to create a new copy of the
:func:`astropy.io.registry.write` with a dynamic docstring.
Returns
-------
write : `function`
A copy of the :func:`astropy.io.registry.write` function
"""
func = FunctionType(write.func_code, write.func_globals,
write.func_name, write.func_defaults,
write.func_closure)
if doc is not None:
func.__doc__ = doc.strip('\n ')
return func
def interpolate(self, obj, name):
"""Inject the formatted listing in the second blank line of `name`."""
# Py2/3 compatible way of calling getattr(obj, name).__func__
f = getattr(obj, name).__get__(None, type(None))
if hasattr(f, 'func_code'):
f2 = FunctionType(f.func_code, f.func_globals, name=f.func_name,
argdefs=f.func_defaults, closure=f.func_closure)
else:
f2 = FunctionType(f.__code__, f.__globals__, name=f.__name__,
argdefs=f.__defaults__, closure=f.__closure__)
# Conveniently the original docstring is on f2, not the new ones if
# inheritence is happening. I have no idea why.
t = f2.__doc__.split("\n\n")
t.insert(2, self.formatted_listing())
f2.__doc__ = "\n\n".join(t)
setattr(obj, name, f2)
def setup_bindings(instance, bindings, identifiers, f_globals):
""" Setup the expression bindings for a declarative instance.
Parameters
----------
instance : Declarative
The declarative instance which owns the bindings.
bindings : list
A list of binding dicts created by the enaml compiler.
identifiers : dict
The identifiers scope to associate with the bindings.
f_globals : dict
The globals dict to associate with the bindings.
"""
operators = instance.operators
for binding in bindings:
opname = binding['operator']
try:
operator = operators[opname]
except KeyError:
filename = binding['filename']
lineno = binding['lineno']
block = binding['block']
raise OperatorLookupError(opname, filename, lineno, block)
code = binding['code']
# If the code is a tuple, it represents a delegation
# expression which is a combination of subscription
# and update functions.
if isinstance(code, tuple):
sub_code, upd_code = code
func = FunctionType(sub_code, f_globals)
func._update = FunctionType(upd_code, f_globals)
else:
func = FunctionType(code, f_globals)
operator(instance, binding['name'], func, identifiers)
def _clone(origin_fun, new_fun_name, name, extra=None):
# update the checkers/accessors dico
original_checkers = origin_fun.checker.arg_type_list
new_dict = dict(original_checkers)
if extra is not None:
new_dict.update(extra)
# clone the function
new_fun = FunctionType(code=origin_fun.__code__,
globals=origin_fun.__globals__,
name=new_fun_name,
argdefs=origin_fun.__defaults__,
closure=origin_fun.__closure__)
# apply decorator
fun_decorator = shellMethod(**new_dict)
fun_decorator(new_fun)
# updat the docstring
new_fun.__doc__ = new_fun.__doc__.replace("environment", name)
return new_fun
def __init__(self,func):
for n in list(n for n in set(dir(func)) - set(dir(self)) if n != '__class__'):
setattr(self, n, getattr(func, n))
self._m=Manager()
self._e= self._m.Event()
self._d=self._m.dict()
self._f=dumps(func.__code__)
self._n=func.__name__
self._q=Queue()
self.func=FunctionType(loads(self._f),globals(),"a_func")
globals()[self._n]=partial(_getValue,self._d,self._q,self._e,True,self.func)
globals()[self._n].apply_async=partial(_getValue,self._d,self._q,self._e,False,self.func)
self._t=Process(target=_taskManager,args=(self._q,self._d,self._f,self._n, self._e))
self._t.start()
def __init__(self,func):
for n in list(n for n in set(dir(func)) - set(dir(self)) if n != '__class__'):
setattr(self, n, getattr(func, n))
setattr(self, "__doc__", getattr(func, "__doc__"))
self._m=Manager()
self._e= self._m.Event()
self._d=self._m.dict()
self._f=dumps(func.__code__)
self._n=func.__name__
self._q=Queue()
self.func=FunctionType(loads(self._f),globals(),"a_func")
globals()[self._n]=partial(_getValue,self._d,self._q,self._e,True,self.func)
globals()[self._n].apply_async=partial(_getValue,self._d,self._q,self._e,False,self.func)
globals()[self._n].batch_async=partial(_batchAsync,self._d,self._q,self.func)
#setattr(globals()[self._n],"__contains__",self.__contains__)
self._t=Process(target=_taskManager,args=(self._q,self._d,self._f,self._n, self._e))
self._t.start()
atexit.register(_closeProcessGracefully, self) #TODO: Make this line not necessary.
class Primer():
'''An asynchronous cache implementation. Maintains multiple recursive calls stably.'''
def __init__(self,func):
for n in list(n for n in set(dir(func)) - set(dir(self)) if n != '__class__'):
setattr(self, n, getattr(func, n))
self._m=Manager()
self._e= self._m.Event()
self._d=self._m.dict()
self._f=dumps(func.__code__)
self._n=func.__name__
self._q=Queue()
self.func=FunctionType(loads(self._f),globals(),"a_func")
globals()[self._n]=partial(_getValue,self._d,self._q,self._e,True,self.func)
globals()[self._n].apply_async=partial(_getValue,self._d,self._q,self._e,False,self.func)
self._t=Process(target=_taskManager,args=(self._q,self._d,self._f,self._n, self._e))
self._t.start()
def apply_async(self,*item):
return _getValue(self._d,self._q,self._e,False,self.func,*item)
def __call__(self,*item):
return _getValue(self._d,self._q,self._e,True,self.func,*item)
def __del__(self):
self._t.terminate()
def __repr__(self):
return 'concurrent.Cache('+self.func.__repr__()+')'
def loads_function(s):
'''Restores a function serialized with :func:`dumps_function`.'''
name, code, globals_, defaults, closure, func_dict, doc, qualname, kwdefaults, annotations = loads(s)
code = marshal.loads(code)
for k, v in globals_.items():
if isinstance(v, Module):
globals_[k] = v.mod
if closure is not None:
import ctypes
ctypes.pythonapi.PyCell_New.restype = ctypes.py_object
ctypes.pythonapi.PyCell_New.argtypes = [ctypes.py_object]
closure = tuple(ctypes.pythonapi.PyCell_New(c) for c in closure)
globals_['__builtins__'] = __builtins__
r = FunctionType(code, globals_, name, defaults, closure)
r.__dict__ = func_dict
r.__doc__ = doc
r.__qualname__ = qualname
r.__kwdefaults__ = kwdefaults
r.__annotations__ = annotations
return r
class Cache():
'''
An asynchronous cache implementation. Maintains multiple recursive calls stably.
The resultant object operates just like a function, but runs the code outside
the main process. When calls are started with :meth:`~Cache.apply_async`, a new process
is created to evaluate the call.
A simple cache can reduce recursive functions such as the naive Fibonacci function
to linear time in the input space, whereas a parallel cache can reduce certain
problems even farther, depending on the layout of the call and the number of processors
available on a computer. The code below demonstrates using :class:`Cache` as a simple
cache::
>>> @Cache
... def fibonacci(n):
... if n < 2: # Not bothering with input value checking here.
... return 1
... return fibonacci(n-1)+fibonacci(n-2)
...
>>> fibonacci(5)
8
Using cache to take advantage of the ability to handle recursion branching, that
same code would become::
>>> @Cache
... def fibonacci(n):
... if n < 2: # Not bothering with input value checking here.
... return 1
... fibonacci.apply_async(n-1)
... fibonacci.apply_async(n-2)
... return fibonacci(n-1)+fibonacci(n-2)
...
>>> fibonacci(100)
573147844013817084101L
.. note:: Be careful when picking how to call your functions if you are looking
for speed. Given that the fibonacci sequence is roughly linear in
dependencies with caching, there isn't a significant speedup. When in
doubt, :mod:`cProfile` (or :mod:`profile`) are your friends.
.. todo:: Eventually provide automatic profiling to help with this part.
A good use for this would be in less sequential computation spaces, such as in
factoring. When a pair of factors are found, each can be factored asynchronously
to find all the prime factors recursively. When a factor in a factor pair is found
that are known to be prime, or otherwise has its factors known, then only
one needs to be factored further. At this point, blindly branching and factoring
will have one side yield the cached value, and the other creating a new process.
Given the Fibonacci example above, this will happen on every call that isn't the
first call, yielding to `n` processes being spawned and using system resources.
Simply caching the naive Fibonacci function is just about the fastest way to use it.
To avoid unnecessary branching automatically, you can use the batch_async method
similarly to the apply_async method, except each set of arguments, even if they're
singular, must be wrapped in a tuple. Applying this to the Fibonacci function yields.
>>> @Cache
... def fibonacci(n):
... if n < 2: # Not bothering with input value checking here.
... return 1
... fibonacci.batch_async((n-1,),(n-2,))
... return fibonacci(n-1)+fibonacci(n-2)
...
>>> fibonacci(200)
453973694165307953197296969697410619233826L
This makes the branching optimal whenever possible. Race conditions might cause
issues, but those caused by python's built in Manager cannot be mitigated easily.
For the fibonnacci sequence, this will likely just revert the computation to a
mostly synchronous and sequential calculation, which is optimal for this version
of calculating the Fibonacci sequence.
.. note:: There are `much better algorithms
<http://en.wikipedia.org/wiki/Fibonacci_sequence#Matrix_form>`_ for
calculating Fibonacci sequence elements; some of which are better suited
for this type of caching.
'''
# Additionally, one can test whether a value has been calculated before by using
# ``(*{item}) in {cache}``. Calls of this type will be faster if iterable objects
# are passed to the ``in`` operator. This allows one to avoid unnecessary branching
# and process creation. Using this, the same example becomes::
#
# >>> @Cache
# ... def fibonacci(n):
# ... if n < 2: # Not bothering with input value checking here.
# ... return 1
# ... if n-1 not in fibonacci or n-2 not in fibonacci:
# ... fibonacci.apply_async(n-1)
# ... fibonacci.apply_async(n-2)
# ... return fibonacci(n-1)+fibonacci(n-2)
# ...
# >>> fibonacci(5)
# 8
def __init__(self,func):
for n in list(n for n in set(dir(func)) - set(dir(self)) if n != '__class__'):
setattr(self, n, getattr(func, n))
setattr(self, "__doc__", getattr(func, "__doc__"))
self._m=Manager()
self._e= self._m.Event()
self._d=self._m.dict()
self._f=dumps(func.__code__)
self._n=func.__name__
self._q=Queue()
self.func=FunctionType(loads(self._f),globals(),"a_func")
globals()[self._n]=partial(_getValue,self._d,self._q,self._e,True,self.func)
globals()[self._n].apply_async=partial(_getValue,self._d,self._q,self._e,False,self.func)
globals()[self._n].batch_async=partial(_batchAsync,self._d,self._q,self.func)
#setattr(globals()[self._n],"__contains__",self.__contains__)
self._t=Process(target=_taskManager,args=(self._q,self._d,self._f,self._n, self._e))
self._t.start()
atexit.register(_closeProcessGracefully, self) #TODO: Make this line not necessary.
def apply_async(self,*item):
"""
Calling this method starts up a new process of the function call in question.
This does not retrieve an answer.
"""
return _getValue(self._d,self._q,self._e,False,self.func,*item)
def batch_async(self,*items):
"""
This method examines the arguments passed in for how to branch optimally
then does so. This does not retrieve the answers, just like apply_async does not.
The arguments must each be a complete set of the arguments passed into the function
but in tuple form. If the cached function only takes one argument, wrap it with
parenthesis and add a comma before the closing parenthesis.
"""
_batchAsync(self._d,self._q,self.func,*items)
def __call__(self,*item):
return _getValue(self._d,self._q,self._e,True,self.func,*item)
def __del__(self):
_closeProcessGracefully(self)
def __repr__(self):
return 'concurrent.Cache('+self.func.__repr__()+')'
def patched_function(function):
new_function = FunctionType(six.get_function_code(function), globals())
if six.PY3:
new_function.__kwdefaults__ = function.__kwdefaults__
new_function.__defaults__ = function.__defaults__
return new_function
def decorator(f: types.FunctionType) -> types.FunctionType:
assert name in f.__annotations__
f.__annotations__[name] = type
return f
def rebindFunction(f, rebindDir=None,
funcName=None, funcDoc=None,
argRebindDir=None, propRebindDir=None,
**rebinds):
'''return a function derived from *f* with rebinds specified by *rebindDir* and/or *rebinds*.
Use *funcName* as function name, instead of `f.func_name` as function
name, if given.
*argRebindDir* is a dictionary mapping function parameter names
to defaults. You can use this to turn required parameters into optional
ones (by providing a default value for them), to change their
default values or to turn optional parameters into required ones.
Note that Python requires that required arguments must preceed
optional ones. A `ValueError` is raised when *argRebindDir* violates
this restriction.
Note: we only support simply named parameters (not constructor expressions).
*propRebindDir* is a dictionary specifying rebinds for the
functions properties.
ATT: *f.func_globals* is copied at rebind time. Later modifications
may affect *f* but not the rebind function.
Note: we would like to rebind closure parts as well but Python currently
does not allow to create `cell` instances. Thus, this would be difficult.
'''
# unwrap a method
f = getattr(f, 'im_func', f)
# handle global variable rebinds
fg = f.func_globals.copy()
if rebindDir: fg.update(rebindDir)
if rebinds: fg.update(rebinds)
# handle argument (default) rebinds
if argRebindDir:
args, _, _, defaults = getargspec(f)
# ensure all arguments are known
unknown = []
for a in argRebindDir:
if a not in args: unknown.append(a)
if unknown:
raise ValueError('unknown arguments in `argRebindDir`: %s'
% ', '.join(unknown)
)
# determine new defaults
defaults = defaults is not None and list(defaults) or []
defaults = [REQUIRED] * (len(args) - len(defaults)) + defaults
funcDefaults = []
for (a,d) in zip(args, defaults):
if isinstance(a, str) and a in argRebindDir: d = argRebindDir[a]
if d is not REQUIRED: funcDefaults.append(d)
elif funcDefaults: raise ValueError('required argument after optional one: %s' % a)
funcDefaults = tuple(funcDefaults)
else: funcDefaults = f.func_defaults or ()
# construct the new function
nf = FunctionType(
f.func_code,
fg, # func_globals
funcName or f.func_name,
funcDefaults,
f.func_closure,
)
# handle the documentation
if funcDoc is not None: nf.func_doc = funcDoc
else: nf.func_doc = f.func_doc
# handle is properties
if f.__dict__ is not None: nf.__dict__ = f.__dict__.copy()
if propRebindDir:
if nf.__dict__ is None: nf.__dict__ = {}
nf.__dict__.update(propRebindDir)
return nf
