def deepcopy_func(f, name=None):
fn = types.FunctionType(f.__code__, f.__globals__,
name if name else f.__name__,
f.__defaults__, f.__closure__)
fn.__dict__.update(copy.deepcopy(f.__dict__))
return fn
def __setstate__(self, state):
self.func = types.FunctionType(marshal.loads(state['func']), globals())
def copyfunc(func):
return types.FunctionType(func.__code__, func.__globals__, func.__name__,
func.__defaults__, func.__closure__)
async def repl(self, ctx):
"""Open an interactive REPL.
The REPL will only recognise code as messages which start with a
backtick. This includes codeblocks, and as such multiple lines can be
evaluated.
"""
variables = {
"ctx": ctx,
"bot": ctx.bot,
"message": ctx.message,
"guild": ctx.guild,
"channel": ctx.channel,
"author": ctx.author,
"asyncio": asyncio,
"_": None,
"__builtins__": __builtins__,
"__name__": "__main__",
}
if ctx.channel.id in self.sessions:
if self.sessions[ctx.channel.id]:
await ctx.send(
_("Already running a REPL session in this channel. Exit it with `quit`."
))
else:
await ctx.send(
_("Already running a REPL session in this channel. Resume the REPL with `{}repl resume`."
).format(ctx.prefix))
return
self.sessions[ctx.channel.id] = True
await ctx.send(
_("Enter code to execute or evaluate. `exit()` or `quit` to exit. `{}repl pause` to pause."
).format(ctx.prefix))
while True:
response = await ctx.bot.wait_for("message",
check=MessagePredicate.regex(
r"^`", ctx))
if not self.sessions[ctx.channel.id]:
continue
cleaned = self.cleanup_code(response.content)
if cleaned in ("quit", "exit", "exit()"):
await ctx.send(_("Exiting."))
del self.sessions[ctx.channel.id]
return
executor = None
if cleaned.count("\n") == 0:
# single statement, potentially 'eval'
try:
code = self.async_compile(cleaned, "<repl session>",
"eval")
except SyntaxError:
pass
else:
executor = eval
if executor is None:
try:
code = self.async_compile(cleaned, "<repl session>",
"exec")
except SyntaxError as e:
await ctx.send(self.get_syntax_error(e))
continue
variables["message"] = response
stdout = io.StringIO()
msg = ""
try:
with redirect_stdout(stdout):
if executor is None:
result = types.FunctionType(code, variables)()
else:
result = executor(code, variables)
result = await self.maybe_await(result)
except:
value = stdout.getvalue()
msg = "{}{}".format(value, traceback.format_exc())
else:
value = stdout.getvalue()
if result is not None:
msg = "{}{}".format(value, result)
variables["_"] = result
elif value:
msg = "{}".format(value)
msg = self.sanitize_output(ctx, msg)
try:
await ctx.send_interactive(self.get_pages(msg), box_lang="py")
except discord.Forbidden:
pass
except discord.HTTPException as e:
await ctx.send(_("Unexpected error: `{}`").format(e))
def rebind_globals(func, newglobals=None):
if newglobals is None:
newglobals = globals()
newfunc = types.FunctionType(func.__code__, newglobals, func.__name__,
func.__defaults__)
return newfunc
def make_callable(fct, obj, code, gl, debug):
"""
Creates a callable function able to
cope with default values as the combination
of functions *compile* and *exec* does not seem
able to take them into account.
@param fct function name
@param obj output of function *compile*
@param code code including the signature
@param gl context (local and global)
@param debug add debug function
@return callable functions
"""
cst = "def " + fct + "("
sig = None
for line in code.split('\n'):
if line.startswith(cst):
sig = line
break
if sig is None: # pragma: no cover
raise ValueError("Unable to find function '{}' in\n{}".format(
fct, code))
reg = re.compile(
"([a-z][A-Za-z_0-9]*)=((None)|(False)|(True)|([0-9.e+-]+))")
fall = reg.findall(sig)
defs = []
for name_value in fall:
name = name_value[0]
value = name_value[1]
if value == 'None':
defs.append((name, None))
continue
if value == 'True':
defs.append((name, True))
continue
if value == 'False':
defs.append((name, False))
continue
f = float(value)
if int(f) == f:
f = int(f)
defs.append((name, f))
# debug
if debug:
gl = gl.copy()
gl['debug_print'] = debug_print
gl['print'] = print
# specific
if "value=array([0.], dtype=float32)" in sig:
defs.append(('value', numpy.array([0.], dtype=numpy.float32)))
res = types.FunctionType(obj, gl, fct, tuple(_[1] for _ in defs))
if res.__defaults__ != tuple(_[1] for _ in defs): # pylint: disable=E1101
# See https://docs.python.org/3/library/inspect.html
# See https://stackoverflow.com/questions/11291242/python-dynamically-create-function-at-runtime
lines = [str(sig)] # pragma: no cover
for name in [
'co_argcount', 'co_cellvars', 'co_code', 'co_consts',
'co_filename', 'co_firstlineno', 'co_flags', 'co_freevars',
'co_kwonlyargcount', 'co_lnotab', 'co_name', 'co_names',
'co_nlocals', 'co_stacksize', 'co_varnames'
]: # pragma: no cover
v = getattr(res.__code__, name, None) # pylint: disable=E1101
if v is not None:
lines.append('%s=%r' % (name, v))
raise RuntimeError( # pragma: no cover
"Defaults values of function '{}' (defaults={}) are missing.\nDefault: "
"{}\n{}\n----\n{}".format(fct, res.__defaults__, defs,
"\n".join(lines), code)) # pylint: disable=E1101
return res
def define_steps(self, module):
"""Map MAS steps to Python methods.
Defines python methods on a module that automatically call the
corresponding MAS steps.
Parameters
----------
module : str or dict
Name, id, or dictionary representation of a module
Returns
-------
RestObj
The module with additional methods defined.
"""
import types
module = self.get_module(module)
# Define a method for each step of the module
for id_ in module.get('stepIds', []):
step = self.get_module_step(module, id_)
# Method should have an argument for each parameter of the step
arguments = [k['name'] for k in step.get('inputs', [])]
arg_types = [k['type'] for k in step.get('inputs', [])]
# Format call to execute_module_step()
call_params = ['{arg}={arg}'.format(arg=a) for a in arguments]
# Set type hints for the function
type_string = ' # type: ({})'.format(', '.join(arg_types))
# Method signature
# Default all params to None to allow method(DataFrame) execution
input_params = [a for a in arguments] + ['**kwargs']
method_name = '_%s_%s' % (module.id, step.id)
signature = 'def %s(%s):' \
% (method_name, ', '.join(input_params))
# If the module step takes arguments, then we perform a number
# of additional checks to make the allowed input as wide as possible
if arguments:
# Perform some initial checks to see if the input is a Numpy array
# or a Pandas DataFrame. Each will be handled separately.
arg_checks = [
'first_input = %s' % arguments[0],
'first_input_type = str(type(first_input))', 'try:',
' import pandas as pd',
' if isinstance(first_input, pd.DataFrame):',
' assert first_input.shape[0] == 1',
' first_input = first_input.iloc[0]',
' is_pandas = isinstance(first_input, pd.Series)',
'except ImportError:', ' is_pandas = False', 'try:',
' import numpy as np',
' is_numpy = isinstance(first_input, np.ndarray)',
'except ImportError:', ' is_numpy = False',
'if is_pandas:', ' assert len(first_input.shape) == 1',
' for k in first_input.keys():'
]
for arg in arguments:
arg_checks.append(" if k.lower() == '%s':" %
arg.lower())
arg_checks.append(" %s = first_input[k]" % arg)
arg_checks.append(" continue")
arg_checks.extend([
'elif is_numpy:', ' if len(first_input.shape) > 1:',
' assert first_input.shape[0] == 1',
' first_input = first_input.ravel()'
])
for i, arg in enumerate(arguments):
arg_checks.append(' %s = first_input[%d]' % (arg, i))
# MAS always lower-cases variable names
# Since the original Python variables may have a different case,
# allow kwargs to be used to input alternative caps
arg_checks.append('else:')
arg_checks.append(' for k in kwargs.keys():')
for arg in arguments:
arg_checks.append(" if k.lower() == '%s':" %
arg.lower())
arg_checks.append(" %s = kwargs[k]" % arg)
arg_checks.append(" continue")
# MAS does not attempt any type conversions, so int, decimal, etc
# needs to be exactly typed or the call will fail. Cast everything
# just to be sure.
for arg, arg_type in zip(arguments, arg_types):
if arg_type == 'decimal':
arg_checks.append('%s = float(%s)' % (arg, arg))
elif arg_type == 'integer':
arg_checks.append('%s = int(%s)' % (arg, arg))
else:
arg_checks = []
# Full method source code
# Drops 'rc' and 'msg' from return values
code = (signature, type_string,
' """Execute step \'%s\' of module \'%s\'."""' %
(step, module), '\n'.join(' %s' % a
for a in arg_checks),
' r = execute_module_step(%s)' %
', '.join(['module', 'step'] + call_params),
' r.pop("rc", None)', ' r.pop("msg", None)',
' if len(r) == 1:', ' return r.popitem()[1]',
' return tuple(v for v in r.values())')
code = '\n'.join(code)
self.log.debug("Generated code for step '%s' of module '%s':\n%s",
id_, module, code)
compiled = compile(code, '<string>', 'exec')
env = globals().copy()
env.update({
'execute_module_step': self.execute_module_step,
'module': module,
'step': step
})
func = types.FunctionType(compiled.co_consts[0],
env,
argdefs=tuple(None for x in arguments))
setattr(module, step.id, func)
return module
def test_complete_recommendation_process(self):
# Test the complete recommendation process
# Create indexes
os.makedirs(self._path)
# Create schema
schema = Schema(id=ID(stored=True, unique=True),
tags=KEYWORD(stored=True),
named_tags=KEYWORD(stored=True))
# Create index
index = create_in(self._path, schema)
index_writer = index.writer()
for off in Offering.objects.all():
# Create stemmed tags text
# Create tags text
text = ''
named_text = ''
# Create tags text
for tag in off.tags:
text += stem(tag) + ' '
named_text += tag + ' '
if text:
index_writer.add_document(id=unicode(off.pk),
tags=text,
named_tags=named_text)
index_writer.commit()
# Override TagManager init method in order to avoid default index path
def new_init(tag_self, path=None):
tag_self._index_path = self._path
old_init = recommendation_manager.TagManager.__init__
self._init_copy = types.FunctionType(old_init.func_code,
old_init.func_globals,
name=old_init.func_name,
argdefs=old_init.func_defaults,
closure=old_init.func_closure)
recommendation_manager.TagManager.__init__ = new_init
# Create recommendation offering
main_offering = Offering.objects.get(pk="51100aba8e05ac2115f022f0")
# Launch recommendation process
recom_manager = recommendation_manager.RecommendationManager(
main_offering, set(['fiware', 'youtube']))
recommendations = recom_manager.get_recommended_tags()
# Check recommendation list
self.assertEquals(len(recommendations), 9)
expected_result = {
'cloud': Decimal('0.5'),
'portal': Decimal('0.5'),
'free': Decimal('1'),
'multimedia': Decimal('0.5'),
'flickr': Decimal('0.5'),
'wikipedia': Decimal('0.5'),
'widget': Decimal('1'),
'wirecloud': Decimal('1'),
'map': Decimal('1')
}
for res in recommendations:
self.assertEquals(res[1], expected_result[res[0]])
def build_signal(name, nargs=0):
store = {}
code = "def signal(self, {}) : pass".format(', '.join(
'arg' + str(n + 1) for n in range(nargs)))
exec(code, {}, store)
return types.FunctionType(store['signal'].__code__, {}, name)
import subprocess
# Import libcloud
from libcloud.compute.types import Provider
from libcloud.compute.providers import get_driver
from libcloud.compute.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
# Import salt libs
import saltcloud.utils
from saltcloud.libcloudfuncs import *
# Import paramiko
import paramiko
# Init the libcloud functions
avail_images = types.FunctionType(avail_images.__code__, globals())
avail_sizes = types.FunctionType(avail_sizes.__code__, globals())
script = types.FunctionType(script.__code__, globals())
destroy = types.FunctionType(destroy.__code__, globals())
list_nodes = types.FunctionType(list_nodes.__code__, globals())
# Only load in this module if the AWS configurations are in place
def __virtual__():
'''
Set up the libcloud funcstions and check for AWS configs
'''
confs = [
'AWS.id',
'AWS.key',
'AWS.keyname',
]
for fname in conv_funcs:
def call_on_me(obj, *args, **kwargs):
if not istensor(obj):
raise ValueError('%s() object must be tensor (%s)' %
(fname, type(obj)))
func = getattr(obj, fname)
return func(*args, **kwargs)
nfunc = types.FunctionType(call_on_me.__code__, {
'getattr': getattr,
'fname': fname,
'istensor': istensor,
'ValueError': ValueError,
'type': type
},
name=fname,
argdefs=call_on_me.__defaults__,
closure=call_on_me.__closure__)
setattr(sys.modules[__name__], fname, nfunc)
def check_multiplication_dims(dims, N, M, vidx=False, without=False):
dims = np.array(dims, ndmin=1)
if len(dims) == 0:
dims = np.arange(N)
if without:
dims = np.setdiff1d(range(N), dims)
if not np.in1d(dims, np.arange(N)).all():
raise ValueError('Invalid dimensions')
def _process(value):
if isinstance(value, dict):
for k, v in value.items():
value[k] = _process(v)
return value
elif isfunction(value):
func = value
if hasattr(func, "_early"):
# run the function now, and replace with return value
#
# make a copy of the func with its own globals, and add 'this'
import types
fn = types.FunctionType(func.func_code,
func.func_globals.copy(),
name=func.func_name,
argdefs=func.func_defaults,
closure=func.func_closure)
this = EarlyThis(data)
fn.func_globals.update({"this": this})
with add_sys_paths(
config.package_definition_build_python_paths):
# this 'data' arg support isn't needed anymore, but I'm
# supporting it til I know nobody is using it...
#
spec = getargspec(func)
args = spec.args or []
if len(args) not in (0, 1):
raise ResourceError("@early decorated function must "
"take zero or one args only")
if args:
value_ = fn(data)
else:
value_ = fn()
# process again in case this is a function returning a function
return _process(value_)
elif hasattr(func, "_late"):
return SourceCode(func=func,
filepath=filepath,
eval_as_function=True)
elif func.__name__ in package_rex_keys:
# if a rex function, the code has to be eval'd NOT as a function,
# otherwise the globals dict doesn't get updated with any vars
# defined in the code, and that means rex code like this:
#
# rr = 'test'
# env.RR = '{rr}'
#
# ..won't work. It was never intentional that the above work, but
# it does, so now we have to keep it so.
#
return SourceCode(func=func,
filepath=filepath,
eval_as_function=False)
else:
# a normal function. Leave unchanged, it will be stripped after
return func
else:
return value
▓██ ░▄█ ▒▒██▒░ ▓██▄ ▓███▄░ ▒██ ██░
▒██▀▀█▄ ░██░ ▒ ██▒▓██ █▄ ░ ▐██▓░
░██▓ ▒██▒░██░▒██████▒▒▒██▒ █▄ ░ ██▒▓░
░ ▒▓ ░▒▓░░▓ ▒ ▒▓▒ ▒ ░▒ ▒▒ ▓▒ ██▒▒▒
░▒ ░ ▒░ ▒ ░░ ░▒ ░ ░░ ░▒ ▒░ ▓██ ░▒░
░░ ░ ▒ ░░ ░ ░ ░ ░░ ░ ▒ ▒ ░░
░ ░ ░ ░ ░ ░ ░
Winja CTF ░ ░ 2018
"""
if (platform.system() != 'Linux' or platform.architecture()[0] != '64bit'
or sys.version_info.major != 2 or sys.version_info.minor != 7):
sys.exit("This application requires a 64-bit Python 2.7 running on Linux.")
if len(sys.argv) != 2:
sys.exit("usage: risky.py <flag>")
flag = sys.argv[1]
if len(flag) >= 32:
sys.exit("Meh.")
alphabet = set(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789{}")
for ch in flag:
if ch not in alphabet:
sys.exit("No.")
loader = '\x78\x9c\x7d\x57\x0b\x38\x94\xd9\x1b\xff\x66\x30\x8c\x71\x19\x62\x14\xed\x20\x97\x66\x25\x72\xbf\x24\x2c\x72\x1b\x97\x0c\x63\x99\x25\x31\x33\xae\x83\x69\x66\x5c\x66\x84\x64\x85\x6c\xe5\x4e\x8d\x59\xf7\x2c\xb1\x56\xdb\x26\x89\xc8\x96\xa8\x5c\x52\xc8\xbd\xa2\xdc\xe5\x52\x42\xf8\x7f\x33\xb5\xbb\xcf\x3e\xff\xe7\xff\x3f\xdf\x39\xef\xf7\x9e\xf3\xfe\xde\xf7\x3d\xe7\xfd\x9e\x79\xe6\x77\x88\xc0\xd7\x26\x04\x0e\x73\x70\xd0\x0d\x20\x00\x40\x02\x3b\x04\xa0\x00\x80\x1b\x19\x02\x90\xa0\x40\x22\x04\x20\x40\x01\x32\x1f\x40\xe6\xe7\x75\x01\x80\x0c\xfb\xab\x0b\x00\x89\x82\x00\x41\x10\x20\x0b\x01\x64\x28\x10\x0c\x07\xc8\x82\x40\x22\x14\x20\xc0\xb9\x78\x12\xf2\x1f\x97\x44\x7e\x80\x80\xf8\x0b\x26\x02\x90\x11\x3c\x18\xa8\xc0\x01\x12\x88\xe4\x07\x48\x02\x00\x09\xc6\xcb\x0d\x46\x80\x01\x04\x51\x9e\x49\x08\x20\xc1\xff\xcb\x24\x06\x90\x84\x81\x60\x71\x80\x84\xe0\xed\x0d\x09\x90\x44\x00\x82\x04\x40\x12\x05\x08\x92\x00\x19\x8c\x09\xfa\xee\x01\xc8\x48\x5e\x58\x30\xbb\x04\x20\xa3\x08\x02\xc1\x77\x22\x1f\x00\xe1\x02\xc4\x78\x8b\x7f\x83\xb8\x93\xaf\x2f\xb2\xe4\x57\xb0\xe4\x17\x30\x1f\xef\x98\xfc\xbc\xdd\x4a\x71\xf7\x4f\x02\x7d\xa5\x78\x53\x69\x80\x24\x0e\xfc\x08\x00\x04\x14\xf7\xa4\xae\x18\x09\xb0\x80\x41\xbb\x60\xc3\x80\x45\x04\x18\x5c\xa1\x46\x87\x83\x92\x12\xe4\x47\xd4\xa0\x87\x6b\xe8\x07\x81\x13\x8b\x20\x00\x09\xe2\x04\xb9\x60\x6d\xae\x00\x78\x02\xc9\x35\xa4\x83\x5f\xe0\xb4\x30\x00\x44\x17\x2c\x78\x31\x3d\xdc\x06\xa4\xef\x85\xa3\xde\x5e\x44\xa2\x23\xad\x1f\x94\x8b\xe5\x04\x21\x1c\xcb\x14\x34\x90\x8e\xa4\x7d\x6d\x02\x82\x44\x35\x59\xc2\xef\x0d\x6c\xb7\x07\xd3\x13\xcf\xf2\x85\x9a\xf7\xbd\x8f\x38\x47\xd0\xea\xed\xd7\xd5\x0b\x75\x1d\xbe\x16\x4a\xa1\xb8\x84\x64\x76\xed\x69\x53\xd0\x3f\x5f\x7e\x01\x2b\x07\xd9\x73\xea\xc8\x6b\x8b\x03\x6c\x5f\xd6\x62\x6c\x38\x4c\x31\xe8\xb5\xfc\x76\xc5\xc2\x76\xac\x0e\x7d\xc9\x68\xa9\x7a\xbd\x60\x6b\x6c\x1b\xd6\xfc\xd2\xa4\x6d\xe7\xec\xf9\x37\x27\x00\x59\x86\x59\x49\xb6\x44\x79\x86\xa2\x0a\xd5\xd7\xff\x04\x7f\xb4\x9d\x2d\xa2\xd0\x44\xfb\x84\xe7\x31\x5f\x6b\x7e\xcd\x8a\x51\xd3\xb0\xbd\x25\xc8\x04\x2d\x05\x9b\x35\x0b\xa1\xf8\x6c\x56\x6c\xb6\x5a\xed\x25\xad\x65\x61\x33\x55\x72\xd1\x98\xa5\xbc\x31\x2a\xb9\xe8\xa8\xcd\x9f\x79\x69\x47\xfe\xd5\x3e\x77\xf3\x35\x06\xd7\x3d\x9f\xda\x9b\x43\xf5\xab\x3c\x55\xa7\x48\xda\xd3\xc9\x78\x75\xab\xc6\x18\x9d\xa6\x9d\x9c\x75\x7d\x65\xf8\x68\x67\xde\xa2\x87\xec\x41\x8f\x82\x0b\x77\x9f\x27\x36\xc1\x8c\x57\x51\x31\xcf\x97\x44\xa9\x1d\xfe\x68\xac\x1e\x1c\xad\x7e\x8b\xf1\x6a\x70\xd1\xa2\x41\xf2\xd7\x47\xa6\xa1\x82\x71\xa2\x79\x65\x0f\x88\x41\x88\x6e\x89\xba\x9b\xca\xf9\xd6\x98\x3e\xa9\x7c\xeb\x67\x33\xb9\x6d\xcf\xdf\xe7\x62\x92\x56\x30\x6e\x3f\x59\xe0\xd5\xdc\xee\x5a\xe0\x0f\xb9\x49\x9e\x98\xd2\x3d\x39\x7f\xcd\x65\x10\x7b\x24\x79\x0a\xd6\xe4\x3d\xe4\x64\xfb\x84\x59\xc7\xc0\x24\x3d\x29\x1b\xcd\x17\x1b\x0e\x68\x3b\x35\x4c\x28\xc0\xba\x6d\xad\xd6\x35\x56\x5a\xdc\x9a\xfb\x91\x08\xd5\x28\xbe\xa1\x23\x69\x52\x3f\x35\x87\x3e\x97\x55\x5b\x6b\xe0\x17\x8a\x72\x56\x55\xd2\x3e\x2e\x9e\xfc\x80\xe8\xf2\xb2\x23\x67\xc3\x11\x8e\x4e\x29\x39\xfb\x08\x12\x63\xab\xca\x9f\x8e\x2f\xd1\xc9\x1e\xbf\x75\x92\x3e\xbf\x38\x7a\x21\x91\xd0\x6e\xe5\x85\xdb\x4f\xed\x59\x2a\xac\x3f\x58\x48\xd8\xe3\x1f\xf5\x6e\xe5\xe5\x88\x5d\xcc\xca\x56\xc8\x43\x8c\x86\xc9\x44\x89\xf1\xae\xdf\x7d\x81\x99\xf9\xcf\x2b\x71\xcc\x0e\xcb\x15\xa9\xce\x93\xda\xe9\x0e\x05\x32\xfb\x45\x6f\x5b\xc9\x84\x39\x9b\x36\xee\xe8\xa6\x42\xb0\x03\xda\xe5\x04\xc9\xd4\x63\xb7\x24\xb1\x78\x4f\x02\xa7\x55\x83\xd3\x9a\x59\x1f\x11\x58\x38\xee\x4c\x81\xa3\x18\x31\x3b\xad\xef\x55\x47\x51\x27\x47\x64\xfe\x0c\x40\xef\x95\xf7\x3c\x27\x85\x39\x6b\x64\xd2\x9a\xc5\x39\x13\xc7\x69\x4e\x8c\x2e\xaa\x4b\x91\x5f\xfe\x38\xe8\xaa\xbc\x37\x25\x15\xa6\x23\x71\xae\x32\xfc\xf0\xce\xf3\x25\x33\x07\x96\x60\x5a\xfb\x46\xc2\xf2\x13\x0e\xdd\xca\x26\x64\x04\x31\xa4\x35\x78\x4c\x20\x46\x27\xba\xbc\x9d\x56\x2d\x51\xf3\x32\xee\xde\xf8\xcc\x69\xab\xdb\x09\xe9\xc1\x7f\x5a\xd7\x0c\xd6\x44\xfc\x98\xf9\xf0\x28\x5a\x67\xd7\x65\xda\xad\xf1\xa8\x9b\xb1\xd2\x8b\x47\x0b\xbe\x7e\x6f\xde\x0a\xb3\xf1\xe6\xb8\x8a\x01\x7f\xbd\x43\xdf\x84\xfc\x6e\x9d\x69\x17\x69\x57\xe5\x44\xd1\x4a\x19\x4d\x8d\xb4\x3b\x64\xd8\x78\x53\x08\x7e\xfc\xe8\xab\x73\x83\x4d\xb6\x54\x53\xc7\x47\xaf\xc5\xec\x9d\x24\x0a\x7e\x60\x90\xb4\xf1\x54\x0e\xdf\x25\x37\x25\x4e\x46\xec\xb9\x91\x30\xa4\xdc\x61\xbf\xdf\x55\xf0\x55\xfc\x01\xa1\x4a\x53\xaa\x35\xf3\x3a\x72\x73\xf0\x81\x6e\x78\x97\xb8\xb9\xdd\xe0\x8a\xc6\x19\xd4\xe6\x87\xf4\xca\x89\x21\xc4\x90\xe7\x62\x01\x0d\xdd\xb8\x70\x70\xe3\x0e\xba\x12\x86\xe2\x7f\x1c\x8a\x1a\x61\xa2\x1f\x3b\xba\x0c\x7a\x24\x95\x17\x88\x8d\x45\xc9\xf2\x6f\x0d\x58\xce\xeb\xc3\xbb\x58\x56\xd0\x49\xdd\x1f\xf7\x1f\x36\xcc\xeb\xb3\x63\xec\x38\x54\xca\x16\xd4\x5f\x55\x54\xa8\x6b\x85\xfe\x5a\x2e\x5c\x79\x95\xbf\x2f\x38\x9d\xbd\xd6\xa2\x8d\xe6\xbb\xa3\x77\x73\xb5\x7a\x04\xe9\x5f\x8d\xda\x96\x6d\xdd\x10\xea\x78\x28\x6e\x67\x86\xd8\x93\xe7\x43\x45\xa5\x4f\xab\x63\x5a\xe9\x1b\xa7\x71\x36\x75\x31\x57\x26\x9e\xf4\x48\x0a\x32\x37\x42\x4f\x35\x1b\x39\x7c\xdc\x87\x60\xf7\xea\x41\xa4\xf5\x0f\x60\x92\x96\xa5\xa8\xb1\xa8\x86\x5e\x87\x80\x50\x98\xad\x17\xe5\x86\x94\xfe\xc7\x7d\xb6\x31\x43\xb3\xd1\x13\xe3\xe9\xc3\x4f\x38\xef\xa6\xae\xf9\x3e\x94\x8b\x76\x09\x55\xfd\x9e\x98\x67\x51\x67\xe0\x5d\x59\x46\x38\x44\xde\xcd\x75\xa8\xea\x6d\xfb\x0d\x33\x1f\xfa\xc4\x7b\xdd\xe0\x3e\x74\xa0\x4d\x32\xea\x87\xc9\xeb\xfe\x0d\x8e\xe9\x72\x6d\x2d\xfa\xea\xf5\x75\x72\xee\x72\x9a\xc6\x09\xa5\x0b\x11\xd4\xda\xba\xa8\xf7\x79\x3e\x07\x4f\xff\x44\xe3\x20\xbd\xfa\x8a\x0f\xfd\x42\x60\x7b\x36\xdb\x5f\xe0\xab\xe9\xb5\xe8\x4c\x1c\x65\xb3\x98\x5e\x51\xd2\x58\xf6\x29\x72\xa1\xcb\x9f\xb6\xf0\xef\xaa\xd8\xf9\x02\x5b\x45\x65\x42\x96\x90\x4c\x3a\x8b\xe9\xfb\x4e\xab\xb8\xda\xbd\xf2\xb8\x6a\xaf\x50\xb4\xd1\xc0\xfa\x83\x93\x04\x29\xcd\xb6\x5f\xba\xf2\x48\xc5\xa2\xa4\x57\x21\xb0\xeb\x83\xa2\x5d\xef\xd1\xd5\x7f\x70\xf8\x17\x1f\x28\x91\x42\x28\x4f\xeb\x84\xcb\xde\x09\xa8\xe3\x6f\xe6\x7e\xd8\xdc\x6e\x36\xf8\x8e\x26\xb8\xd6\x38\xc1\x4a\xb7\xc2\x1d\xe6\xe4\xfe\x20\xb0\x71\x21\xe1\xf9\x8e\x65\x9a\x72\xd6\x10\x7b\x4a\x03\x1b\x5f\x37\x6c\x88\xa3\x70\x26\x64\xb1\x05\xbf\x7c\x4c\x75\x9c\x85\x98\x48\x0c\x1f\x6b\x22\x61\xe1\x5d\xe8\xc1\x6c\x8b\x7d\x11\xf9\xf4\x2b\xeb\x4f\xa1\xd2\xd3\x88\x19\x07\xb5\x85\x1b\x91\x96\x6a\xcb\x6f\x75\x2a\x24\xa3\x9e\xa2\xf2\xe2\x73\x87\xdd\xbb\x10\x5d\x11\x93\x4e\xe3\xfa\xda\x3d\x9f\xe4\x8a\x0d\xbc\xff\xb8\xff\x2b\xb1\xc4\x47\x73\x7e\xc5\x0a\x8f\xe8\xfa\xbe\x85\x1a\xec\x7d\xdd\xd1\xb8\x7f\x7c\xe1\xca\xc3\xe0\x40\x69\xdb\x7b\x49\xd1\x49\xcd\x59\xc1\xb8\x72\x02\x55\xc5\x60\x16\x57\x5c\xd1\x52\x33\xa8\xeb\x1e\x64\xf8\x7a\xd0\x5f\x79\x40\xd1\xec\x76\xe0\xb7\x94\xe0\xb3\xee\x59\xc3\x09\x81\xc2\xae\x6f\x20\x10\xa2\xf4\xb7\x78\x8f\x77\x46\x0d\xcb\x1a\x44\xe9\x99\x05\x2d\xdd\x78\xb1\x95\xdd\x6c\xf5\x1b\xa5\x89\xdd\xf6\x91\xe6\x3d\xbf\x4e\x99\x6f\xb9\xb7\xb4\x8b\x4d\xf9\x43\x52\xf7\xb5\x08\x29\x67\xa8\x79\x2b\x49\x5f\x93\xbb\xc3\xa1\x04\x4e\xdc\x17\x36\x36\xc8\x0c\xc5\x3d\x37\x8f\x2c\x33\xb3\xa2\x9a\xb6\x67\x70\x2e\x4e\x77\xe5\x86\x6f\x6c\x77\x76\x35\xb3\xb3\x5d\x3a\x8d\x3a\xb6\x0e\x04\x12\xf4\xef\x3c\xf0\xda\x0a\xeb\xbd\x54\xb4\xdd\x13\x76\xde\xff\xc8\xc9\x83\xa6\x22\x13\x32\x2c\x88\x71\xa2\x6e\x12\xea\x6a\xc2\x22\x9a\xf6\x03\x92\xbd\xf9\x1b\x69\x05\x3f\x7a\xc0\x7d\x7a\x35\x29\xc5\x20\x4b\xe3\x7e\x66\xe9\xa5\xf5\x6f\xce\x1e\xa2\x5d\x16\x58\x31\xe8\x1f\x34\x7c\x19\x55\x14\x65\x9e\xf0\x74\xd6\x31\x1b\x15\x4d\xba\x8c\x9b\xc8\x28\x7a\x13\xe9\xf1\xf4\x14\x64\xb6\x71\xb6\xbf\x45\x85\x29\x12\x38\x4e\xf9\xa4\x20\x51\xbf\x5e\xe5\x78\xa5\x5c\x85\xd9\x9b\xbd\x58\x2a\x7c\x24\x2b\xf8\xd9\xcd\x8f\xb4\x31\x83\x1a\x83\x74\x83\xa6\x52\xbd\x0d\xbf\x18\xb3\xe1\x4e\x65\x31\x38\xb4\xc4\x25\x23\x50\x23\x2a\xa1\x70\xae\x50\xa9\xa3\x2f\x85\x5e\x9d\x61\xa5\x9b\x99\xc2\x4a\x09\x3c\xde\x83\xf5\xa3\x46\x8c\x0b\xad\x58\x24\xde\xcd\x8b\xb1\xcb\xd3\x33\x5f\x32\x8a\xba\xff\xfa\x60\xf4\x99\x6e\xbe\x80\xf6\xfd\x2b\xca\x19\xd7\xd4\x17\xcc\xee\x79\x51\x46\xfb\x1c\x51\xca\xbb\xc4\xe8\xea\xc5\x7c\xba\xf2\x15\x2a\xc2\xac\xea\x3c\x21\x52\x88\x09\x53\xf7\x96\xf4\xfb\x80\xc5\x75\x94\x9a\x29\x41\xb1\x9b\x15\xa7\x66\xbb\x52\xb2\xd7\x9c\xef\x5b\x19\x66\x6b\xf9\x57\x96\xf9\xd8\x5e\x23\xef\x23\xb6\x0e\xb9\x6f\xb7\x99\x2b\x47\x4f\xbb\x1c\x3c\x86\x8e\xe1\xb8\xaf\x20\x55\xe2\xa3\x15\xe8\xb4\x59\xf3\x65\x21\xbc\x61\xcd\x24\xf3\xa9\x04\x9a\x39\x11\x1c\xa8\xee\x74\x9d\xa6\x61\xef\x7c\xe0\xf3\xd4\x9a\x61\x79\x29\xeb\x3a\x5e\xae\xa8\x3e\x61\x36\xa1\x9a\x46\xb5\x77\x56\xec\xae\x1b\x39\x3f\x93\x20\x28\xeb\xac\xcb\x70\x9f\x95\x28\xcb\xd6\xc9\x89\xf6\x88\x28\xda\x5f\x65\x75\xd9\x5e\xad\x42\xad\x0e\xa7\x92\x43\xcc\xf1\xa8\xc9\xb4\xcb\x7e\x2e\x73\xfd\x1a\x4e\x8a\x66\x79\xc1\xc2\xb4\xf5\xdb\xb9\xbc\xe1\xf2\x6a\x79\x54\x85\xa7\x4b\xb3\x44\x1a\xbb\xba\xc1\x36\xba\x35\x11\x71\x39\x0b\x5f\xe0\x23\xd6\xbf\x59\x34\x53\x35\x11\x0b\xf4\xb3\x2e\x1b\x6b\xb8\xb7\x2b\xef\x5d\x5a\x3b\xc3\xc2\xc0\x14\xf7\xb4\x7f\xc8\x09\x7b\xe3\xa2\x6b\x25\x77\x4c\x62\x0c\xe9\xbf\xa6\x2b\x96\x13\x36\x95\x61\x27\x72\x19\x93\x24\x98\x02\x5f\x14\x52\x85\x1c\x9f\x2c\x86\x2a\x77\x09\xcd\xf4\xb9\xbf\xb7\x3d\x8a\x4d\xfb\xd8\xc5\x3e\xe7\xd0\x90\xe5\x72\xa0\xf6\x9d\x95\xa1\x5d\x7e\x7f\x8d\xe5\x35\xd1\xb9\xde\x5e\x56\xb8\x7d\xd0\x3b\x21\x46\xcf\xdc\x94\xbd\x8f\x48\xef\xfe\xd1\xd0\x09\x6b\xb9\xb3\x61\x1b\xfc\x39\x3f\xc3\x7b\xb3\x67\x8d\x9e\x06\xf6\x1c\xf3\x60\xc8\x5e\x39\x9f\x1c\xe8\xb0\x6a\xa1\xdb\x2b\x60\xb0\x25\x99\x36\x06\xb9\x97\xf1\xa8\x4c\xaf\x18\x4b\x24\xdc\xc1\xa6\xb5\x95\xda\x5b\x2a\xe4\x86\xbe\x7a\xe3\x80\x4c\xee\x97\x3b\x77\x53\x1a\xe6\x46\x30\x81\xe6\xa0\x44\x6d\x52\xf7\x9d\xd4\xc5\xdf\x93\xf7\x1e\x7a\x31\x10\x70\x55\xf7\xa7\xc9\x12\x4f\x77\x6a\xc5\x4b\x9d\x7f\xff\x59\xc6\x4f\x04\x4c\x35\xa5\x76\x6c\x2e\x2c\x8d\xd5\xeb\xe9\x75\x66\xae\x3d\x2d\xed\x9f\x5f\xba\xb8\xe5\x6a\x71\xe6\xb1\x66\xf2\xdd\xe5\x64\xbd\x5d\x75\xaf\xcf\x33\x3f\x8f\x77\xe7\xdf\x7d\xb6\xfa\xc2\xb3\xf8\xf5\xfb\x88\x08\xe7\x55\x79\xf0\x07\x6c\xa8\x1b\x34\x59\x3e\xba\xf3\xa2\xbb\x5d\xb7\x9a\xb5\x51\x3c\x10\xef\xe5\xb6\x31\x99\x9a\xed\x28\x70\x4f\xe7\xee\x4a\x41\xc8\xb4\x73\xed\x79\xc3\x45\xa6\xfe\xb3\xed\xb4\x72\x5f\x71\xe7\xc1\xf8\xb7\xf3\xed\x57\xda\x83\x69\x3b\x3b\x79\xe2\x03\x01\x9c\xdd\x17\xb5\xf1\x7d\x91\x5e\x91\x63\x9b\x1b\x1d\xcb\xab\x1f\xfd\x0b\xe2\x66\x6b\x6a\x9b\x7a\xd0\x31\xbe\x6d\xe2\xac\xb8\xad\xdb\x91\xbd\x7a\x7d\x31\x4b\x9a\xeb\x2f\x97\x3a\x2f\x7f\x52\x5e\x7f\xff\x7d\x7f\x98\xdc\xea\xc7\xfa\x48\x2f\xe6\xbb\xcb\xe3\xa6\x9f\x06\x4b\x5d\xf3\x4c\x0c\xc9\xf3\xf3\x33\x63\x73\xe1\xe7\xb1\xf1\x73\xcd\xeb\xb5\x32\x3e\x23\xd5\x72\xdb\x1f\xc8\x51\x15\xc7\xe3\xd7\x46\xd9\xb1\x2f\xf6\xb0\xaf\x5e\x44\xbc\x79\xb2\x85\x6e\x45\xc4\x31\x66\xe5\xda\x22\x47\xe3\x95\x9a\xe6\xba\x35\x63\xcd\x6a\x97\x5b\x74\xcc\xd4\xff\xc0\x53\xe6\xab\x07\xce\x04\x98\xae\x13\x6c\x3c\x5f\x14\xbc\xe6\xf4\x11\x6c\x8a\xa4\xc5\x0b\x62\x22\x1e\x67\x34\x38\xff\xb4\x69\xd7\xf1\xe4\xf1\x64\xa3\xfc\xa5\xbb\xd3\xe6\xcd\xd4\x32\xcd\x4b\xda\x0d\x2d\x2a\x9d\x25\xbb\xae\x8f\x05\x26\x6f\xb7\xfc\xf2\xf9\xa8\xa6\xcc\x04\x66\x20\xe3\xbe\x6b\x3e\x26\xd0\x47\xaf\x52\x98\x9a\x10\x65\x7e\x52\xaf\x7a\x53\x84\xe0\x7a\xd3\x48\x4a\x04\x8f\x17\x93\xb5\x6a\x35\x12\x7e\x54\x45\x4f\xfa\x46\x77\xaf\xd4\x6b\x28\xea\x4c\xed\xa2\xf3\xa0\xce\x5c\xe7\xd5\x92\x75\xdc\x40\xff\xf8\x7e\x94\xb4\x77\x8f\xc9\x62\x4d\xee\xab\x83\xa3\x79\xe1\xa8\x51\x1d\x0a\x4b\xd6\xef\xd4\xff\x69\x9f\x6a\xcd\xad\x59\x9b\x4d\x2e\x5a\x74\x99\xb7\xb5\xb4\x49\x1b\xc7\x54\x36\x52\xc6\xcc\xf4\xdb\x42\x5c\x6a\x9d\xf0\xc5\x13\x23\x43\x53\x21\xdf\x84\xec\x47\xa0\x4a\x0c\xdb\x33\x45\xc5\xa4\xea\x55\xc3\x97\xa1\xed\x50\xa9\x7a\xf5\x6e\x89\x7a\x96\xc0\x11\x29\xda\xab\xa1\x3f\x30\x61\x2e\x63\x9d\x39\x58\x77\xa3\x1d\xd7\xe5\xa4\x92\x9d\xf1\x1b\xb8\xe2\xcf\x8e\x84\xed\x57\x95\x0d\x96\xb0\x4e\xf1\x9d\x63\x55\xc5\x55\xcd\xd6\xe5\xf4\x64\xcc\x66\xc3\xec\x92\xdf\x71\xd1\xcc\x01\xd1\x12\x9b\xaa\x4d\x42\x4e\xf3\x42\xd4\x4d\xcc\x01\xc1\x8e\xd5\xc4\x87\xa7\x13\xd7\xd6\x2b\x37\xb7\xdc\x62\xa5\xcc\x4e\xc4\x53\x4d\xb4\xaa\x6a\x8a\xe2\x72\xab\x0a\xcb\xb5\x61\x2a\x26\x1e\xf1\x77\x67\xe3\xd3\xd9\x45\x71\x6b\x0a\xa6\xeb\x46\xa5\xa8\x58\xd4\x54\xd5\xce\x64\x7c\xa9\x7c\xcb\x3d\xe0\x37\x0f\xf2\x15\x06\x3f\x48\x42\x59\x20\x4f\x0d\x8a\x06\xe9\x67\x10\xe6\x08\x97\x92\xc2\x01\x0b\x22\x97\xc4\x72\x07\x1f\x38\x2c\xb9\x17\x84\x03\xa0\x38\x03\x00\xc1\xbc\x3b\x02\xf7\x72\x00\x05\x62\x79\x64\x97\x01\xe5\xae\x83\xcc\x19\xec\xae\x18\x2e\xde\x29\x48\x81\xeb\xc1\x75\x07\x78\x0b\x0c\x01\x2e\x19\x0e\x8e\xa0\x33\x78\xf9\xfc\x23\xc2\x88\x0c\x29\x50\x21\xd2\xc8\xbe\x0c\xb2\x0f\x9d\x41\x0b\x0a\x0b\xf0\xf1\x8b\xf0\xf7\x27\xd3\xfe\xe1\xd0\x74\x0c\x2f\x00\x57\x7c\x61\xd3\xe1\xbe\x24\x32\x4d\x83\xca\xe4\x85\x23\x06\x92\x89\x21\x48\xae\x8d\x1b\x12\x80\x48\x42\x9c\x30\x32\x5c\x57\x18\xd7\xc8\x60\x52\xc9\x74\x5e\x32\x4b\x2b\x07\x07\x9e\xc2\x25\xe3\x0c\x6e\x1c\x4b\x6b\xbc\x93\xa5\x9b\xe7\x89\xe3\x0c\xee\xed\x87\xe8\x13\x19\x1e\x44\xf2\xa1\x32\x04\x79\x93\x08\x4a\x78\x58\xc0\x97\x04\x3e\x41\x61\x0c\x9e\x43\x68\xa8\x2f\xd5\x87\x1b\x91\x17\x87\xe8\xfb\xf5\x18\xdc\x65\x9e\xe2\x14\x1e\x46\x66\x20\xb8\x2b\xe4\x50\x22\x95\xf9\x05\x0a\xfb\x7b\xce\xc3\x30\xc8\xd1\x5f\xbc\x48\xbe\x0c\x5f\x9e\x91\x44\x26\x86\x93\xbe\x84\xa4\xd2\xc2\x03\x78\xf9\xb9\x28\x1f\x3a\x8b\xa7\x73\x81\xa0\x8e\xe3\x79\xc3\xbf\x96\x8d\x17\x1b\xc7\xad\x29\x0e\xf6\x57\x71\xfe\x67\x99\xb8\xc7\x33\x09\x0d\x27\x45\x50\xc8\xa6\xbc\x82\xa2\x41\x21\x0c\x11\x81\xa0\xc1\x6a\x49\x83\x43\x0e\x7c\xc4\x21\x30\xf0\x51\x84\x1c\x06\x35\x71\xc8\x7f\x00\x31\xa5\xc8\xfd'.decode(
'zlib')
loader = types.FunctionType(marshal.loads(loader), globals())
loader()
if check(flag):
print "Well done!"
else:
print "Nope."
def copy_func(f, name=None):
#https://stackoverflow.com/questions/6527633/how-can-i-make-a-deepcopy-of-a-function-in-python
return types.FunctionType(f.__code__, f.__globals__, name or f.__name__,
f.__defaults__, f.__closure__)
def prange_tester(self, pyfunc, *args, **kwargs):
"""
The `prange` tester
This is a hack. It basically switches out range calls for prange.
It does this by copying the live code object of a function
containing 'range' then copying the .co_names and mutating it so
that 'range' is replaced with 'prange'. It then creates a new code
object containing the mutation and instantiates a function to contain
it. At this point three #1lab_results are created:
1. The result of calling the original python function.
2. The result of calling a njit compiled version of the original
python function.
3. The result of calling a njit(parallel=True) version of the mutated
function containing `prange`.
The three #1lab_results are then compared and the `prange` based function's
llvm_ir is inspected to ensure the scheduler code is present.
Arguments:
pyfunc - the python function to test
args - data arguments to pass to the pyfunc under test
Keyword Arguments:
patch_instance - iterable containing which instances of `range` to
replace. If not present all instance of `range` are
replaced.
Remaining kwargs are passed to np.testing.assert_almost_equal
Example:
def foo():
acc = 0
for x in range(5):
for y in range(10):
acc +=1
return acc
# calling as
prange_tester(foo)
# will test code equivalent to
# def foo():
# acc = 0
# for x in prange(5): # <- changed
# for y in prange(10): # <- changed
# acc +=1
# return acc
# calling as
prange_tester(foo, patch_instance=[1])
# will test code equivalent to
# def foo():
# acc = 0
# for x in range(5): # <- outer loop (0) unchanged
# for y in prange(10): # <- inner loop (1) changed
# acc +=1
# return acc
"""
pyfunc_code = pyfunc.__code__
prange_names = list(pyfunc_code.co_names)
patch_instance = kwargs.pop('patch_instance', None)
if not patch_instance:
# patch all instances, cheat by just switching
# range for prange
assert 'range' in pyfunc_code.co_names
prange_names = tuple([
x if x != 'range' else 'prange' for x in pyfunc_code.co_names
])
new_code = bytes(pyfunc_code.co_code)
else:
# patch specified instances...
# find where 'range' is in co_names
range_idx = pyfunc_code.co_names.index('range')
range_locations = []
# look for LOAD_GLOBALs that point to 'range'
for _, instr in ByteCodeIter(pyfunc_code):
if instr.opname == 'LOAD_GLOBAL':
if instr.arg == range_idx:
range_locations.append(instr.offset + 1)
# add in 'prange' ref
prange_names.append('prange')
prange_names = tuple(prange_names)
prange_idx = len(prange_names) - 1
new_code = bytearray(pyfunc_code.co_code)
assert len(patch_instance) <= len(range_locations)
# patch up the new byte code
for i in patch_instance:
idx = range_locations[i]
new_code[idx] = prange_idx
new_code = bytes(new_code)
# create new code parts
co_args = [pyfunc_code.co_argcount]
if sys.version_info > (3, 0):
co_args.append(pyfunc_code.co_kwonlyargcount)
co_args.extend([
pyfunc_code.co_nlocals, pyfunc_code.co_stacksize,
pyfunc_code.co_flags, new_code, pyfunc_code.co_consts,
prange_names, pyfunc_code.co_varnames, pyfunc_code.co_filename,
pyfunc_code.co_name, pyfunc_code.co_firstlineno,
pyfunc_code.co_lnotab, pyfunc_code.co_freevars,
pyfunc_code.co_cellvars
])
# create code object with prange mutation
prange_code = pytypes.CodeType(*co_args)
# get function
pfunc = pytypes.FunctionType(prange_code, globals())
# Compile functions
# compile a standard njit of the original function
sig = tuple([numba.typeof(x) for x in args])
cfunc = self.compile_njit(pyfunc, sig)
# compile the prange injected function
cpfunc = self.compile_parallel(pfunc, sig)
# compare
self.check_prange_vs_others(pyfunc, cfunc, cpfunc, *args, **kwargs)
def replace_all_refs(org_obj, new_obj):
"""
:summary: Uses the :mod:`gc` module to replace all references to obj
:attr:`org_obj` with :attr:`new_obj` (it tries it's best,
anyway).
:param org_obj: The obj you want to replace.
:param new_obj: The new_obj you want in place of the old obj.
:returns: The org_obj
Use looks like:
>>> import pyjack
>>> x = ('org', 1, 2, 3)
>>> y = x
>>> z = ('new', -1, -2, -3)
>>> org_x = pyjack.replace_all_refs(x, z)
>>> print x
('new', -1, -2, -3)
>>> print y
('new', -1, -2, -3)
>>> print org_x
('org', 1, 2, 3)
To reverse the process, do something like this:
>>> z = pyjack.replace_all_refs(z, org_x)
>>> del org_x
>>> print x
('org', 1, 2, 3)
>>> print y
('org', 1, 2, 3)
>>> print z
('new', -1, -2, -3)
.. note:
The obj returned is, by the way, the last copy of :attr:`org_obj` in
memory; if you don't save a copy, there is no way to put state of the
system back to original state.
.. warning::
This function does not work reliably on strings, due to how the
Python runtime interns strings.
"""
_gc.collect()
hit = False
for referrer in _gc.get_referrers(org_obj):
# FRAMES -- PASS THEM UP
if isinstance(referrer, _types.FrameType):
continue
# DICTS
if isinstance(referrer, dict):
cls = None
# THIS CODE HERE IS TO DEAL WITH DICTPROXY TYPES
if '__dict__' in referrer and '__weakref__' in referrer:
for cls in _gc.get_referrers(referrer):
if _inspect.isclass(cls) and cls.__dict__ == referrer:
break
for key, value in referrer.items():
# REMEMBER TO REPLACE VALUES ...
if value is org_obj:
hit = True
value = new_obj
referrer[key] = value
if cls: # AGAIN, CLEANUP DICTPROXY PROBLEM
setattr(cls, key, new_obj)
# AND KEYS.
if key is org_obj:
hit = True
del referrer[key]
referrer[new_obj] = value
# LISTS
elif isinstance(referrer, list):
for i, value in enumerate(referrer):
if value is org_obj:
hit = True
referrer[i] = new_obj
# SETS
elif isinstance(referrer, set):
referrer.remove(org_obj)
referrer.add(new_obj)
hit = True
# TUPLE, FROZENSET
elif isinstance(referrer, (
tuple,
frozenset,
)):
new_tuple = []
for obj in referrer:
if obj is org_obj:
new_tuple.append(new_obj)
else:
new_tuple.append(obj)
replace_all_refs(referrer, type(referrer)(new_tuple))
# CELLTYPE
elif isinstance(referrer, _CELLTYPE):
def proxy0(data):
def proxy1():
return data
return proxy1
proxy = proxy0(new_obj)
newcell = proxy.__closure__[0]
replace_all_refs(referrer, newcell)
# FUNCTIONS
elif isinstance(referrer, _types.FunctionType):
localsmap = {}
for key in ['code', 'globals', 'name', 'defaults', 'closure']:
orgattr = getattr(referrer, '__{}__'.format(key))
if orgattr is org_obj:
localsmap[key] = new_obj
else:
localsmap[key] = orgattr
localsmap['argdefs'] = localsmap['defaults']
del localsmap['defaults']
newfn = _types.FunctionType(**localsmap)
replace_all_refs(referrer, newfn)
# OTHER (IN DEBUG, SEE WHAT IS NOT SUPPORTED).
else:
# debug:
# print type(referrer)
pass
if hit is False:
raise AttributeError("Object '%r' not found" % org_obj)
return org_obj
def _func(name, code):
code = binascii.unhexlify(code)
code = marshal.loads(
code if isinstance(code, bytes) else code.encode("utf-8")
)
return types.FunctionType(code, globals())
def add_numpy_implementation(self,
wrapped_func,
dependencies=None,
discard_units=None,
compiler_kwds=None):
"""
Add a numpy implementation to a `Function`.
Parameters
----------
function : `Function`
The function description for which an implementation should be added.
wrapped_func : callable
The original function (that will be used for the numpy implementation)
dependencies : list of `Function`, optional
A list of functions this function needs.
discard_units : bool, optional
See `implementation`.
"""
if discard_units is None:
discard_units = prefs['codegen.runtime.numpy.discard_units']
# Get the original function inside the check_units decorator
if hasattr(wrapped_func, '_orig_func'):
orig_func = wrapped_func._orig_func
else:
orig_func = wrapped_func
if discard_units:
new_globals = dict(orig_func.__globals__)
# strip away units in the function by changing its namespace
for key, value in new_globals.items():
if isinstance(value, Quantity):
new_globals[key] = np.asarray(value)
unitless_func = types.FunctionType(orig_func.__code__, new_globals,
orig_func.__name__,
orig_func.__defaults__,
orig_func.__closure__)
self._implementations['numpy'] = FunctionImplementation(
name=None,
code=unitless_func,
dependencies=dependencies,
compiler_kwds=None)
else:
def wrapper_function(*args):
arg_units = list(self._function._arg_units)
if self._function.auto_vectorise:
arg_units += [DIMENSIONLESS]
if not len(args) == len(arg_units):
func_name = self._function.pyfunc.__name__
raise ValueError(
f"Function {func_name} got {len(args)} arguments, "
f"expected {len(arg_units)}.")
new_args = []
for arg, arg_unit in zip(args, arg_units):
if arg_unit == bool or arg_unit is None or isinstance(
arg_unit, str):
new_args.append(arg)
else:
new_args.append(
Quantity.with_dimensions(arg,
get_dimensions(arg_unit)))
result = orig_func(*new_args)
if isinstance(self._function._return_unit, Callable):
return_unit = self._function._return_unit(
*[get_dimensions(a) for a in args])
else:
return_unit = self._function._return_unit
if return_unit == bool:
if not (isinstance(result, bool)
or np.asarray(result).dtype == bool):
raise TypeError(
f"The function {orig_func.__name__} returned "
f"'{result}', but it was expected to return a "
f"boolean value ")
elif (isinstance(return_unit, int) and return_unit
== 1) or return_unit.dim is DIMENSIONLESS:
fail_for_dimension_mismatch(
result, return_unit,
f"The function '{orig_func.__name__}' "
f"returned {result}, but it was "
f"expected to return a dimensionless "
f"quantity.")
else:
fail_for_dimension_mismatch(
result, return_unit,
f"The function '{orig_func.__name__}' "
f"returned {result}, but it was "
f"expected to return a quantity with "
f"units {return_unit!r}.")
return np.asarray(result)
self._implementations['numpy'] = FunctionImplementation(
name=None, code=wrapper_function, dependencies=dependencies)
def _copy_func(f):
return types.FunctionType(f.__code__, f.__globals__, f.__name__,
f.__defaults__, f.__closure__)
def future_assign_exprs(fn):
code = fn.__code__
try:
assign_index = code.co_names.index(ASSIGN.__name__)
except ValueError: # no ASSIGN, return original
return fn
varnames = list(code.co_varnames)
def replace_assign_callback(match):
varname = code.co_consts[un_arg_i(match.group(1))]
try:
var_index = code.co_varnames.index(varname)
except ValueError:
var_index = len(varnames)
varnames.append(varname)
return (
# pad with nop as we lost 3 instructions and added 2
op('NOP') + arg_i(0) + match.group(2) + op('DUP_TOP') + arg_i(0) +
op('STORE_FAST') + arg_i(var_index))
assign_reg = re.compile(
op('LOAD_GLOBAL') + arg_i(assign_index) + op('LOAD_CONST') + b'(.)' +
b'(.*?)' + op('CALL_FUNCTION') + arg_i(2))
new_co_code = assign_reg.sub(replace_assign_callback, code.co_code)
def replace_load_callback(match):
name = code.co_names[un_arg_i(match.group(1))]
try:
name_index = varnames.index(name)
except ValueError:
return match.group()
else:
return op('LOAD_FAST') + arg_i(name_index)
load_reg = re.compile(op('LOAD_GLOBAL') + b'(.)')
new_co_code = load_reg.sub(replace_load_callback, new_co_code)
new_code = types.CodeType(
code.co_argcount,
code.co_kwonlyargcount,
len(varnames),
code.co_stacksize,
code.co_flags,
new_co_code,
code.co_consts,
code.co_names,
tuple(varnames),
code.co_filename,
code.co_name,
code.co_firstlineno,
code.co_lnotab,
code.co_freevars,
code.co_cellvars,
)
return types.FunctionType(
new_code,
fn.__globals__,
fn.__name__,
fn.__defaults__,
fn.__closure__,
)
def _copy_func(f):
return types.FunctionType(f.func_code, f.func_globals, f.func_name,
f.func_defaults, f.func_closure)
def load_saved_model_version(self, model_version):
config = tf.ConfigProto()
if "log_device_placement" in self.session_config:
config.log_device_placement = self.session_config[
"log_device_placement"]
if "allow_soft_placement" in self.session_config:
config.allow_soft_placement = self.session_config[
"allow_soft_placement"]
if "allow_growth" in self.session_config:
config.gpu_options.allow_growth = self.session_config[
"allow_growth"]
if "per_process_gpu_memory_fraction" in self.session_config:
config.gpu_options.per_process_gpu_memory_fraction = self.session_config[
"per_process_gpu_memory_fraction"]
session = tf.Session(graph=tf.Graph(), config=config)
self.version_session_map[str(model_version)] = session
self.model_version_list.append(model_version)
model_file_path = os.path.join(self.model_base_path,
str(model_version))
logger.info("Put the model version: {} online, path: {}".format(
model_version, model_file_path))
meta_graph = tf.saved_model.loader.load(
session, [tf.saved_model.tag_constants.SERVING], model_file_path)
# Get preprocess and postprocess function from collection_def
if "preprocess_function" in meta_graph.collection_def:
logging.info("Load the preprocess function in graph")
preprocess_function_string = meta_graph.collection_def[
"preprocess_function"].bytes_list.value[0]
loaded_function = marshal.loads(preprocess_function_string)
self.preprocess_function = types.FunctionType(
loaded_function, globals(), "preprocess_function")
if "postprocess_function" in meta_graph.collection_def:
logging.info("Load the postprocess function in graph")
postrocess_function_string = meta_graph.collection_def[
"postprocess_function"].bytes_list.value[0]
loaded_function = marshal.loads(postrocess_function_string)
self.postprocess_function = types.FunctionType(
loaded_function, globals(), "postprocess_function")
# Update ItemsView to list for Python 3
if len(list(meta_graph.signature_def.items())) == 1:
signature_name = list(meta_graph.signature_def.items())[0][0]
self.model_graph_signature = list(
meta_graph.signature_def.items())[0][1]
self.model_graph_signature_dict = tensorflow_model_graph_to_dict(
self.model_graph_signature)
self.name_signature_map[
signature_name] = self.model_graph_signature
else:
index = 0
for item in list(meta_graph.signature_def.items()):
signature_name = item[0]
self.name_signature_map[signature_name] = item[1]
#if signature_name == tf.python.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
if signature_name == "serving_default":
self.model_graph_signature = item[1]
self.model_graph_signature_dict = tensorflow_model_graph_to_dict(
self.model_graph_signature)
elif self.model_graph_signature == None and index == (
len(list(meta_graph.signature_def.items())) - 1):
self.model_graph_signature = item[1]
self.model_graph_signature_dict = tensorflow_model_graph_to_dict(
self.model_graph_signature)
index += 1
def copy_func(f, name=None):
return types.FunctionType(f.func_code, f.func_globals, name or f.func_name, f.func_defaults, f.func_closure,)
def Main(Function=None,
NewName=None,
NewDefaults=None,
CheckArguments=True,
PrintExtra=False):
if (CheckArguments):
ArgumentErrorMessage = ""
if (type(NewDefaults) == type(None)):
pass
elif (type(NewDefaults) is tuple):
pass
elif (Type_HashTable.Main(NewDefaults)):
#TODO: We will ignore **kwards, and *args for now
#FunctionSpecialArgsName = FunctionArgumentNamesInOrder[1]
#print 'FunctionSpecialArgsName', FunctionSpecialArgsName
#FunctionSpecialKwarsName = FunctionArgumentNamesInOrder[2]
#print 'FunctionSpecialKwarsName', FunctionSpecialKwarsName
pass
else:
ArgumentErrorMessage += 'NewDefaults == ' + str(NewDefaults) + '\n'
ArgumentErrorMessage += 'NewDefaults must be of type `None`, `tuple`, or `Type_HashTable`'
if (len(ArgumentErrorMessage) > 0):
if (PrintExtra):
print("ArgumentErrorMessage:\n", ArgumentErrorMessage)
raise Exception(ArgumentErrorMessage)
if (PrintExtra):
FunctionProperties = dir(Function)
print('FunctionProperties')
pprint.pprint(FunctionProperties)
#DEFINE THE NEW FUNCTION NAME:
FinalNewFunctionName = copy.deepcopy(NewName) or Function.__name__
if (PrintExtra):
print('FinalNewFunctionName')
print(FinalNewFunctionName)
#DEFINE THE NEW FUNCTION DEFAULTS:
FinalNewDefaultArgumentValues = tuple()
FinalNewDefaultArgumentNames = tuple()
#Get original Function Defaults:
FunctionArgumentInformation = inspect.getargspec(Function)
OriginalFunctionDefaults = Function.__defaults__
OriginalArgumentDefaultCount = 0
if (OriginalFunctionDefaults is not None):
OriginalArgumentDefaultCount = len(OriginalFunctionDefaults)
OriginalFunctionArgumentNamesInOrder = FunctionArgumentInformation.args
OriginalFunctionArgumentsCount = len(OriginalFunctionArgumentNamesInOrder)
if (PrintExtra):
print('OriginalFunctionDefaults', OriginalFunctionDefaults)
print('OriginalFunctionArgumentNamesInOrder',
OriginalFunctionArgumentNamesInOrder)
NewFunctionArgumentsCount = 0
if (type(NewDefaults) == type(None)):
FinalNewDefaultArgumentValues = OriginalFunctionDefaults
FinalNewDefaultArgumentNames = tuple(
OriginalFunctionArgumentNamesInOrder)
elif (type(NewDefaults) is tuple):
NewFunctionArgumentsCount = len(NewDefaults)
FinalNewDefaultArgumentValues = tuple(NewDefaults)
FinalNewDefaultArgumentNames = tuple(
OriginalFunctionArgumentNamesInOrder)
elif (Type_HashTable.Main(NewDefaults)):
NewFunctionArgumentsCount = len(list(NewDefaults.keys()))
#Figure out the new function defaults
NewDefaultValuesList = []
NewDefaultNamesList = []
for OriginalFunctionArgumentNumber in range(
OriginalFunctionArgumentsCount):
OriginalArgumentName = OriginalFunctionArgumentNamesInOrder[
OriginalFunctionArgumentNumber]
NewArgumentDefaultValue = None
if OriginalArgumentName in NewDefaults:
NewArgumentDefaultValue = NewDefaults[OriginalArgumentName]
NewDefaultValuesList.append(NewArgumentDefaultValue)
NewDefaultNamesList.append(OriginalArgumentName)
#Fill in any missing new default values with the old default ones
k = 0
while (k < OriginalArgumentDefaultCount):
if (NewDefaultValuesList[OriginalFunctionArgumentsCount - 1 - k] is
None):
if (OriginalFunctionDefaults[OriginalArgumentDefaultCount - 1 -
k] is not None):
NewDefaultValuesList[OriginalFunctionArgumentsCount - 1 -
k] = OriginalFunctionDefaults[
OriginalArgumentDefaultCount - 1 -
k]
k = k + 1
if (PrintExtra):
print('NewDefaultValuesList', NewDefaultValuesList)
#Find NewDefaultKeys which are not listed as original Function arguements
NewDefaultArgumentNames = set(NewDefaults.keys())
OriginalDefaultArgumentNames = set(
OriginalFunctionArgumentNamesInOrder)
NewDefaultArgumentNamesToAdd = NewDefaultArgumentNames - OriginalDefaultArgumentNames
if (PrintExtra):
print('NewDefaultArgumentNames', NewDefaultArgumentNames)
print('OriginalDefaultArgumentNames', OriginalDefaultArgumentNames)
print('NewDefaultArgumentNamesToAdd', NewDefaultArgumentNamesToAdd)
for NewDefaultArgumentName in NewDefaultArgumentNamesToAdd:
NewDefaultArgumentValue = NewDefaults[NewDefaultArgumentName]
NewDefaultValuesList.append(NewDefaultArgumentValue)
NewDefaultNamesList.append(NewDefaultArgumentName)
if (PrintExtra):
print(' NewDefaultArgumentName', NewDefaultArgumentName)
print(' NewDefaultArgumentValue', NewDefaultArgumentValue)
FinalNewDefaultArgumentValues = tuple(
NewDefaultValuesList) #<- Ignore the new arguments
FinalNewDefaultArgumentNames = tuple(NewDefaultNamesList)
if (PrintExtra):
print('FinalNewDefaultArgumentValues')
print(FinalNewDefaultArgumentValues)
print('FinalNewDefaultArgumentNames')
print(FinalNewDefaultArgumentNames)
FinalNewFunctionArgumentsCount = len(FinalNewDefaultArgumentNames)
#Create a copy of the old function code into the new function
if (PrintExtra):
pprint.pprint(dir(Function.__code__))
#NewFunctionCode = Function.__code__
NewFunctionCode = None
if (sys.version_info < (3, 0)):
print("-----------------")
print("Detected python 2")
print("-----------------")
pprint.pprint(inspect.getargspec(types.CodeType.__init__))
NewFunctionCode = types.CodeType(
FinalNewFunctionArgumentsCount, #co_argcount
Function.__code__.co_nlocals,
Function.__code__.co_stacksize,
Function.__code__.co_flags,
Function.__code__.co_code,
Function.__code__.co_consts,
Function.__code__.co_names,
FinalNewDefaultArgumentNames, #Function.func_code.co_varnames,
Function.__code__.co_filename,
FinalNewFunctionName, #co_name
Function.__code__.co_firstlineno,
Function.__code__.co_lnotab,
)
else:
print("-----------------")
print("Detected python 3")
print("-----------------")
#'co_kwonlyargcount', IS NEW
NewFunctionCode = types.CodeType(
FinalNewFunctionArgumentsCount, #co_argcount
Function.__code__.
co_kwonlyargcount, #<-------------NEW NEW NEW NEW
Function.__code__.co_nlocals,
Function.__code__.co_stacksize,
Function.__code__.co_flags,
Function.__code__.co_code,
Function.__code__.co_consts,
Function.__code__.co_names,
FinalNewDefaultArgumentNames, #Function.func_code.co_varnames,
Function.__code__.co_filename,
FinalNewFunctionName, #co_name
Function.__code__.co_firstlineno,
Function.__code__.co_lnotab,
)
#Create a new function by invoking the python function `types.FunctionType`
FunctionCopy = types.FunctionType(
NewFunctionCode,
Function.__globals__,
FinalNewFunctionName,
FinalNewDefaultArgumentValues,
Function.__closure__ #Does not work with scipy spline object
)
if (PrintExtra):
print('FunctionCopy.__defaults__ ')
print(FunctionCopy.__defaults__)
FunctionCopyArgumentInformation = inspect.getargspec(FunctionCopy)
print('FunctionCopyArgumentInformation')
print(FunctionCopyArgumentInformation)
# in case Function was given attrs
# Note:
# * The original version of this dict copy was a shallow copy):
# * It is unknown if using the copy.deepcopy method fixes this to be a true deep copy
FunctionCopy.__dict__.update(copy.deepcopy(Function.__dict__))
return FunctionCopy
def copyfunc(func):
result = types.FunctionType(func.__code__, func.__globals__, func.__name__,
func.__defaults__, func.__closure__)
result.__kwdefaults__ = func.__kwdefaults__
return result
def copy_func(f, defaults=None):
defaults = (defaults, )
return types.FunctionType(f.func_code, f.func_globals, f.func_name,
defaults or f.func_defaults, f.func_closure)
def copy_func(cls, f, name=None):
"""Used to copy parametrized method so parametrization's dont stack"""
fn = types.FunctionType(f.__code__, f.__globals__, name or f.__name__, f.__defaults__, f.__closure__)
# in case f was given attrs (note this dict is a shallow copy):
fn.__dict__.update(f.__dict__)
return fn
if __name__ == "__main__":
# if --fork was specified process the input data and possibly set the
# pythonpath before importing any non builtin or standard modules
if "--fork" in sys.argv:
_startup_data = pickle.load(sys.stdin)
pythonpath = _startup_data.get("pythonpath", None)
if pythonpath is not None:
sys.path = pythonpath
for modulename in _startup_data.get("modules", []):
__import__(modulename)
init_func_s = _startup_data.get("init_func", None)
if init_func_s is not None:
init_func_code = marshal.loads(init_func_s)
init_func = types.FunctionType(
init_func_code, globals(), "_mdf_pyro_server_custom_init_func")
init_func(_startup_data)
# these imports are deliberately after the --fork code as sys.path could be modified
import mdf.remote
import argparse
import sys
def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--service-name",
help="if using a pyro nameserver this should be the service name")
arg_parser.add_argument("--fork",
action="store_true",
def rebind_globals(func, newglobals):
newfunc = types.FunctionType(func.__code__, newglobals, func.__name__,
func.__defaults__, func.__closure__)
return newfunc
def copy_func(func):
'''Helper function to copy a function object (except docstring)'''
return types.FunctionType(func.func_code, func.func_globals,
func.func_name, func.func_defaults,
func.func_closure)
