def __call__(self, *args, **kwargs):
# Compare the function code with the previous to see if the
# function code has changed
output_dir, argument_hash = self.get_output_dir(*args, **kwargs)
# FIXME: The statements below should be try/excepted
if not (self._check_previous_func_code(stacklevel=3)
and os.path.exists(output_dir)):
if self._verbose > 10:
_, name = get_func_name(self.func)
self.warn('Computing func %s, argument hash %s in '
'directory %s' % (name, argument_hash, output_dir))
return self.call(*args, **kwargs)
else:
try:
t0 = time.time()
out = self.load_output(output_dir)
if self._verbose > 4:
t = time.time() - t0
_, name = get_func_name(self.func)
msg = '%s cache loaded - %s' % (name, format_time(t))
print max(0, (80 - len(msg))) * '_' + msg
return out
except Exception:
# XXX: Should use an exception logger
self.warn('Exception while loading results for '
'(args=%s, kwargs=%s)\n %s' %
(args, kwargs, traceback.format_exc()))
shutil.rmtree(output_dir, ignore_errors=True)
return self.call(*args, **kwargs)
def __call__(self, *args, **kwargs):
# Compare the function code with the previous to see if the
# function code has changed
output_dir, argument_hash = self.get_output_dir(*args, **kwargs)
# FIXME: The statements below should be try/excepted
if not (self._check_previous_func_code(stacklevel=3) and
os.path.exists(output_dir)):
if self._verbose > 10:
_, name = get_func_name(self.func)
self.warn('Computing func %s, argument hash %s in '
'directory %s'
% (name, argument_hash, output_dir))
return self.call(*args, **kwargs)
else:
try:
t0 = time.time()
out = self.load_output(output_dir)
if self._verbose > 4:
t = time.time() - t0
_, name = get_func_name(self.func)
msg = '%s cache loaded - %s' % (name, format_time(t))
print max(0, (80 - len(msg))) * '_' + msg
return out
except Exception:
# XXX: Should use an exception logger
self.warn('Exception while loading results for '
'(args=%s, kwargs=%s)\n %s' %
(args, kwargs, traceback.format_exc()))
shutil.rmtree(output_dir, ignore_errors=True)
return self.call(*args, **kwargs)
def format_signature(self, func, *args, **kwds):
# XXX: This should be moved out to a function
# XXX: Should this use inspect.formatargvalues/formatargspec?
module, name = get_func_name(func)
module = [m for m in module if m]
if module:
module.append(name)
module_path = '.'.join(module)
else:
module_path = name
arg_str = list()
previous_length = 0
for arg in args:
arg = self.format(arg, indent=2)
if len(arg) > 1500:
arg = '%s...' % arg[:700]
if previous_length > 80:
arg = '\n%s' % arg
previous_length = len(arg)
arg_str.append(arg)
arg_str.extend(
['%s=%s' % (v, self.format(i)) for v, i in kwds.iteritems()])
arg_str = ', '.join(arg_str)
signature = '%s(%s)' % (name, arg_str)
return module_path, signature
def format_signature(self, func, *args, **kwds):
# XXX: This should be moved out to a function
# XXX: Should this use inspect.formatargvalues/formatargspec?
module, name = get_func_name(func)
module = [m for m in module if m]
if module:
module.append(name)
module_path = '.'.join(module)
else:
module_path = name
arg_str = list()
previous_length = 0
for arg in args:
arg = self.format(arg, indent=2)
if len(arg) > 1500:
arg = '%s...' % arg[:700]
if previous_length > 80:
arg = '\n%s' % arg
previous_length = len(arg)
arg_str.append(arg)
arg_str.extend(['%s=%s' % (v, self.format(i)) for v, i in
kwds.iteritems()])
arg_str = ', '.join(arg_str)
signature = '%s(%s)' % (name, arg_str)
return module_path, signature
def _get_func_dir(self, mkdir=True):
""" Get the directory corresponding to the cache for the
function.
"""
module, name = get_func_name(self.func)
module.append(name)
func_dir = os.path.join(self.cachedir, *module)
if mkdir:
mkdirp(func_dir)
return func_dir
def _get_func_dir(self, mkdir=True):
""" Get the directory corresponding to the cache for the
function.
"""
module, name = get_func_name(self.func)
module.append(name)
func_dir = os.path.join(self.cachedir, *module)
if mkdir:
mkdirp(func_dir)
return func_dir
def call(self, *args, **kwargs):
""" Force the execution of the function with the given arguments and
persist the output values.
"""
start_time = time.time()
output_dir, argument_hash = self.get_output_dir(*args, **kwargs)
if self._verbose:
print self.format_call(*args, **kwargs)
output = self.func(*args, **kwargs)
self._persist_output(output, output_dir)
duration = time.time() - start_time
if self._verbose:
_, name = get_func_name(self.func)
msg = '%s - %s' % (name, format_time(duration))
print max(0, (80 - len(msg))) * '_' + msg
return output
def call(self, *args, **kwargs):
""" Force the execution of the function with the given arguments and
persist the output values.
"""
start_time = time.time()
output_dir, argument_hash = self.get_output_dir(*args, **kwargs)
if self._verbose:
print self.format_call(*args, **kwargs)
output = self.func(*args, **kwargs)
self._persist_output(output, output_dir)
duration = time.time() - start_time
if self._verbose:
_, name = get_func_name(self.func)
msg = '%s - %s' % (name, format_time(duration))
print max(0, (80 - len(msg))) * '_' + msg
return output
def _check_previous_func_code(self, stacklevel=2):
"""
stacklevel is the depth a which this function is called, to
issue useful warnings to the user.
"""
# Here, we go through some effort to be robust to dynamically
# changing code and collision. We cannot inspect.getsource
# because it is not reliable when using IPython's magic "%run".
func_code, source_file, first_line = get_func_code(self.func)
func_dir = self._get_func_dir()
func_code_file = os.path.join(func_dir, 'func_code.py')
try:
infile = open(func_code_file)
old_func_code, old_first_line = extract_first_line(infile.read())
infile.close()
except IOError:
self._write_func_code(func_code_file, func_code, first_line)
return False
if old_func_code == func_code:
return True
# We have differing code, is this because we are refering to
# differing functions, or because the function we are refering as
# changed?
if old_first_line == first_line == -1:
_, func_name = get_func_name(self.func,
resolv_alias=False,
win_characters=False)
if not first_line == -1:
func_description = '%s (%s:%i)' % (func_name, source_file,
first_line)
else:
func_description = func_name
warnings.warn(JobLibCollisionWarning(
"Cannot detect name collisions for function '%s'" %
func_description),
stacklevel=stacklevel)
# Fetch the code at the old location and compare it. If it is the
# same than the code store, we have a collision: the code in the
# file has not changed, but the name we have is pointing to a new
# code block.
if (not old_first_line == first_line and source_file is not None
and os.path.exists(source_file)):
_, func_name = get_func_name(self.func, resolv_alias=False)
num_lines = len(func_code.split('\n'))
on_disk_func_code = file(
source_file).readlines()[old_first_line - 1:old_first_line -
1 + num_lines - 1]
on_disk_func_code = ''.join(on_disk_func_code)
if on_disk_func_code.rstrip() == old_func_code.rstrip():
warnings.warn(JobLibCollisionWarning(
'Possible name collisions between functions '
"'%s' (%s:%i) and '%s' (%s:%i)" %
(func_name, source_file, old_first_line, func_name,
source_file, first_line)),
stacklevel=stacklevel)
# The function has changed, wipe the cache directory.
# XXX: Should be using warnings, and giving stacklevel
if self._verbose > 10:
_, func_name = get_func_name(self.func, resolv_alias=False)
self.warn("Function %s (stored in %s) has changed." %
(func_name, func_dir))
self.clear(warn=True)
return False
def _check_previous_func_code(self, stacklevel=2):
"""
stacklevel is the depth a which this function is called, to
issue useful warnings to the user.
"""
# Here, we go through some effort to be robust to dynamically
# changing code and collision. We cannot inspect.getsource
# because it is not reliable when using IPython's magic "%run".
func_code, source_file, first_line = get_func_code(self.func)
func_dir = self._get_func_dir()
func_code_file = os.path.join(func_dir, 'func_code.py')
try:
infile = open(func_code_file)
old_func_code, old_first_line = extract_first_line(infile.read())
infile.close()
except IOError:
self._write_func_code(func_code_file, func_code, first_line)
return False
if old_func_code == func_code:
return True
# We have differing code, is this because we are refering to
# differing functions, or because the function we are refering as
# changed?
if old_first_line == first_line == -1:
_, func_name = get_func_name(self.func, resolv_alias=False,
win_characters=False)
if not first_line == -1:
func_description = '%s (%s:%i)' % (func_name,
source_file, first_line)
else:
func_description = func_name
warnings.warn(JobLibCollisionWarning(
"Cannot detect name collisions for function '%s'"
% func_description), stacklevel=stacklevel)
# Fetch the code at the old location and compare it. If it is the
# same than the code store, we have a collision: the code in the
# file has not changed, but the name we have is pointing to a new
# code block.
if (not old_first_line == first_line
and source_file is not None
and os.path.exists(source_file)):
_, func_name = get_func_name(self.func, resolv_alias=False)
num_lines = len(func_code.split('\n'))
on_disk_func_code = file(source_file).readlines()[
old_first_line - 1:old_first_line - 1 + num_lines - 1]
on_disk_func_code = ''.join(on_disk_func_code)
if on_disk_func_code.rstrip() == old_func_code.rstrip():
warnings.warn(JobLibCollisionWarning(
'Possible name collisions between functions '
"'%s' (%s:%i) and '%s' (%s:%i)" %
(func_name, source_file, old_first_line,
func_name, source_file, first_line)),
stacklevel=stacklevel)
# The function has changed, wipe the cache directory.
# XXX: Should be using warnings, and giving stacklevel
if self._verbose > 10:
_, func_name = get_func_name(self.func, resolv_alias=False)
self.warn("Function %s (stored in %s) has changed." %
(func_name, func_dir))
self.clear(warn=True)
return False