def __call__(self, *args, **kwargs): # Compare the function code with the previous to see if the # function code has changed output_dir, argument_hash = self.get_output_dir(*args, **kwargs) # FIXME: The statements below should be try/excepted if not (self._check_previous_func_code(stacklevel=3) and os.path.exists(output_dir)): if self._verbose > 10: _, name = get_func_name(self.func) self.warn('Computing func %s, argument hash %s in ' 'directory %s' % (name, argument_hash, output_dir)) return self.call(*args, **kwargs) else: try: t0 = time.time() out = self.load_output(output_dir) if self._verbose > 4: t = time.time() - t0 _, name = get_func_name(self.func) msg = '%s cache loaded - %s' % (name, format_time(t)) print max(0, (80 - len(msg))) * '_' + msg return out except Exception: # XXX: Should use an exception logger self.warn('Exception while loading results for ' '(args=%s, kwargs=%s)\n %s' % (args, kwargs, traceback.format_exc())) shutil.rmtree(output_dir, ignore_errors=True) return self.call(*args, **kwargs)
def __call__(self, *args, **kwargs): # Compare the function code with the previous to see if the # function code has changed output_dir, argument_hash = self.get_output_dir(*args, **kwargs) # FIXME: The statements below should be try/excepted if not (self._check_previous_func_code(stacklevel=3) and os.path.exists(output_dir)): if self._verbose > 10: _, name = get_func_name(self.func) self.warn('Computing func %s, argument hash %s in ' 'directory %s' % (name, argument_hash, output_dir)) return self.call(*args, **kwargs) else: try: t0 = time.time() out = self.load_output(output_dir) if self._verbose > 4: t = time.time() - t0 _, name = get_func_name(self.func) msg = '%s cache loaded - %s' % (name, format_time(t)) print max(0, (80 - len(msg))) * '_' + msg return out except Exception: # XXX: Should use an exception logger self.warn('Exception while loading results for ' '(args=%s, kwargs=%s)\n %s' % (args, kwargs, traceback.format_exc())) shutil.rmtree(output_dir, ignore_errors=True) return self.call(*args, **kwargs)
def format_signature(self, func, *args, **kwds): # XXX: This should be moved out to a function # XXX: Should this use inspect.formatargvalues/formatargspec? module, name = get_func_name(func) module = [m for m in module if m] if module: module.append(name) module_path = '.'.join(module) else: module_path = name arg_str = list() previous_length = 0 for arg in args: arg = self.format(arg, indent=2) if len(arg) > 1500: arg = '%s...' % arg[:700] if previous_length > 80: arg = '\n%s' % arg previous_length = len(arg) arg_str.append(arg) arg_str.extend( ['%s=%s' % (v, self.format(i)) for v, i in kwds.iteritems()]) arg_str = ', '.join(arg_str) signature = '%s(%s)' % (name, arg_str) return module_path, signature
def format_signature(self, func, *args, **kwds): # XXX: This should be moved out to a function # XXX: Should this use inspect.formatargvalues/formatargspec? module, name = get_func_name(func) module = [m for m in module if m] if module: module.append(name) module_path = '.'.join(module) else: module_path = name arg_str = list() previous_length = 0 for arg in args: arg = self.format(arg, indent=2) if len(arg) > 1500: arg = '%s...' % arg[:700] if previous_length > 80: arg = '\n%s' % arg previous_length = len(arg) arg_str.append(arg) arg_str.extend(['%s=%s' % (v, self.format(i)) for v, i in kwds.iteritems()]) arg_str = ', '.join(arg_str) signature = '%s(%s)' % (name, arg_str) return module_path, signature
def _get_func_dir(self, mkdir=True): """ Get the directory corresponding to the cache for the function. """ module, name = get_func_name(self.func) module.append(name) func_dir = os.path.join(self.cachedir, *module) if mkdir: mkdirp(func_dir) return func_dir
def _get_func_dir(self, mkdir=True): """ Get the directory corresponding to the cache for the function. """ module, name = get_func_name(self.func) module.append(name) func_dir = os.path.join(self.cachedir, *module) if mkdir: mkdirp(func_dir) return func_dir
def call(self, *args, **kwargs): """ Force the execution of the function with the given arguments and persist the output values. """ start_time = time.time() output_dir, argument_hash = self.get_output_dir(*args, **kwargs) if self._verbose: print self.format_call(*args, **kwargs) output = self.func(*args, **kwargs) self._persist_output(output, output_dir) duration = time.time() - start_time if self._verbose: _, name = get_func_name(self.func) msg = '%s - %s' % (name, format_time(duration)) print max(0, (80 - len(msg))) * '_' + msg return output
def call(self, *args, **kwargs): """ Force the execution of the function with the given arguments and persist the output values. """ start_time = time.time() output_dir, argument_hash = self.get_output_dir(*args, **kwargs) if self._verbose: print self.format_call(*args, **kwargs) output = self.func(*args, **kwargs) self._persist_output(output, output_dir) duration = time.time() - start_time if self._verbose: _, name = get_func_name(self.func) msg = '%s - %s' % (name, format_time(duration)) print max(0, (80 - len(msg))) * '_' + msg return output
def _check_previous_func_code(self, stacklevel=2): """ stacklevel is the depth a which this function is called, to issue useful warnings to the user. """ # Here, we go through some effort to be robust to dynamically # changing code and collision. We cannot inspect.getsource # because it is not reliable when using IPython's magic "%run". func_code, source_file, first_line = get_func_code(self.func) func_dir = self._get_func_dir() func_code_file = os.path.join(func_dir, 'func_code.py') try: infile = open(func_code_file) old_func_code, old_first_line = extract_first_line(infile.read()) infile.close() except IOError: self._write_func_code(func_code_file, func_code, first_line) return False if old_func_code == func_code: return True # We have differing code, is this because we are refering to # differing functions, or because the function we are refering as # changed? if old_first_line == first_line == -1: _, func_name = get_func_name(self.func, resolv_alias=False, win_characters=False) if not first_line == -1: func_description = '%s (%s:%i)' % (func_name, source_file, first_line) else: func_description = func_name warnings.warn(JobLibCollisionWarning( "Cannot detect name collisions for function '%s'" % func_description), stacklevel=stacklevel) # Fetch the code at the old location and compare it. If it is the # same than the code store, we have a collision: the code in the # file has not changed, but the name we have is pointing to a new # code block. if (not old_first_line == first_line and source_file is not None and os.path.exists(source_file)): _, func_name = get_func_name(self.func, resolv_alias=False) num_lines = len(func_code.split('\n')) on_disk_func_code = file( source_file).readlines()[old_first_line - 1:old_first_line - 1 + num_lines - 1] on_disk_func_code = ''.join(on_disk_func_code) if on_disk_func_code.rstrip() == old_func_code.rstrip(): warnings.warn(JobLibCollisionWarning( 'Possible name collisions between functions ' "'%s' (%s:%i) and '%s' (%s:%i)" % (func_name, source_file, old_first_line, func_name, source_file, first_line)), stacklevel=stacklevel) # The function has changed, wipe the cache directory. # XXX: Should be using warnings, and giving stacklevel if self._verbose > 10: _, func_name = get_func_name(self.func, resolv_alias=False) self.warn("Function %s (stored in %s) has changed." % (func_name, func_dir)) self.clear(warn=True) return False
def _check_previous_func_code(self, stacklevel=2): """ stacklevel is the depth a which this function is called, to issue useful warnings to the user. """ # Here, we go through some effort to be robust to dynamically # changing code and collision. We cannot inspect.getsource # because it is not reliable when using IPython's magic "%run". func_code, source_file, first_line = get_func_code(self.func) func_dir = self._get_func_dir() func_code_file = os.path.join(func_dir, 'func_code.py') try: infile = open(func_code_file) old_func_code, old_first_line = extract_first_line(infile.read()) infile.close() except IOError: self._write_func_code(func_code_file, func_code, first_line) return False if old_func_code == func_code: return True # We have differing code, is this because we are refering to # differing functions, or because the function we are refering as # changed? if old_first_line == first_line == -1: _, func_name = get_func_name(self.func, resolv_alias=False, win_characters=False) if not first_line == -1: func_description = '%s (%s:%i)' % (func_name, source_file, first_line) else: func_description = func_name warnings.warn(JobLibCollisionWarning( "Cannot detect name collisions for function '%s'" % func_description), stacklevel=stacklevel) # Fetch the code at the old location and compare it. If it is the # same than the code store, we have a collision: the code in the # file has not changed, but the name we have is pointing to a new # code block. if (not old_first_line == first_line and source_file is not None and os.path.exists(source_file)): _, func_name = get_func_name(self.func, resolv_alias=False) num_lines = len(func_code.split('\n')) on_disk_func_code = file(source_file).readlines()[ old_first_line - 1:old_first_line - 1 + num_lines - 1] on_disk_func_code = ''.join(on_disk_func_code) if on_disk_func_code.rstrip() == old_func_code.rstrip(): warnings.warn(JobLibCollisionWarning( 'Possible name collisions between functions ' "'%s' (%s:%i) and '%s' (%s:%i)" % (func_name, source_file, old_first_line, func_name, source_file, first_line)), stacklevel=stacklevel) # The function has changed, wipe the cache directory. # XXX: Should be using warnings, and giving stacklevel if self._verbose > 10: _, func_name = get_func_name(self.func, resolv_alias=False) self.warn("Function %s (stored in %s) has changed." % (func_name, func_dir)) self.clear(warn=True) return False