def clean_cache_relations(job_id, db): #print('cleaning cache relations for %r ' % job_id) # for all jobs that were done cache = get_job_cache(job_id, db) if cache.state == Cache.DONE: for parent in direct_parents(job_id, db): parent_job = get_job(parent, db) #print(' parent %r has dynamic %s' % (parent, parent_job.dynamic_children)) if not job_id in parent_job.dynamic_children: #print(' skipping parent %r ' % parent) continue else: dynamic_children = parent_job.dynamic_children[job_id] #print(' dynamic_children %s' % parent_job.dynamic_children) #print(' children %s' % parent_job.children) del parent_job.dynamic_children[job_id] parent_job.children = parent_job.children - dynamic_children set_job(parent, parent_job, db)
def comp(command, *args, **kwargs): ''' Main method to define a computation. Extra arguments: :arg:job_id: sets the job id (respects job_prefix) :arg:extra_dep: extra dependencies (not passed as arguments) ''' if compmake.compmake_status == compmake_status_slave: return None # Check that this is a pickable function try: pickle.dumps(command) except: msg = ('Cannot pickle %r. Make sure it is not a lambda function or a ' 'nested function. (This is a limitation of Python)' % command) raise SerializationError(msg) args = list(args) # args is a non iterable tuple # Get job id from arguments job_id_key = 'job_id' if job_id_key in kwargs: # make sure that command does not have itself a job_id key #available = command.func_code.co_varnames argspec = inspect.getargspec(command) if job_id_key in argspec.args: msg = ("You cannot define the job id in this way because 'job_id' " "is already a parameter of this function.") raise UserError(msg) job_id = kwargs[job_id_key] if job_prefix: job_id = '%s-%s' % (job_prefix, job_id) del kwargs[job_id_key] if job_id in jobs_defined_in_this_session: raise UserError('Job %r already defined.' % job_id) else: job_id = generate_job_id(command) jobs_defined_in_this_session.add(job_id) if 'extra_dep' in kwargs: extra_dep = collect_dependencies(kwargs['extra_dep']) del kwargs['extra_dep'] else: extra_dep = set() children = collect_dependencies([args, kwargs]) children.update(extra_dep) all_args = (command, args, kwargs) command_desc = command.__name__ c = Job(job_id=job_id, children=list(children), command_desc=command_desc) # TODO: check for loops for child in children: child_comp = get_job(child) if not job_id in child_comp.parents: child_comp.parents.append(job_id) set_job(child, child_comp) if job_exists(job_id): # OK, this is going to be black magic. # We want to load the previous job definition, # however, by unpickling(), it will start # __import__()ing the modules, perhaps # even the one that is calling us. # What happens, then is that it will try to # add another time this computation recursively. # What we do, is that we temporarely switch to # slave mode, so that recursive calls to comp() # are disabled. if compmake_config.check_params: #@UndefinedVariable old_status = compmake_status set_compmake_status(compmake_status_slave) old_computation = get_job(job_id) set_compmake_status(old_status) assert False, 'update for job_args' same, reason = old_computation.same_computation(c) if not same: set_job(job_id, c) set_job_args(job_id, all_args) publish('job-redefined', job_id=job_id , reason=reason) # XXX TODO clean the cache else: publish('job-already-defined', job_id=job_id) else: # We assume everything's ok set_job(job_id, c) set_job_args(job_id, all_args) publish('job-defined', job_id=job_id) else: set_job(job_id, c) set_job_args(job_id, all_args) publish('job-defined', job_id=job_id) assert job_exists(job_id) assert job_args_exists(job_id) return Promise(job_id)