def clean_cache_relations(job_id, db):
#print('cleaning cache relations for %r ' % job_id)
# for all jobs that were done
cache = get_job_cache(job_id, db)
if cache.state == Cache.DONE:
for parent in direct_parents(job_id, db):
parent_job = get_job(parent, db)
#print(' parent %r has dynamic %s' % (parent, parent_job.dynamic_children))
if not job_id in parent_job.dynamic_children:
#print(' skipping parent %r ' % parent)
continue
else:
dynamic_children = parent_job.dynamic_children[job_id]
#print(' dynamic_children %s' % parent_job.dynamic_children)
#print(' children %s' % parent_job.children)
del parent_job.dynamic_children[job_id]
parent_job.children = parent_job.children - dynamic_children
set_job(parent, parent_job, db)
def comp(command, *args, **kwargs):
''' Main method to define a computation.
Extra arguments:
:arg:job_id: sets the job id (respects job_prefix)
:arg:extra_dep: extra dependencies (not passed as arguments)
'''
if compmake.compmake_status == compmake_status_slave:
return None
# Check that this is a pickable function
try:
pickle.dumps(command)
except:
msg = ('Cannot pickle %r. Make sure it is not a lambda function or a '
'nested function. (This is a limitation of Python)' % command)
raise SerializationError(msg)
args = list(args) # args is a non iterable tuple
# Get job id from arguments
job_id_key = 'job_id'
if job_id_key in kwargs:
# make sure that command does not have itself a job_id key
#available = command.func_code.co_varnames
argspec = inspect.getargspec(command)
if job_id_key in argspec.args:
msg = ("You cannot define the job id in this way because 'job_id' "
"is already a parameter of this function.")
raise UserError(msg)
job_id = kwargs[job_id_key]
if job_prefix:
job_id = '%s-%s' % (job_prefix, job_id)
del kwargs[job_id_key]
if job_id in jobs_defined_in_this_session:
raise UserError('Job %r already defined.' % job_id)
else:
job_id = generate_job_id(command)
jobs_defined_in_this_session.add(job_id)
if 'extra_dep' in kwargs:
extra_dep = collect_dependencies(kwargs['extra_dep'])
del kwargs['extra_dep']
else:
extra_dep = set()
children = collect_dependencies([args, kwargs])
children.update(extra_dep)
all_args = (command, args, kwargs)
command_desc = command.__name__
c = Job(job_id=job_id, children=list(children), command_desc=command_desc)
# TODO: check for loops
for child in children:
child_comp = get_job(child)
if not job_id in child_comp.parents:
child_comp.parents.append(job_id)
set_job(child, child_comp)
if job_exists(job_id):
# OK, this is going to be black magic.
# We want to load the previous job definition,
# however, by unpickling(), it will start
# __import__()ing the modules, perhaps
# even the one that is calling us.
# What happens, then is that it will try to
# add another time this computation recursively.
# What we do, is that we temporarely switch to
# slave mode, so that recursive calls to comp()
# are disabled.
if compmake_config.check_params: #@UndefinedVariable
old_status = compmake_status
set_compmake_status(compmake_status_slave)
old_computation = get_job(job_id)
set_compmake_status(old_status)
assert False, 'update for job_args'
same, reason = old_computation.same_computation(c)
if not same:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-redefined', job_id=job_id , reason=reason)
# XXX TODO clean the cache
else:
publish('job-already-defined', job_id=job_id)
else:
# We assume everything's ok
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
else:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
assert job_exists(job_id)
assert job_args_exists(job_id)
return Promise(job_id)
