def expand_job_list_token(token):
''' Parses a token (string). Returns a generator of jobs.
Raises UserError, CompmakeSyntaxError '''
assert isinstance(token, str)
if token.find('*') > -1:
return expand_wildcard(token, all_jobs())
elif is_alias(token):
return eval_alias(token)
elif token.endswith('()'):
return list_matching_functions(token)
#raise UserError('Syntax reserved but not used yet. ("%s")' % token)
else:
# interpret as a job id
job_id = token
if not job_exists(job_id):
raise UserError('Job or expression "%s" not found ' % job_id)
return [job_id]
def comp(command, *args, **kwargs):
''' Main method to define a computation.
Extra arguments:
:arg:job_id: sets the job id (respects job_prefix)
:arg:extra_dep: extra dependencies (not passed as arguments)
'''
if compmake.compmake_status == compmake_status_slave:
return None
# Check that this is a pickable function
try:
pickle.dumps(command)
except:
msg = ('Cannot pickle %r. Make sure it is not a lambda function or a '
'nested function. (This is a limitation of Python)' % command)
raise SerializationError(msg)
args = list(args) # args is a non iterable tuple
# Get job id from arguments
job_id_key = 'job_id'
if job_id_key in kwargs:
# make sure that command does not have itself a job_id key
#available = command.func_code.co_varnames
argspec = inspect.getargspec(command)
if job_id_key in argspec.args:
msg = ("You cannot define the job id in this way because 'job_id' "
"is already a parameter of this function.")
raise UserError(msg)
job_id = kwargs[job_id_key]
if job_prefix:
job_id = '%s-%s' % (job_prefix, job_id)
del kwargs[job_id_key]
if job_id in jobs_defined_in_this_session:
raise UserError('Job %r already defined.' % job_id)
else:
job_id = generate_job_id(command)
jobs_defined_in_this_session.add(job_id)
if 'extra_dep' in kwargs:
extra_dep = collect_dependencies(kwargs['extra_dep'])
del kwargs['extra_dep']
else:
extra_dep = set()
children = collect_dependencies([args, kwargs])
children.update(extra_dep)
all_args = (command, args, kwargs)
command_desc = command.__name__
c = Job(job_id=job_id, children=list(children), command_desc=command_desc)
# TODO: check for loops
for child in children:
child_comp = get_job(child)
if not job_id in child_comp.parents:
child_comp.parents.append(job_id)
set_job(child, child_comp)
if job_exists(job_id):
# OK, this is going to be black magic.
# We want to load the previous job definition,
# however, by unpickling(), it will start
# __import__()ing the modules, perhaps
# even the one that is calling us.
# What happens, then is that it will try to
# add another time this computation recursively.
# What we do, is that we temporarely switch to
# slave mode, so that recursive calls to comp()
# are disabled.
if compmake_config.check_params: #@UndefinedVariable
old_status = compmake_status
set_compmake_status(compmake_status_slave)
old_computation = get_job(job_id)
set_compmake_status(old_status)
assert False, 'update for job_args'
same, reason = old_computation.same_computation(c)
if not same:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-redefined', job_id=job_id , reason=reason)
# XXX TODO clean the cache
else:
publish('job-already-defined', job_id=job_id)
else:
# We assume everything's ok
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
else:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
assert job_exists(job_id)
assert job_args_exists(job_id)
return Promise(job_id)
def check_job(job_id, context):
db = context.get_compmake_db()
job = get_job(job_id, db)
defined_by = job.defined_by
assert 'root' in defined_by
dparents = direct_parents(job_id, db=db)
all_parents = parents(job_id, db=db)
dchildren = direct_children(job_id, db=db)
all_children = children(job_id, db=db)
#print(job_id)
#print('d children: %s' % dchildren)
#print('all children: %s' % all_children)
errors = []
def e(msg):
errors.append(msg)
for defb in defined_by:
if defb == 'root': continue
if not job_exists(defb, db=db):
s = ('%r defined by %r but %r not existing.' %
(job_id, defined_by, defb))
e(s)
for dp in dparents:
if not job_exists(dp, db=db):
s = 'Direct parent %r of %r does not exist.' % (dp, job_id)
e(s)
else:
if not job_id in direct_children(dp, db=db):
s = '%s thinks %s is its direct parent;' % (job_id, dp)
s += 'but %s does not think %s is its direct child' % (dp,
job_id)
e(s)
for ap in all_parents:
if not job_exists(ap, db=db):
s = 'Parent %r of %r does not exist.' % (ap, job_id)
e(s)
else:
if not job_id in children(ap, db=db):
e('%s is parent but no child relation' % ap)
for dc in dchildren:
if not job_exists(dc, db=db):
s = 'Direct child %r of %r does not exist.' % (dc, job_id)
e(s)
else:
if not job_id in direct_parents(dc, db=db):
e('%s is direct child but no direct_parent relation' % dc)
for ac in all_children:
if not job_exists(ac, db=db):
s = 'A child %r of %r does not exist.' % (ac, job_id)
e(s)
else:
if not job_id in parents(ac, db=db):
e('%s is direct child but no parent relation' % ac)
if errors:
s = ('Inconsistencies for %s:\n' % job_id)
s += '\n'.join('- %s' % msg for msg in errors)
error(s)
return False, errors
else:
return True, []
def check_job(job_id, context):
db = context.get_compmake_db()
job = get_job(job_id, db)
defined_by = job.defined_by
assert 'root' in defined_by
dparents = direct_parents(job_id, db=db)
all_parents = parents(job_id, db=db)
dchildren = direct_children(job_id, db=db)
all_children = children(job_id, db=db)
#print(job_id)
#print('d children: %s' % dchildren)
#print('all children: %s' % all_children)
errors = []
def e(msg):
errors.append(msg)
for defb in defined_by:
if defb == 'root': continue
if not job_exists(defb, db=db):
s = ('%r defined by %r but %r not existing.'
%(job_id, defined_by, defb))
e(s)
for dp in dparents:
if not job_exists(dp, db=db):
s = 'Direct parent %r of %r does not exist.' % (dp, job_id)
e(s)
else:
if not job_id in direct_children(dp, db=db):
s = '%s thinks %s is its direct parent;' % (job_id, dp)
s += 'but %s does not think %s is its direct child' % (dp, job_id)
e(s)
for ap in all_parents:
if not job_exists(ap, db=db):
s = 'Parent %r of %r does not exist.' % (ap, job_id)
e(s)
else:
if not job_id in children(ap, db=db):
e('%s is parent but no child relation' % ap)
for dc in dchildren:
if not job_exists(dc, db=db):
s = 'Direct child %r of %r does not exist.' % (dc, job_id)
e(s)
else:
if not job_id in direct_parents(dc, db=db):
e('%s is direct child but no direct_parent relation' % dc)
for ac in all_children:
if not job_exists(ac, db=db):
s = 'A child %r of %r does not exist.' % (ac, job_id)
e(s)
else:
if not job_id in parents(ac, db=db):
e('%s is direct child but no parent relation' % ac)
if errors:
s = ('Inconsistencies for %s:\n' % job_id)
s += '\n'.join('- %s' % msg for msg in errors)
error(s)
return False, errors
else:
return True, []
