def expand_job_list_token(token): ''' Parses a token (string). Returns a generator of jobs. Raises UserError, CompmakeSyntaxError ''' assert isinstance(token, str) if token.find('*') > -1: return expand_wildcard(token, all_jobs()) elif is_alias(token): return eval_alias(token) elif token.endswith('()'): return list_matching_functions(token) #raise UserError('Syntax reserved but not used yet. ("%s")' % token) else: # interpret as a job id job_id = token if not job_exists(job_id): raise UserError('Job or expression "%s" not found ' % job_id) return [job_id]
def comp(command, *args, **kwargs): ''' Main method to define a computation. Extra arguments: :arg:job_id: sets the job id (respects job_prefix) :arg:extra_dep: extra dependencies (not passed as arguments) ''' if compmake.compmake_status == compmake_status_slave: return None # Check that this is a pickable function try: pickle.dumps(command) except: msg = ('Cannot pickle %r. Make sure it is not a lambda function or a ' 'nested function. (This is a limitation of Python)' % command) raise SerializationError(msg) args = list(args) # args is a non iterable tuple # Get job id from arguments job_id_key = 'job_id' if job_id_key in kwargs: # make sure that command does not have itself a job_id key #available = command.func_code.co_varnames argspec = inspect.getargspec(command) if job_id_key in argspec.args: msg = ("You cannot define the job id in this way because 'job_id' " "is already a parameter of this function.") raise UserError(msg) job_id = kwargs[job_id_key] if job_prefix: job_id = '%s-%s' % (job_prefix, job_id) del kwargs[job_id_key] if job_id in jobs_defined_in_this_session: raise UserError('Job %r already defined.' % job_id) else: job_id = generate_job_id(command) jobs_defined_in_this_session.add(job_id) if 'extra_dep' in kwargs: extra_dep = collect_dependencies(kwargs['extra_dep']) del kwargs['extra_dep'] else: extra_dep = set() children = collect_dependencies([args, kwargs]) children.update(extra_dep) all_args = (command, args, kwargs) command_desc = command.__name__ c = Job(job_id=job_id, children=list(children), command_desc=command_desc) # TODO: check for loops for child in children: child_comp = get_job(child) if not job_id in child_comp.parents: child_comp.parents.append(job_id) set_job(child, child_comp) if job_exists(job_id): # OK, this is going to be black magic. # We want to load the previous job definition, # however, by unpickling(), it will start # __import__()ing the modules, perhaps # even the one that is calling us. # What happens, then is that it will try to # add another time this computation recursively. # What we do, is that we temporarely switch to # slave mode, so that recursive calls to comp() # are disabled. if compmake_config.check_params: #@UndefinedVariable old_status = compmake_status set_compmake_status(compmake_status_slave) old_computation = get_job(job_id) set_compmake_status(old_status) assert False, 'update for job_args' same, reason = old_computation.same_computation(c) if not same: set_job(job_id, c) set_job_args(job_id, all_args) publish('job-redefined', job_id=job_id , reason=reason) # XXX TODO clean the cache else: publish('job-already-defined', job_id=job_id) else: # We assume everything's ok set_job(job_id, c) set_job_args(job_id, all_args) publish('job-defined', job_id=job_id) else: set_job(job_id, c) set_job_args(job_id, all_args) publish('job-defined', job_id=job_id) assert job_exists(job_id) assert job_args_exists(job_id) return Promise(job_id)
def check_job(job_id, context): db = context.get_compmake_db() job = get_job(job_id, db) defined_by = job.defined_by assert 'root' in defined_by dparents = direct_parents(job_id, db=db) all_parents = parents(job_id, db=db) dchildren = direct_children(job_id, db=db) all_children = children(job_id, db=db) #print(job_id) #print('d children: %s' % dchildren) #print('all children: %s' % all_children) errors = [] def e(msg): errors.append(msg) for defb in defined_by: if defb == 'root': continue if not job_exists(defb, db=db): s = ('%r defined by %r but %r not existing.' % (job_id, defined_by, defb)) e(s) for dp in dparents: if not job_exists(dp, db=db): s = 'Direct parent %r of %r does not exist.' % (dp, job_id) e(s) else: if not job_id in direct_children(dp, db=db): s = '%s thinks %s is its direct parent;' % (job_id, dp) s += 'but %s does not think %s is its direct child' % (dp, job_id) e(s) for ap in all_parents: if not job_exists(ap, db=db): s = 'Parent %r of %r does not exist.' % (ap, job_id) e(s) else: if not job_id in children(ap, db=db): e('%s is parent but no child relation' % ap) for dc in dchildren: if not job_exists(dc, db=db): s = 'Direct child %r of %r does not exist.' % (dc, job_id) e(s) else: if not job_id in direct_parents(dc, db=db): e('%s is direct child but no direct_parent relation' % dc) for ac in all_children: if not job_exists(ac, db=db): s = 'A child %r of %r does not exist.' % (ac, job_id) e(s) else: if not job_id in parents(ac, db=db): e('%s is direct child but no parent relation' % ac) if errors: s = ('Inconsistencies for %s:\n' % job_id) s += '\n'.join('- %s' % msg for msg in errors) error(s) return False, errors else: return True, []
def check_job(job_id, context): db = context.get_compmake_db() job = get_job(job_id, db) defined_by = job.defined_by assert 'root' in defined_by dparents = direct_parents(job_id, db=db) all_parents = parents(job_id, db=db) dchildren = direct_children(job_id, db=db) all_children = children(job_id, db=db) #print(job_id) #print('d children: %s' % dchildren) #print('all children: %s' % all_children) errors = [] def e(msg): errors.append(msg) for defb in defined_by: if defb == 'root': continue if not job_exists(defb, db=db): s = ('%r defined by %r but %r not existing.' %(job_id, defined_by, defb)) e(s) for dp in dparents: if not job_exists(dp, db=db): s = 'Direct parent %r of %r does not exist.' % (dp, job_id) e(s) else: if not job_id in direct_children(dp, db=db): s = '%s thinks %s is its direct parent;' % (job_id, dp) s += 'but %s does not think %s is its direct child' % (dp, job_id) e(s) for ap in all_parents: if not job_exists(ap, db=db): s = 'Parent %r of %r does not exist.' % (ap, job_id) e(s) else: if not job_id in children(ap, db=db): e('%s is parent but no child relation' % ap) for dc in dchildren: if not job_exists(dc, db=db): s = 'Direct child %r of %r does not exist.' % (dc, job_id) e(s) else: if not job_id in direct_parents(dc, db=db): e('%s is direct child but no direct_parent relation' % dc) for ac in all_children: if not job_exists(ac, db=db): s = 'A child %r of %r does not exist.' % (ac, job_id) e(s) else: if not job_id in parents(ac, db=db): e('%s is direct child but no parent relation' % ac) if errors: s = ('Inconsistencies for %s:\n' % job_id) s += '\n'.join('- %s' % msg for msg in errors) error(s) return False, errors else: return True, []