def testDep2(self): ''' Testing advanced dependencies discovery (double) ''' cf1 = comp(f1) cf2 = comp(f2, cf1, cf1) self.assertTrue(cf1.job_id in direct_children(cf2.job_id)) self.assertEqual(1, len(direct_children(cf2.job_id))) self.assertEqual(1, len(direct_parents(cf1.job_id)))
def list_job_detail(job_id): #computation = get_computation(job_id) cache = get_job_cache(job_id) parents = direct_parents(job_id) children = direct_children(job_id) up, reason = up_to_date(job_id) red = lambda x: colored(x, 'red') bold = lambda x: colored(rjust(x + ' ', 15), attrs=['bold']) try: print bold('Job ID:') + '%s' % job_id print bold('Status:') + '%s' % Cache.state2desc[cache.state] print bold('Uptodate:') + '%s (%s)' % (up, reason) print bold('Children:') + '%s' % ', '.join(children) print bold('Parents:') + '%s' % ', '.join(parents) if cache.state == Cache.DONE and cache.done_iterations > 1: print bold('Iterations:') + '%s' % cache.done_iterations print bold('Wall Time:') + '%.4f s' % cache.walltime_used print bold('CPU Time:') + '%.4f s' % cache.cputime_used print bold('Host:') + '%s' % cache.host if cache.state == Cache.IN_PROGRESS: print bold('Progress:') + '%s/%s' % \ (cache.iterations_in_progress, cache.iterations_goal) if cache.state == Cache.FAILED: print red(cache.exception) print red(cache.backtrace) def display_with_prefix(buffer, prefix, transform=lambda x:x, out=sys.stdout): for line in buffer.split('\n'): out.write('%s%s\n' % (prefix, transform(line))) if cache.captured_stdout: print "-----> captured stdout <-----" display_with_prefix(cache.captured_stdout, prefix='|', transform=lambda x: colored(x, attrs=['dark'])) if cache.captured_stderr: print "-----> captured stderr <-----" display_with_prefix(cache.captured_stdout, prefix='|', transform=lambda x: colored(x, attrs=['dark'])) except AttributeError: pass
def direct_uptodate_deps(job_id, db): """ Returns all direct 'dependencies' of this job: the jobs that are children (arguemnts) plus the job that defined it (if not root). """ from compmake.jobs.queries import direct_children dependencies = direct_children(job_id, db) # plus jobs that defined it from compmake.jobs.storage import get_job defined_by = get_job(job_id, db).defined_by last = defined_by[-1] if last != 'root': dependencies.add(last) return dependencies
def list_bottom_jobs(): ''' Returns a list of jobs that do not depend on anything else. ''' from compmake.jobs.queries import direct_children for job_id in all_jobs(): if not direct_children(job_id): yield job_id
def direct_children(self, job_id): from compmake.jobs.queries import direct_children return direct_children(job_id, db=self.db)
def graph(job_list, filename='compmake', compact=0, filter='dot', format='png'): '''Creates a graph of the given targets and dependencies graph filename=filename compact=0,1 format=png,... Params: filename: name of generated filename in the dot format compact=0: whether to include the job names in the nodes filter=[dot,circo,twopi,...] which algorithm to use to arrange the nodes. This depends on the topology of your computation. The default is 'dot' (hierarchy top-bottom). format=[png,...] The output file format. ''' if not job_list: job_list = top_targets() job_list = tree(job_list) try: import gvgen #@UnresolvedImport except: gvgen_url = 'http://software.inl.fr/trac/wiki/GvGen' raise UserError('To use the "graph" command' + ' you have to install the "gvgen" package from %s' % gvgen_url) graph = gvgen.GvGen() state2color = { Cache.NOT_STARTED: 'grey', Cache.IN_PROGRESS: 'yellow', Cache.MORE_REQUESTED: 'blue', Cache.FAILED: 'red', Cache.DONE: 'green' } job2node = {} for job_id in job_list: if int(compact): job2node[job_id] = graph.newItem("") else: job2node[job_id] = graph.newItem(job_id) cache = get_job_cache(job_id) graph.styleAppend(job_id, "style", "filled") graph.styleAppend(job_id, "fillcolor", state2color[cache.state]) graph.styleApply(job_id, job2node[job_id]) for job_id in job_list: #c = get_computation(job_id) #children_id = [x.job_id for x in c.depends] for child in direct_children(job_id): graph.newLink(job2node[job_id], job2node[child]) # TODO: add check? with open(filename, 'w') as f: graph.dot(f) output = filename + '.' + format cmd_line = '%s %s -T%s -o%s' % (filter, filename, format, output) try: os.system(cmd_line) except: raise UserError("Could not run dot (cmdline='%s')\ Make sure graphviz is installed" % cmd_line) # XXX maybe not UserError info("Written output on files %s, %s." % (filename, output))
def testDep3(self): ''' Testing advanced dependencies discovery in dicts''' cf1 = comp(f1) cf2 = comp(f2, [1, {'ciao': cf1}]) self.assertTrue(cf1.job_id in direct_children(cf2.job_id)) self.assertTrue(cf2.job_id in direct_parents(cf1.job_id))
def testDep(self): ''' Testing advanced dependencies discovery ''' cf1 = comp(f1) cf2 = comp(f2, cf1) self.assertTrue(cf1.job_id in direct_children(cf2.job_id)) self.assertTrue(cf2.job_id in direct_parents(cf1.job_id))