def testDep2(self):
''' Testing advanced dependencies discovery (double) '''
cf1 = comp(f1)
cf2 = comp(f2, cf1, cf1)
self.assertTrue(cf1.job_id in direct_children(cf2.job_id))
self.assertEqual(1, len(direct_children(cf2.job_id)))
self.assertEqual(1, len(direct_parents(cf1.job_id)))
def list_job_detail(job_id):
#computation = get_computation(job_id)
cache = get_job_cache(job_id)
parents = direct_parents(job_id)
children = direct_children(job_id)
up, reason = up_to_date(job_id)
red = lambda x: colored(x, 'red')
bold = lambda x: colored(rjust(x + ' ', 15), attrs=['bold'])
try:
print bold('Job ID:') + '%s' % job_id
print bold('Status:') + '%s' % Cache.state2desc[cache.state]
print bold('Uptodate:') + '%s (%s)' % (up, reason)
print bold('Children:') + '%s' % ', '.join(children)
print bold('Parents:') + '%s' % ', '.join(parents)
if cache.state == Cache.DONE and cache.done_iterations > 1:
print bold('Iterations:') + '%s' % cache.done_iterations
print bold('Wall Time:') + '%.4f s' % cache.walltime_used
print bold('CPU Time:') + '%.4f s' % cache.cputime_used
print bold('Host:') + '%s' % cache.host
if cache.state == Cache.IN_PROGRESS:
print bold('Progress:') + '%s/%s' % \
(cache.iterations_in_progress, cache.iterations_goal)
if cache.state == Cache.FAILED:
print red(cache.exception)
print red(cache.backtrace)
def display_with_prefix(buffer, prefix,
transform=lambda x:x, out=sys.stdout):
for line in buffer.split('\n'):
out.write('%s%s\n' % (prefix, transform(line)))
if cache.captured_stdout:
print "-----> captured stdout <-----"
display_with_prefix(cache.captured_stdout, prefix='|',
transform=lambda x: colored(x, attrs=['dark']))
if cache.captured_stderr:
print "-----> captured stderr <-----"
display_with_prefix(cache.captured_stdout, prefix='|',
transform=lambda x: colored(x, attrs=['dark']))
except AttributeError:
pass
def direct_uptodate_deps(job_id, db):
""" Returns all direct 'dependencies' of this job:
the jobs that are children (arguemnts)
plus the job that defined it (if not root).
"""
from compmake.jobs.queries import direct_children
dependencies = direct_children(job_id, db)
# plus jobs that defined it
from compmake.jobs.storage import get_job
defined_by = get_job(job_id, db).defined_by
last = defined_by[-1]
if last != 'root':
dependencies.add(last)
return dependencies
def direct_uptodate_deps(job_id, db):
""" Returns all direct 'dependencies' of this job:
the jobs that are children (arguemnts)
plus the job that defined it (if not root).
"""
from compmake.jobs.queries import direct_children
dependencies = direct_children(job_id, db)
# plus jobs that defined it
from compmake.jobs.storage import get_job
defined_by = get_job(job_id, db).defined_by
last = defined_by[-1]
if last != 'root':
dependencies.add(last)
return dependencies
def list_bottom_jobs():
''' Returns a list of jobs that do not depend on anything else. '''
from compmake.jobs.queries import direct_children
for job_id in all_jobs():
if not direct_children(job_id):
yield job_id
def direct_children(self, job_id):
from compmake.jobs.queries import direct_children
return direct_children(job_id, db=self.db)
def graph(job_list, filename='compmake', compact=0,
filter='dot', format='png'):
'''Creates a graph of the given targets and dependencies
graph filename=filename compact=0,1 format=png,...
Params:
filename: name of generated filename in the dot format
compact=0: whether to include the job names in the nodes
filter=[dot,circo,twopi,...] which algorithm to use to arrange
the nodes. This depends on the topology of your
computation. The default is 'dot'
(hierarchy top-bottom).
format=[png,...] The output file format.
'''
if not job_list:
job_list = top_targets()
job_list = tree(job_list)
try:
import gvgen #@UnresolvedImport
except:
gvgen_url = 'http://software.inl.fr/trac/wiki/GvGen'
raise UserError('To use the "graph" command' +
' you have to install the "gvgen" package from %s' %
gvgen_url)
graph = gvgen.GvGen()
state2color = {
Cache.NOT_STARTED: 'grey',
Cache.IN_PROGRESS: 'yellow',
Cache.MORE_REQUESTED: 'blue',
Cache.FAILED: 'red',
Cache.DONE: 'green'
}
job2node = {}
for job_id in job_list:
if int(compact):
job2node[job_id] = graph.newItem("")
else:
job2node[job_id] = graph.newItem(job_id)
cache = get_job_cache(job_id)
graph.styleAppend(job_id, "style", "filled")
graph.styleAppend(job_id, "fillcolor", state2color[cache.state])
graph.styleApply(job_id, job2node[job_id])
for job_id in job_list:
#c = get_computation(job_id)
#children_id = [x.job_id for x in c.depends]
for child in direct_children(job_id):
graph.newLink(job2node[job_id], job2node[child])
# TODO: add check?
with open(filename, 'w') as f:
graph.dot(f)
output = filename + '.' + format
cmd_line = '%s %s -T%s -o%s' % (filter, filename, format, output)
try:
os.system(cmd_line)
except:
raise UserError("Could not run dot (cmdline='%s')\
Make sure graphviz is installed" % cmd_line) # XXX maybe not UserError
info("Written output on files %s, %s." % (filename, output))
def testDep3(self):
''' Testing advanced dependencies discovery in dicts'''
cf1 = comp(f1)
cf2 = comp(f2, [1, {'ciao': cf1}])
self.assertTrue(cf1.job_id in direct_children(cf2.job_id))
self.assertTrue(cf2.job_id in direct_parents(cf1.job_id))
def testDep(self):
''' Testing advanced dependencies discovery '''
cf1 = comp(f1)
cf2 = comp(f2, cf1)
self.assertTrue(cf1.job_id in direct_children(cf2.job_id))
self.assertTrue(cf2.job_id in direct_parents(cf1.job_id))
def direct_children(self, job_id):
from compmake.jobs.queries import direct_children
return direct_children(job_id, db=self.db)