def run_quickapp(self, qapp, cmd: str):
args = ['-o', self.root0, '-c', cmd, '--compress']
self.assertEqual(0, quickapp_main(qapp, args, sys_exit=False))
# tell the context that it's all good
jobs = all_jobs(self.db)
self.cc.reset_jobs_defined_in_this_session(jobs)
def clean_other_jobs():
''' Cleans jobs not defined in the session '''
if compmake.compmake_status == compmake_status_slave:
return
from compmake.ui.console import ask_question
answers = {'a':'a', 'n':'n', 'y':'y', 'N':'N'}
clean_all = False
for job_id in all_jobs(force_db=True):
if not job_id in jobs_defined_in_this_session:
if not clean_all:
answer = ask_question(
"Found spurious job %s; cleaning? [y]es, [a]ll, [n]o, [N]one " \
% job_id, allowed=answers)
if answer == 'n':
continue
if answer == 'N':
break
if answer == 'a':
clean_all = True
clean_target(job_id)
delete_job(job_id)
def check_consistency(args, context, cq,
raise_if_error=False): # @ReservedAssignment
""" Checks in the DB that the relations between jobs are consistent. """
db = context.get_compmake_db()
# Do not use cq
if not args:
job_list = all_jobs(db=db)
else:
job_list = parse_job_list(args, context=context)
job_list = list(job_list)
#print('Checking consistency of %d jobs.' % len(job_list))
errors = {}
for job_id in job_list:
try:
ok, reasons = check_job(job_id, context)
if not ok:
errors[job_id] = reasons
except CompmakeBug as e:
errors[job_id] = ['bug: %s' % e]
if errors:
msg = "Inconsistency with %d jobs:" % len(errors)
for job_id, es in errors.items():
msg += '\n- job %r:\n%s' % (job_id, '\n'.join(es))
if raise_if_error:
raise CompmakeBug(msg)
else:
error(msg)
return 0
def list_ready_jobs():
''' Returns a list of jobs that can be done now,
as their dependencies are up-to-date. '''
from compmake.jobs.uptodate import dependencies_up_to_date
for job_id in all_jobs():
if dependencies_up_to_date(job_id):
yield job_id
def run_quickapp(self, qapp, cmd):
args = ['-o', self.root0, '-c', cmd]
self.assertEqual(0, quickapp_main(qapp, args, sys_exit=False))
# tell the context that it's all good
jobs = all_jobs(self.db)
self.cc.reset_jobs_defined_in_this_session(jobs)
def stats(args):
'''Displays a coarse summary of the jobs state. '''
if not args:
job_list = all_jobs()
else:
job_list = parse_job_list(args)
display_stats(job_list)
def tab_completion2(text, state):
available = get_commands().keys()
available.extend(list(all_jobs())) # give it a list
matches = sorted(x for x in available if x.startswith(text))
try:
response = matches[state]
except IndexError:
response = None
return response
def console_starting(event): #@UnusedVariable
# starting console
print "%s %s -- ``%s,, -- %s " % (
colored('Compmake', attrs=['bold']),
version, banner, banner2)
print "Welcome to the compmake console. " + \
"(write 'help' for a list of commands)"
njobs = len(list(all_jobs()))
print("%d jobs loaded; using namespace '%s'." % (njobs, get_namespace()))
def eval_ops(ops):
''' Evaluates an expression.
ops: list of strings and int representing operators '''
assert isinstance(ops, list)
def list_split(l, index):
''' Splits a list in two '''
return l[0:index], l[index + 1:]
# The sequence of the following operations
# defines the associativity rules
# in > except > not
if Operators.INTERSECTION in ops:
left, right = list_split(ops, ops.index(Operators.INTERSECTION))
if not left or not right:
raise CompmakeSyntaxError(''' INTERSECTION requires only a right \
argument. Interpreting "%s" INTERSECTION "%s". ''' %
(' '.join(left), ' '.join(right)))
left = eval_ops(left)
right = set(eval_ops(right))
for x in left:
if x in right:
yield x
elif Operators.DIFFERENCE in ops:
left, right = list_split(ops, ops.index(Operators.DIFFERENCE))
if not left or not right:
raise CompmakeSyntaxError(''' EXCEPT requires a left and right \
argument. Interpreting "%s" EXCEPT "%s". ''' %
(' '.join(left), ' '.join(right)))
left = eval_ops(left)
right = set(eval_ops(right))
for x in left:
if x not in right:
yield x
elif Operators.NOT in ops:
left, right = list_split(ops, ops.index(Operators.NOT))
if left or not right: # forbid left, require right
raise CompmakeSyntaxError(\
''' NOT requires only a right argument. Interpreting "%s" NOT "%s". ''' %
(' '.join(left), ' '.join(right)))
all = all_jobs()
right = set(eval_ops(right))
for x in all:
if x not in right:
yield x
else:
# no operators: simple list
# cannot do this anymore, now it's a generator. assert_list_of_strings(ops)
for x in expand_job_list_tokens(ops):
yield x
def list(args):
'''Lists the status of the selected targets (or all targets \
if not specified).
If only one job is specified, then it is listed in more detail. '''
if not args:
job_list = all_jobs()
else:
job_list = parse_job_list(args)
list_jobs(job_list)
def junit_xml(compmake_db):
jobs = list(all_jobs(compmake_db))
logger.info('Loaded %d jobs' % len(jobs))
if len(jobs) < 10:
logger.error('too few jobs')
sys.exit(128)
test_cases = []
for job_id in jobs:
tc = junit_test_case_from_compmake(compmake_db, job_id)
test_cases.append(tc)
ts = TestSuite("comptests_test_suite", test_cases)
return TestSuite.to_xml_string([ts])
def list_matching_functions(token):
assert token.endswith('()')
if len(token) < 3:
raise UserError('Malformed token "%s".' % token)
function_id = token[:-2]
num_matches = 0
for job_id in all_jobs():
if function_id.lower() == get_job(job_id).command.__name__.lower():
yield job_id
num_matches += 1
if num_matches == 0:
raise UserError('Could not find matches for function "%s()".' %
function_id)
def expand_job_list_token(token):
''' Parses a token (string). Returns a generator of jobs.
Raises UserError, CompmakeSyntaxError '''
assert isinstance(token, str)
if token.find('*') > -1:
return expand_wildcard(token, all_jobs())
elif is_alias(token):
return eval_alias(token)
elif token.endswith('()'):
return list_matching_functions(token)
#raise UserError('Syntax reserved but not used yet. ("%s")' % token)
else:
# interpret as a job id
job_id = token
if not job_exists(job_id):
raise UserError('Job or expression "%s" not found ' % job_id)
return [job_id]
def junit_xml(compmake_db):
from junit_xml import TestSuite
jobs = list(all_jobs(compmake_db))
logger.info('Loaded %d jobs' % len(jobs))
N = 10
if len(jobs) < N:
logger.error('too few jobs (I expect at least %s)' % N)
sys.exit(128)
test_cases = []
for job_id in jobs:
tc = junit_test_case_from_compmake(compmake_db, job_id)
test_cases.append(tc)
ts = TestSuite("comptests_test_suite", test_cases)
res = TestSuite.to_xml_string([ts])
check_isinstance(res, six.text_type)
return res
def clean(job_list):
'''Cleans the result of the selected computation \
(or everything is nothing specified). '''
job_list = list(job_list)
if not job_list:
job_list = list(all_jobs())
if not job_list:
return
from compmake.ui.console import ask_question
if get_compmake_status() == compmake_status_interactive:
question = "Should I clean %d jobs? [y/n] " % len(job_list)
answer = ask_question(question)
if not answer:
info('Not cleaned.')
return
for job_id in job_list:
clean_target(job_id)
def comptest_to_junit_main():
args = sys.argv[1:]
if not args:
msg = 'Require the path to a Compmake DB.'
raise UserError(msg)
dirname = args[0]
# try compressed
try:
db = StorageFilesystem(dirname, compress=True)
except Exception:
db = StorageFilesystem(dirname, compress=False)
jobs = list(all_jobs(db))
if not jobs:
msg = 'Could not find any job, compressed or not.'
logger.error(msg)
sys.exit(1)
s = junit_xml(db)
check_isinstance(s, six.text_type)
s = s.encode('utf8')
sys.stdout.buffer.write(s)
def check_consistency(args,
context,
cq,
raise_if_error=False): # @ReservedAssignment
""" Checks in the DB that the relations between jobs are consistent. """
db = context.get_compmake_db()
# Do not use cq
if not args:
job_list = all_jobs(db=db)
else:
job_list = parse_job_list(args, context=context)
job_list = list(job_list)
#print('Checking consistency of %d jobs.' % len(job_list))
errors = {}
for job_id in job_list:
try:
ok, reasons = check_job(job_id, context)
if not ok:
errors[job_id] = reasons
except CompmakeBug as e:
errors[job_id] = ['bug: %s' % e]
if errors:
msg = "Inconsistency with %d jobs:" % len(errors)
for job_id, es in errors.items():
msg += '\n- job %r:\n%s' % (job_id, '\n'.join(es))
if raise_if_error:
raise CompmakeBug(msg)
else:
error(msg)
return 0
def all_jobs(self, root):
""" Returns the list of jobs corresponding to the given expression. """
db = StorageFilesystem(root, compress=True)
return sorted(list(all_jobs(db)))
def list_bottom_jobs():
''' Returns a list of jobs that do not depend on anything else. '''
from compmake.jobs.queries import direct_children
for job_id in all_jobs():
if not direct_children(job_id):
yield job_id
def list_top_jobs():
''' Returns a list of jobs that are top-level targets. '''
from compmake.jobs.queries import direct_parents
for job_id in all_jobs():
if not direct_parents(job_id):
yield job_id
def list_todo_jobs():
''' Returns a list of jobs that haven't been DONE. '''
for job_id in all_jobs():
if get_job_cache(job_id).state != Cache.DONE:
yield job_id
def list_jobs_with_state(state):
''' Returns a list of jobs in the given state. '''
for job_id in all_jobs():
if get_job_cache(job_id).state == state:
yield job_id
def gantt(job_list, context, filename='gantt.html'):
"""
"""
db = context.get_compmake_db()
if not job_list:
# job_list = list(top_targets(db))
job_list = all_jobs(db)
# plus all the jobs that were defined by them
job_list = set(job_list)
# job_list.update(definition_closure(job_list, db))
from networkx import DiGraph
G = DiGraph()
cq = CacheQueryDB(db)
for job_id in job_list:
cache = cq.get_job_cache(job_id)
length = cache.int_make.get_cputime_used()
attr_dict = dict(cache=cache, length=length)
G.add_node(job_id, **attr_dict)
dependencies = cq.direct_children(job_id)
for c in dependencies:
G.add_edge(c, job_id)
defined = cq.jobs_defined(job_id)
for c in defined:
G.add_edge(job_id, c)
order = topological_sort(G)
for job_id in order:
length = G.node[job_id]['length']
pre = list(G.predecessors(job_id))
# print('%s pred %s' % (job_id, pre))
if not pre:
T0 = 0
G.node[job_id]['CP'] = None
else:
# find predecessor with highest T1
import numpy as np
T1s = list(G.node[_]['T1'] for _ in pre)
i = np.argmax(T1s)
T0 = T1s[i]
G.node[job_id]['CP'] = pre[i]
T1 = T0 + length
G.node[job_id]['T0'] = T0
G.node[job_id]['T1'] = T1
G.node[job_id]['critical'] = False
sg_ideal = SimpleGantt()
by_ideal_completion = sorted(order, key=lambda _: G.node[_]['T1'])
last = by_ideal_completion[-1]
path = []
while last is not None:
path.append(last)
G.node[last]['critical'] = True
last = G.node[last]['CP']
print('Critical path:')
for job_id in reversed(path):
length = G.node[job_id]['length']
print('- %.1f s %s' % (length, job_id))
for job_id in by_ideal_completion:
T0 = G.node[job_id]['T0']
T1 = G.node[job_id]['T1']
# length = G.node[job_id]['length']
dependencies = list(G.predecessors(job_id))
# cache = G.node[job_id]['cache']
periods = OrderedDict()
periods['ideal'] = (T0, T1)
critical = G.node[job_id]['critical']
sg_ideal.add_job(job_id, dependencies, periods=periods, critical=critical)
sg_actual = SimpleGantt()
order_actual = sorted(order, key=lambda _: G.node[_]['cache'].int_make.t0)
for job_id in order_actual:
cache = G.node[job_id]['cache']
critical = G.node[job_id]['critical']
dependencies = list(G.predecessors(job_id))
periods = OrderedDict()
periods['make'] = cache.int_make.walltime_interval()
sg_actual.add_job(job_id, dependencies, periods=periods, critical=critical)
sg_actual_detailed = SimpleGantt()
for job_id in order_actual:
cache = G.node[job_id]['cache']
critical = G.node[job_id]['critical']
periods = OrderedDict()
periods['load'] = cache.int_load_results.walltime_interval()
periods['compute'] = cache.int_compute.walltime_interval()
periods['gc'] = cache.int_gc.walltime_interval()
periods['save'] = cache.int_save_results.walltime_interval()
assert periods['load'][1] <= periods['compute'][0]
assert periods['compute'][1] <= periods['save'][0]
sg_actual_detailed.add_job(job_id, dependencies, periods=periods, critical=critical)
html = ''
width_pixels = 1000
if True:
html += '\n<h1>Actual</h1>'
html += sg_actual.as_html(width_pixels)
if True:
html += '\n<h1>Actual (detailed)</h1>'
html += sg_actual_detailed.as_html(width_pixels)
if True:
html += '\n<h1>Ideal</h1>'
html += sg_ideal.as_html(width_pixels)
html += '''
<style>
tr:hover td:first-child {
}
td:first-child {
font-size: 10px;
}
td:nth-child(2) {
background-color: grey;
width: %spx;
}
.compute, .make, .save, .load, .ideal, .gc {
outline: solid 1px black;
float: left;
clear: left;
}
.compute {
background-color: red;
}
.make {
background-color: blue;
}
.gc {
background-color: brown;
}
.save {
background-color: green;
}
.load {
background-color: yellow;
}
.ideal {
background-color: magenta;
}
.critical {
/* outline: solid 2px red !important; */
background-color: pink;
}
</style>
''' % width_pixels
with open(filename, 'w') as f:
f.write(html)
print('written to %s' % filename)
def top_targets():
""" Returns a list of all jobs which are not needed by anybody """
return [x for x in all_jobs() if not direct_parents(x)]
def bottom_targets():
""" Returns a list of all jobs with no dependencies """
return [x for x in all_jobs() if not direct_children(x)]
def all_jobs(self, root):
""" Returns the list of jobs corresponding to the given expression. """
db = StorageFilesystem(root, compress=True)
return sorted(list(all_jobs(db)))
