def learning_times_rlearn(outdir, learners, streams, nthreads, nrefines):
job_ids = CompmakeGlobalState.jobs_defined_in_this_session
id_learner, id_stream = learners[0], streams[0]
cputime_index_level_learn = np.ones((nthreads, nrefines)) * np.NaN
walltime_index_level_learn = np.ones((nthreads, nrefines)) * np.NaN
for i in range(nthreads):
for ref in range(nrefines):
search_id = 'learn-%s-%s-%sof%s-refined%s' % (
id_stream, id_learner, i + 1, nthreads, ref)
for job_id in job_ids:
if job_id == search_id:
job_cache = get_job_cache(job_id)
cputime_index_level_learn[i, ref] = job_cache.cputime_used
walltime_index_level_learn[i,
ref] = job_cache.walltime_used
print cputime_index_level_learn
print walltime_index_level_learn
cputime_index_level_summarize = np.ones((nthreads, nrefines)) * np.NaN
walltime_index_level_summarize = np.ones((nthreads, nrefines)) * np.NaN
for i in range(nthreads):
for ref in range(nrefines):
search_id = 'learn-%s-%s-%s-refined%s-summarize' % (
id_stream, id_learner, i, ref
) # bug in computation naming, should be i+1
print(search_id)
for job_id in job_ids:
if job_id == search_id:
job_cache = get_job_cache(job_id)
cputime_index_level_summarize[i,
ref] = job_cache.cputime_used
walltime_index_level_summarize[
i, ref] = job_cache.walltime_used
print cputime_index_level_summarize
print walltime_index_level_summarize
times = {
'cputime_index_level_learn': cputime_index_level_learn,
'walltime_index_level_learn': walltime_index_level_learn,
'cputime_index_level_summarize': cputime_index_level_summarize,
'walltime_index_level_summarize': walltime_index_level_summarize
}
# pdb.set_trace()
pickle.dump(times, open(os.path.join(outdir, 'times.pickle'), 'wb'))
def my_get_job_cache(job_id):
""" Gets the job cache, making sure it was done """
cache = get_job_cache(job_id)
if cache.state != Cache.DONE:
msg = 'The job %s was supposed to be finished: %s' % (job_id, cache)
raise Exception(msg)
return cache
def my_get_job_cache(context, the_job):
""" Gets the job cache, making sure it was done """
db = context.get_compmake_db()
cache = get_job_cache(the_job, db=db)
if cache.state != Cache.DONE:
msg = 'The job %s was supposed to be finished: %s' % (the_job, cache)
raise Exception(msg)
return cache
def learning_times_rlearn(outdir, learners, streams, nthreads, nrefines):
job_ids = CompmakeGlobalState.jobs_defined_in_this_session
id_learner, id_stream = learners[0], streams[0]
cputime_index_level_learn = np.ones((nthreads, nrefines)) * np.NaN
walltime_index_level_learn = np.ones((nthreads, nrefines)) * np.NaN
for i in range(nthreads):
for ref in range(nrefines):
search_id = 'learn-%s-%s-%sof%s-refined%s' % (id_stream, id_learner, i + 1, nthreads, ref)
for job_id in job_ids:
if job_id == search_id:
job_cache = get_job_cache(job_id)
cputime_index_level_learn[i, ref] = job_cache.cputime_used
walltime_index_level_learn[i, ref] = job_cache.walltime_used
print cputime_index_level_learn
print walltime_index_level_learn
cputime_index_level_summarize = np.ones((nthreads, nrefines)) * np.NaN
walltime_index_level_summarize = np.ones((nthreads, nrefines)) * np.NaN
for i in range(nthreads):
for ref in range(nrefines):
search_id = 'learn-%s-%s-%s-refined%s-summarize' % (id_stream, id_learner, i , ref) # bug in computation naming, should be i+1
print(search_id)
for job_id in job_ids:
if job_id == search_id:
job_cache = get_job_cache(job_id)
cputime_index_level_summarize[i, ref] = job_cache.cputime_used
walltime_index_level_summarize[i, ref] = job_cache.walltime_used
print cputime_index_level_summarize
print walltime_index_level_summarize
times = {'cputime_index_level_learn': cputime_index_level_learn,
'walltime_index_level_learn': walltime_index_level_learn,
'cputime_index_level_summarize': cputime_index_level_summarize,
'walltime_index_level_summarize': walltime_index_level_summarize }
# pdb.set_trace()
pickle.dump(times, open(os.path.join(outdir, 'times.pickle'), 'wb'))
def count_resources(context, the_job):
db = context.get_compmake_db()
cache = get_job_cache(the_job, db=db)
if cache.state != Cache.DONE:
msg = 'The job %s was supposed to be finished: %s' % (the_job, cache)
raise Exception(msg)
cq = CacheQueryDB(db)
children = cq.tree_children_and_uodeps(the_job)
check_isinstance(children, set)
children.add(the_job)
res = {}
for j in children:
res[j] = context.comp_dynamic(my_get_job_cache, j, extra_dep=[Promise(j)],
job_id='count-%s-%s' % (the_job, j))
return context.comp(finalize_result, res)
def pysnip_make(dirname, compmake_command):
files = glob(os.path.join(dirname, '*.py'))
prefixes = [os.path.splitext(os.path.basename(f))[0] for f in files]
logger.info('Found %d snippets in directory %s' % (len(prefixes), dirname))
use_filesystem(os.path.join(dirname, '.compmake'))
ntodo = 0
for p in prefixes:
job = Job(dirname, p)
job_id = job.basename
current_state = None
if job_exists(job_id):
current_state = get_job_cache(job_id).state
if job.status == DONE_UPTODATE:
# logger.info('%s: done' % job.basename)
if current_state != Cache.DONE:
mark_as_done(job_id)
pass
elif job.status == FAILED:
# logger.info('%s: failed' % job.basename)
if current_state != Cache.FAILED:
mark_as_failed(job_id)
elif job.status == DONE_NEEDSUPDATE:
mark_as_notstarted(job_id)
# logger.info('%s: done (but needs update)' % job.basename)
pass
elif job.status == NOTSTARTED:
mark_as_notstarted(job_id)
# logger.info('%s: not started' % job.basename)
pass
comp(run_job, job, job_id=job_id)
if job.status != DONE_UPTODATE:
ntodo += 1
# logger.info('%d/%d jobs to do' % (ntodo, len(prefixes)))
batch_command('stats')
if compmake_command is not None:
return batch_command(compmake_command)
else:
compmake_console()
return 0