def compute(self):
# randomly wait a few seconds so that multiple processes accessing the same
# Theano function do not cause a lock problem. I do not know why.
# I do not know if this does anything useful.
# Sleep in seconds.
time.sleep(np.random.rand(1) * 3)
paired_source = self.paired_source
r = self.rep
n = self.n
job_func = self.job_func
pdata = paired_source.sample(n, seed=r)
with util.ContextTimer() as t:
logger.info("computing. %s. prob=%s, r=%d, n=%d" %
(job_func.__name__, pdata.label, r, n))
tr, te = pdata.split_tr_te(tr_proportion=tr_proportion,
seed=r + 21)
prob_label = self.prob_label
job_result = job_func(paired_source, tr, te, r)
# create ScalarResult instance
result = SingleResult(job_result)
# submit the result to my own aggregator
self.aggregator.submit_result(result)
func_name = job_func.__name__
logger.info("done. ex1: %s, prob=%s, r=%d, n=%d. Took: %.3g s " %
(func_name, pdata.label, r, n, t.secs))
# save result
fname = '%s-%s-r%d_n%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, r, n, alpha, tr_proportion)
glo.ex_save_result(ex, job_result, prob_label, fname)
def compute(self):
# randomly wait a few seconds so that multiple processes accessing the same
# Theano function do not cause a lock problem. I do not know why.
# I do not know if this does anything useful.
# Sleep in seconds.
time.sleep(np.random.rand(1) * 2)
# load the data and construct a PairedSource here
# The data can be big. We have to load it in this job function i.e.,
# each computing node loads by itself (no data passing).
folder_path = self.folder_path
prob_label = self.prob_label
paired_source, _, is_h0 = exglo.get_problem_pickle(
folder_path, prob_label + '.n0')
n = self.n
r = self.rep
job_func = self.job_func
pdata = paired_source.sample(n, seed=r)
with util.ContextTimer() as t:
logger.info("computing. %s. prob=%s, r=%d, n=%d" %
(job_func.__name__, pdata.label, r, n))
tr, te = pdata.split_tr_te(tr_proportion=tr_proportion,
seed=r + 21)
prob_label = self.prob_label
job_result = job_func(paired_source, tr, te, r)
# create ScalarResult instance
result = SingleResult(job_result)
# submit the result to my own aggregator
self.aggregator.submit_result(result)
func_name = job_func.__name__
logger.info("done. ex1: %s, prob=%s, r=%d, n=%d. Took: %.3g s " %
(func_name, pdata.label, r, n, t.secs))
# save result
fname = '%s-%s-r%d_n%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, r, n, alpha, tr_proportion)
glo.ex_save_result(ex, job_result, prob_label, fname)
def run_problem(folder_path, prob_label):
"""Run the experiment"""
pl = exglo.parse_prob_label(prob_label)
is_h0 = pl['is_h0']
n = pl['n']
# /////// submit jobs //////////
# create folder name string
#result_folder = glo.result_folder()
#tmp_dir = tempfile.gettempdir()
from fsic.config import expr_configs
tmp_dir = expr_configs['scratch_dir']
foldername = os.path.join(tmp_dir, 'wj_slurm', 'e%d' % ex)
logger.info("Setting engine folder to %s" % foldername)
# create parameter instance that is needed for any batch computation engine
logger.info("Creating batch parameter instance")
batch_parameters = BatchClusterParameters(foldername=foldername,
job_name_base="e%d_" % ex,
parameter_prefix="")
# Use the following line if Slurm queue is not used.
#engine = SerialComputationEngine()
engine = SlurmComputationEngine(batch_parameters)
n_methods = len(method_job_funcs)
# repetitions x sample_sizes x #methods
aggregators = np.empty((reps, n_methods), dtype=object)
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
# name used to save the result
func_name = f.__name__
fname = '%s-%s-r%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, r, alpha, tr_proportion)
if not is_rerun and glo.ex_file_exists(ex, prob_label, fname):
logger.info('%s exists. Load and return.' % fname)
job_result = glo.ex_load_result(ex, prob_label, fname)
sra = SingleResultAggregator()
sra.submit_result(SingleResult(job_result))
aggregators[r, mi] = sra
else:
# result not exists or rerun
job = Ex4Job(SingleResultAggregator(), folder_path, prob_label,
r, f)
agg = engine.submit_job(job)
aggregators[r, mi] = agg
# let the engine finish its business
logger.info("Wait for all call in engine")
engine.wait_for_all()
# ////// collect the results ///////////
logger.info("Collecting results")
job_results = np.empty((reps, n_methods), dtype=object)
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
logger.info("Collecting result (%s, r=%d, n=%d)" %
(f.__name__, r, n))
# let the aggregator finalize things
aggregators[r, mi].finalize()
# aggregators[i].get_final_result() returns a SingleResult instance,
# which we need to extract the actual result
job_result = aggregators[r, mi].get_final_result().result
job_results[r, mi] = job_result
#func_names = [f.__name__ for f in method_job_funcs]
#func2labels = exglobal.get_func2label_map()
#method_labels = [func2labels[f] for f in func_names if f in func2labels]
# save results
# - Do not store PairedSource because it can be very big.
results = {
'job_results': job_results,
'n': n,
'is_h0': is_h0,
'alpha': alpha,
'repeats': reps,
'tr_proportion': tr_proportion,
'method_job_funcs': method_job_funcs,
'prob_label': prob_label,
}
# class name
fname = 'ex%d-%s-me%d_rs%d_a%.3f_trp%.2f.p' \
%(ex, prob_label, n_methods, reps, alpha, tr_proportion)
glo.ex_save_result(ex, results, fname)
logger.info('Saved aggregated results to %s' % fname)