def compute(self): # randomly wait a few seconds so that multiple processes accessing the same # Theano function do not cause a lock problem. I do not know why. # I do not know if this does anything useful. # Sleep in seconds. time.sleep(np.random.rand(1) * 3) paired_source = self.paired_source r = self.rep n = self.n job_func = self.job_func pdata = paired_source.sample(n, seed=r) with util.ContextTimer() as t: logger.info("computing. %s. prob=%s, r=%d, n=%d" % (job_func.__name__, pdata.label, r, n)) tr, te = pdata.split_tr_te(tr_proportion=tr_proportion, seed=r + 21) prob_label = self.prob_label job_result = job_func(paired_source, tr, te, r) # create ScalarResult instance result = SingleResult(job_result) # submit the result to my own aggregator self.aggregator.submit_result(result) func_name = job_func.__name__ logger.info("done. ex1: %s, prob=%s, r=%d, n=%d. Took: %.3g s " % (func_name, pdata.label, r, n, t.secs)) # save result fname = '%s-%s-r%d_n%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, r, n, alpha, tr_proportion) glo.ex_save_result(ex, job_result, prob_label, fname)
def compute(self): # randomly wait a few seconds so that multiple processes accessing the same # Theano function do not cause a lock problem. I do not know why. # I do not know if this does anything useful. # Sleep in seconds. time.sleep(np.random.rand(1) * 2) # load the data and construct a PairedSource here # The data can be big. We have to load it in this job function i.e., # each computing node loads by itself (no data passing). folder_path = self.folder_path prob_label = self.prob_label paired_source, _, is_h0 = exglo.get_problem_pickle( folder_path, prob_label + '.n0') n = self.n r = self.rep job_func = self.job_func pdata = paired_source.sample(n, seed=r) with util.ContextTimer() as t: logger.info("computing. %s. prob=%s, r=%d, n=%d" % (job_func.__name__, pdata.label, r, n)) tr, te = pdata.split_tr_te(tr_proportion=tr_proportion, seed=r + 21) prob_label = self.prob_label job_result = job_func(paired_source, tr, te, r) # create ScalarResult instance result = SingleResult(job_result) # submit the result to my own aggregator self.aggregator.submit_result(result) func_name = job_func.__name__ logger.info("done. ex1: %s, prob=%s, r=%d, n=%d. Took: %.3g s " % (func_name, pdata.label, r, n, t.secs)) # save result fname = '%s-%s-r%d_n%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, r, n, alpha, tr_proportion) glo.ex_save_result(ex, job_result, prob_label, fname)
def run_problem(folder_path, prob_label): """Run the experiment""" pl = exglo.parse_prob_label(prob_label) is_h0 = pl['is_h0'] n = pl['n'] # /////// submit jobs ////////// # create folder name string #result_folder = glo.result_folder() #tmp_dir = tempfile.gettempdir() from fsic.config import expr_configs tmp_dir = expr_configs['scratch_dir'] foldername = os.path.join(tmp_dir, 'wj_slurm', 'e%d' % ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters(foldername=foldername, job_name_base="e%d_" % ex, parameter_prefix="") # Use the following line if Slurm queue is not used. #engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters) n_methods = len(method_job_funcs) # repetitions x sample_sizes x #methods aggregators = np.empty((reps, n_methods), dtype=object) for r in range(reps): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ fname = '%s-%s-r%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, r, alpha, tr_proportion) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): logger.info('%s exists. Load and return.' % fname) job_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() sra.submit_result(SingleResult(job_result)) aggregators[r, mi] = sra else: # result not exists or rerun job = Ex4Job(SingleResultAggregator(), folder_path, prob_label, r, f) agg = engine.submit_job(job) aggregators[r, mi] = agg # let the engine finish its business logger.info("Wait for all call in engine") engine.wait_for_all() # ////// collect the results /////////// logger.info("Collecting results") job_results = np.empty((reps, n_methods), dtype=object) for r in range(reps): for mi, f in enumerate(method_job_funcs): logger.info("Collecting result (%s, r=%d, n=%d)" % (f.__name__, r, n)) # let the aggregator finalize things aggregators[r, mi].finalize() # aggregators[i].get_final_result() returns a SingleResult instance, # which we need to extract the actual result job_result = aggregators[r, mi].get_final_result().result job_results[r, mi] = job_result #func_names = [f.__name__ for f in method_job_funcs] #func2labels = exglobal.get_func2label_map() #method_labels = [func2labels[f] for f in func_names if f in func2labels] # save results # - Do not store PairedSource because it can be very big. results = { 'job_results': job_results, 'n': n, 'is_h0': is_h0, 'alpha': alpha, 'repeats': reps, 'tr_proportion': tr_proportion, 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, } # class name fname = 'ex%d-%s-me%d_rs%d_a%.3f_trp%.2f.p' \ %(ex, prob_label, n_methods, reps, alpha, tr_proportion) glo.ex_save_result(ex, results, fname) logger.info('Saved aggregated results to %s' % fname)