def run_example_cluster(): """ run a set of jobs on cluster """ print "" print "" print "=====================================" print "======== Submit and Wait ========" print "=====================================" print "" print "" functionJobs = make_jobs() print "output ret field in each job before sending it onto the cluster" for (i, job) in enumerate(functionJobs): print "Job #", i, "- ret: ", job.ret print "" print "sending function jobs to cluster" print "" processedFunctionJobs = process_jobs(functionJobs) print "ret fields AFTER execution on cluster" for (i, job) in enumerate(processedFunctionJobs): print "Job #", i, "- ret: ", str(job.ret)[0:10]
def run_example_local_multithreading(): """ run a set of jobs on local machine using several cores """ print "=====================================" print "====== Local Multithreading =======" print "=====================================" print "" print "" print "generating function jobs" functionJobs = make_jobs() # KybJob object start out with an empty ret field, which is only filled after execution print "output ret field in each job before multithreaded computation" for (i, job) in enumerate(functionJobs): print "Job #", i, "- ret: ", job.ret print "" print "executing jobs on local machine using 3 threads" processedFunctionJobs = process_jobs(functionJobs, local=True, maxNumThreads=3) print "ret fields AFTER execution on local machine" for (i, job) in enumerate(processedFunctionJobs): print "Job #", i, "- ret: ", str(job.ret)[0:10]
def execute_runs(run_ids, mem, local, threads): """ takes a list of run ids and computes them """ print "created", len(run_ids), " runs: ", run_ids # use pythongrid jobs = [] for run_id in run_ids: job = KybJob(expenv.execute_run, [run_id]) job.h_vmem = mem jobs.append(job) #global local print "local", local print "maxNumThreads", threads finished_jobs = process_jobs(jobs, local=local, maxNumThreads=threads) MultiSourceExperiment._connection.expireAll() return finished_jobs
tmp = comment + " " + str(myarg) exp_id = expenv_runner.run_multi_example(317, "method_hierarchy_svm", tmp, myarg) return exp_id if __name__ == '__main__': #Bs = numpy.double(range(0,41))/4 #Bs = numpy.double(range(0,2)) #Bs = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 2.0, 3.0, 10.0, 100.0, 1000.0] Bs = [pow(10, i) for i in (numpy.double(range(0, 41)) / 10 - 4) ] #[pow(10, j) for j in (numpy.double(range(0, 21))/10 - 2)] jobs = [] mycomment = "global C=1.0, toy_0" for B in Bs: job = KybJob(analysis.run_job, [B, mycomment]) jobs.append(job) idx = [job.ret for job in process_jobs(jobs, True, 4)] print "using idx", idx plot_analysis(idx)
tmp = comment + " " + str(myarg) exp_id = expenv_runner.run_multi_example(317, "method_hierarchy_svm", tmp, myarg) return exp_id if __name__ == '__main__': #Bs = numpy.double(range(0,41))/4 #Bs = numpy.double(range(0,2)) #Bs = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 2.0, 3.0, 10.0, 100.0, 1000.0] Bs = [pow(10, i) for i in (numpy.double(range(0, 41))/10 - 4)] #[pow(10, j) for j in (numpy.double(range(0, 21))/10 - 2)] jobs = [] mycomment = "global C=1.0, toy_0" for B in Bs: job = KybJob(analysis.run_job, [B, mycomment]) jobs.append(job) idx = [job.ret for job in process_jobs(jobs, True, 4)] print "using idx", idx plot_analysis(idx)
def run(self, debug=False, resume=False, config=None, pythonpathdir=None, init_stage=0): #Initialize modules for stage in self.stages: for module in stage: module.initialize(self.work_path) logging.debug("Initialization Finished") #default configuration default_config = {'*': {'h_cpu': '1:0:0', 'h_vmem': '1G'}} if not config: logging.debug("Using default configuration") config = default_config else: if not isinstance(config, dict): raise TypeError('config is given as a dictionary') if all(isinstance(v, basestring) for v in config.values()): logging.debug("Using same configuration for all modules") config = {'*': config} config = dict_merge(config, default_config) logging.debug("Merged user-provided configuration: {0}".format(config)) #By default, if no path is given, it adds to the python path #the directory where the module was defined if not pythonpathdir: import inspect pythonpathdirs = set() #Add to the pythonpath the directory where the pipeline is defined pythonpathdirs.add(os.path.dirname(os.path.abspath( inspect.getfile(type(self))))) for stage in self.stages: for module in stage: #Add to the pythonpath the directory where each module #is defined pythonpathdirs.add(os.path.dirname(os.path.abspath( inspect.getfile(type(module))))) for arg in module.args: try: #Add to the pythonpath the directory where each argument #is defined pythonpathdirs.add(os.path.dirname( os.path.abspath(inspect.getfile(arg)))) except TypeError: pass pythonpathdir = ":".join(pythonpathdirs) logging.debug("Adding potential useful paths to the pythonpath: {0}" .format(pythonpathdir)) with contextlib.nested(*self.ctx_mgrs): if debug: #give time for context managers to initialize #(for kyototycoon debugging) time.sleep(1) for i_stage, stage in enumerate(self.stages): if i_stage < init_stage: continue jobs = [] for module in stage: if not resume or not module.finished(): job = KybJob(run_clmodule, [module], pythonpathdir=pythonpathdir, logdir=os.path.abspath(module.work_path)) self.apply_config(config, module, job) jobs.append(job) elif resume: logging.info("{0}: SKIP".format(module)) process_jobs(jobs, local=debug) for module in stage: module.finalize()