def run_example_cluster():
"""
run a set of jobs on cluster
"""
print ""
print ""
print "====================================="
print "======== Submit and Wait ========"
print "====================================="
print ""
print ""
functionJobs = make_jobs()
print "output ret field in each job before sending it onto the cluster"
for (i, job) in enumerate(functionJobs):
print "Job #", i, "- ret: ", job.ret
print ""
print "sending function jobs to cluster"
print ""
processedFunctionJobs = process_jobs(functionJobs)
print "ret fields AFTER execution on cluster"
for (i, job) in enumerate(processedFunctionJobs):
print "Job #", i, "- ret: ", str(job.ret)[0:10]
def run_example_local_multithreading():
"""
run a set of jobs on local machine using several cores
"""
print "====================================="
print "====== Local Multithreading ======="
print "====================================="
print ""
print ""
print "generating function jobs"
functionJobs = make_jobs()
# KybJob object start out with an empty ret field, which is only filled after execution
print "output ret field in each job before multithreaded computation"
for (i, job) in enumerate(functionJobs):
print "Job #", i, "- ret: ", job.ret
print ""
print "executing jobs on local machine using 3 threads"
processedFunctionJobs = process_jobs(functionJobs, local=True, maxNumThreads=3)
print "ret fields AFTER execution on local machine"
for (i, job) in enumerate(processedFunctionJobs):
print "Job #", i, "- ret: ", str(job.ret)[0:10]
def execute_runs(run_ids, mem, local, threads):
"""
takes a list of run ids and computes them
"""
print "created", len(run_ids), " runs: ", run_ids
# use pythongrid
jobs = []
for run_id in run_ids:
job = KybJob(expenv.execute_run, [run_id])
job.h_vmem = mem
jobs.append(job)
#global local
print "local", local
print "maxNumThreads", threads
finished_jobs = process_jobs(jobs, local=local, maxNumThreads=threads)
MultiSourceExperiment._connection.expireAll()
return finished_jobs
def execute_runs(run_ids, mem, local, threads):
"""
takes a list of run ids and computes them
"""
print "created", len(run_ids), " runs: ", run_ids
# use pythongrid
jobs = []
for run_id in run_ids:
job = KybJob(expenv.execute_run, [run_id])
job.h_vmem = mem
jobs.append(job)
#global local
print "local", local
print "maxNumThreads", threads
finished_jobs = process_jobs(jobs, local=local, maxNumThreads=threads)
MultiSourceExperiment._connection.expireAll()
return finished_jobs
tmp = comment + " " + str(myarg)
exp_id = expenv_runner.run_multi_example(317, "method_hierarchy_svm", tmp,
myarg)
return exp_id
if __name__ == '__main__':
#Bs = numpy.double(range(0,41))/4
#Bs = numpy.double(range(0,2))
#Bs = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 2.0, 3.0, 10.0, 100.0, 1000.0]
Bs = [pow(10, i) for i in (numpy.double(range(0, 41)) / 10 - 4)
] #[pow(10, j) for j in (numpy.double(range(0, 21))/10 - 2)]
jobs = []
mycomment = "global C=1.0, toy_0"
for B in Bs:
job = KybJob(analysis.run_job, [B, mycomment])
jobs.append(job)
idx = [job.ret for job in process_jobs(jobs, True, 4)]
print "using idx", idx
plot_analysis(idx)
tmp = comment + " " + str(myarg)
exp_id = expenv_runner.run_multi_example(317, "method_hierarchy_svm", tmp, myarg)
return exp_id
if __name__ == '__main__':
#Bs = numpy.double(range(0,41))/4
#Bs = numpy.double(range(0,2))
#Bs = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 2.0, 3.0, 10.0, 100.0, 1000.0]
Bs = [pow(10, i) for i in (numpy.double(range(0, 41))/10 - 4)] #[pow(10, j) for j in (numpy.double(range(0, 21))/10 - 2)]
jobs = []
mycomment = "global C=1.0, toy_0"
for B in Bs:
job = KybJob(analysis.run_job, [B, mycomment])
jobs.append(job)
idx = [job.ret for job in process_jobs(jobs, True, 4)]
print "using idx", idx
plot_analysis(idx)
def run(self, debug=False, resume=False, config=None, pythonpathdir=None,
init_stage=0):
#Initialize modules
for stage in self.stages:
for module in stage:
module.initialize(self.work_path)
logging.debug("Initialization Finished")
#default configuration
default_config = {'*': {'h_cpu': '1:0:0', 'h_vmem': '1G'}}
if not config:
logging.debug("Using default configuration")
config = default_config
else:
if not isinstance(config, dict):
raise TypeError('config is given as a dictionary')
if all(isinstance(v, basestring) for v in config.values()):
logging.debug("Using same configuration for all modules")
config = {'*': config}
config = dict_merge(config, default_config)
logging.debug("Merged user-provided configuration: {0}".format(config))
#By default, if no path is given, it adds to the python path
#the directory where the module was defined
if not pythonpathdir:
import inspect
pythonpathdirs = set()
#Add to the pythonpath the directory where the pipeline is defined
pythonpathdirs.add(os.path.dirname(os.path.abspath(
inspect.getfile(type(self)))))
for stage in self.stages:
for module in stage:
#Add to the pythonpath the directory where each module
#is defined
pythonpathdirs.add(os.path.dirname(os.path.abspath(
inspect.getfile(type(module)))))
for arg in module.args:
try:
#Add to the pythonpath the directory where each argument
#is defined
pythonpathdirs.add(os.path.dirname(
os.path.abspath(inspect.getfile(arg))))
except TypeError:
pass
pythonpathdir = ":".join(pythonpathdirs)
logging.debug("Adding potential useful paths to the pythonpath: {0}"
.format(pythonpathdir))
with contextlib.nested(*self.ctx_mgrs):
if debug:
#give time for context managers to initialize
#(for kyototycoon debugging)
time.sleep(1)
for i_stage, stage in enumerate(self.stages):
if i_stage < init_stage:
continue
jobs = []
for module in stage:
if not resume or not module.finished():
job = KybJob(run_clmodule, [module],
pythonpathdir=pythonpathdir,
logdir=os.path.abspath(module.work_path))
self.apply_config(config, module, job)
jobs.append(job)
elif resume:
logging.info("{0}: SKIP".format(module))
process_jobs(jobs, local=debug)
for module in stage:
module.finalize()
