class App(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
self.neigbor_set = []
def get_neigbor_set(self):
"""
Get set of neighbors
"""
return self.neigbor_set
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, l: list):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(len(l)):
# 'data' will be passed to the slave and can be anything
self.work_queue.add_work(data=l[i])
#
# Keep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, child, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
if (child is not None) and (len(child) != 0):
self.neigbor_set.append(copy.deepcopy(child))
# sleep some time: this is a crucial detail discussed below!
time.sleep(0.03)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=100):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(tasks):
#
# the slave will be working on one out of 3 resources
#
resource_id = random.randint(1, 3)
data = ('Do something', i, resource_id)
self.work_queue.add_work(data, resource_id)
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
# sleep some time
time.sleep(0.3)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=None):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
task_list = np.loadtxt(BRICKSTAT_DIR + 'UnfinishedBricks.txt',
dtype=np.str)
if tasks is None:
tasks = len(task_list)
for i in range(tasks):
# 'data' will be passed to the slave and can be anything
self.work_queue.add_work(data=(task_list[i], i))
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
# sleep some time
time.sleep(0.3)
def __init__(self, slaves, masters_details):
self.slaves = list(slaves)
self.work_queue = {}
self.num_slaves = {}
for task_id, master, num_slaves in masters_details:
self.work_queue[task_id] = WorkQueue(master)
self.num_slaves[task_id] = num_slaves
# assign slaves to Masters
slaves = list(slaves)
while slaves:
for task_id, work_queue in self.work_queue.items():
if not slaves:
break
num_slaves = self.num_slaves[task_id]
master = work_queue.master
if num_slaves is None or master.num_slaves() < num_slaves:
master.add_slave(slaves.pop(0), ready=True)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, root_dir):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
# """
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for dirName, subdirList, fileList in os.walk(root_dir):
for fname in fileList:
self.work_queue.add_work(data=os.path.join(dirName, fname))
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if not done:
print(message)
class ParallelOde(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self,
ode_rhs_fun,
flux_fun,
t_final,
parameters,
ode_opts,
initial_value=(),
value_id=()):
"""
create tasks and add it to the work queue
Every task has specific arguments
"""
# set ode rhs function, initial condition and parameters
data = {
'ode_fun': ode_rhs_fun,
'y0': initial_value,
'id': value_id,
'ode_sys_opts': parameters,
'ode_opts': ode_opts,
't_final': t_final,
'flux_fun': flux_fun
}
# add data to work queue
self.work_queue.add_work(data)
def run_i_value(self, ode_rhs_fun, flux_fun, ode_sys_opts, initial_values,
t_final, ode_opts, experiment_id):
"""
This is the core where I keep starting slaves
as long as there is work to do
"""
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for j_experiment_id, j_value in zip(experiment_id, initial_values):
self.__add_next_task(ode_rhs_fun=ode_rhs_fun,
flux_fun=flux_fun,
t_final=t_final,
parameters=ode_sys_opts,
ode_opts=ode_opts,
initial_value=j_value,
value_id=j_experiment_id)
# Keeep starting slaves as long as there is work to do
all_boolean = []
all_tout = []
all_yout = []
all_y0_id = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
y0_id, done, time_course, y_result = slave_return_data
# import pdb; pdb.set_trace()
all_boolean.append(done)
all_tout.append(time_course)
all_yout.append(y_result)
all_y0_id.append(y0_id)
if done:
print('Master: slave finished its task returning: %s)' %
str(y0_id))
# sleep some time
# time.sleep(0.3)
results = {
'time': all_tout,
'y': all_yout,
'id': all_y0_id,
'boolean': all_boolean
}
return results
def run_i_parameter(self, ode_rhs_fun, flux_fun, ode_sys_opts,
initial_value, t_final, ode_opts, experiment_id):
"""
This is the core where I keep starting slaves
as long as there is work to do
"""
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for j_experiment_id, j_parameter in zip(experiment_id, ode_sys_opts):
self.__add_next_task(ode_rhs_fun=ode_rhs_fun,
flux_fun=flux_fun,
t_final=t_final,
parameters=j_parameter,
ode_opts=ode_opts,
initial_value=initial_value,
value_id=j_experiment_id)
# Keeep starting slaves as long as there is work to do
all_boolean = []
all_tout = []
all_yout = []
all_y0_id = []
all_flux = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
y0_id, done, time_course, y_result, flux = slave_return_data
# import pdb; pdb.set_trace()
all_boolean.append(done)
all_tout.append(time_course)
all_yout.append(y_result)
all_y0_id.append(y0_id)
all_flux.append(flux)
if done:
print('Master: slave finished its task returning: %s)' %
str(y0_id))
# sleep some time
# time.sleep(0.3)
results = {
'time': all_tout,
'y': all_yout,
'id': all_y0_id,
'boolean': all_boolean,
'flux': all_flux
}
return results
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating taahe Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, name_for_run = None, split_idx = 0, N_splits = 1, tasks=None):
"""
name for run list: elg_like_run,elg_ngc_run
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
#version 1:
#PB_fn = '/global/cscratch1/sd/huikong/obiwan_Aug/repos_for_docker/obiwan_code/py/obiwan/more/obiwan_run/brickstat/elg_200per_run/FinishedBricks.txt'
#bricknames = np.loadtxt(PB_fn,dtype=n.str).transpose()
#ntasks = len(bricknames)
#print('total of %d tasks' % ntasks)
#version1 end
#version 2:
import glob
from astropy.table import vstack
global BRICKPATH
global topdir_obiwan_out
print(BRICKPATH)
#paths = glob.glob(os.path.join(os.environ[name_for_run],'tractor','*','*'))
#paths = np.array_split(paths, N_splits)[split_idx]
bricknames = np.loadtxt(BRICKPATH,dtype=np.str)
bricknames = np.array_split(bricknames, N_splits)[split_idx]
final_sim = None
n=0
bricknames.sort()
for brickname in bricknames:
#brickname = os.path.basename(path)
self.work_queue.add_work(data=(n, brickname))
n+=1
#end of verion 2
#version 1:
#for i in range(ntasks):
# 'data' will be passed to the slave and can be anything
# self.work_queue.add_work(data=(i, bricknames[i]))
#version 1 end
#
# Keeep starting slaves as long as there is work to do
#
sim_table = None
tab = None
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, sim = slave_return_data
print('No %d is done' % done)
if sim is not None:
if final_sim is not None:
final_sim = vstack((final_sim,sim))
else:
final_sim=sim
# sleep some time
time.sleep(0.3)
print(topdir_obiwan_out,'subset','sim_%s_part%d_of_%d.fits' %(name_for_run, split_idx, N_splits))
print('writing all the output to one table...')
final_sim.write(os.path.join(topdir_obiwan_out,'subset','sim_%s_part%d_of_%d.fits' % (name_for_run, split_idx, N_splits)), format='fits',overwrite=True)
print('done!')
class ParallelValidate(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
""",
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, task, **kwargs):
"""
create tasks and add it to the work queue. Every task has specific arguments
"""
if task == 'initial_sim':
data = {
'sim_obj': kwargs['sim_obj'],
'ode_sys_opts': kwargs['ode_sys_opts'],
'id': kwargs['estimate_id'],
'y0': kwargs['y0'],
'task': task
}
elif task == 'perturbation_sim':
data = {
'sim_obj': kwargs['sim_obj'],
'ode_sys_opts': kwargs['ode_sys_opts'],
'id': kwargs['estimate_id'],
'y0': kwargs['y0'],
'perturbation_id': kwargs['perturbation_id'],
'task': task
}
self.work_queue.add_work(data)
def run_all(self, task, **kwargs):
"""parallel application core"""
if task == 'initial_sim':
estimates = kwargs['parameters']
estimate_id = kwargs['estimate_info']
sim_obj = kwargs['sim_obj']
for j_index, (j_estimate, j_estimate_id) in enumerate(
zip(estimates, estimate_id)):
self.__add_next_task(task=task,
**{
'sim_obj': sim_obj,
'ode_sys_opts': j_estimate,
'estimate_id': j_estimate_id,
'y0': sim_obj.wt_y0
})
# Keeep starting slaves as long as there is work to do
results = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
task, output = slave_return_data
if task == 'initial_sim':
j_slave_t, j_slave_y, j_slave_f, estimate_id, sample_id, data_set_id, sim_obj, ode_sys_opts = output
j_slave_dyn = {
't': j_slave_t,
'y': j_slave_y,
'flux': j_slave_f,
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id
}
# info on bistability
if j_slave_y[-1, 0] > j_slave_y[-1, 1]:
stability_id = 1
elif j_slave_y[-1, 0] < j_slave_y[-1, 1]:
stability_id = 2
else:
stability_id = 0
# get ss values
j_slave_ss = {
'y': j_slave_y[-1, :],
'flux': j_slave_f[-1, :],
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'ssid': stability_id
}
i_slave_result = {
'dynamic': j_slave_dyn,
'ss': j_slave_ss,
'initial': True,
'perturbation': False
}
# append initial sim results
results.append(i_slave_result)
# create list of perturbated parameter for given parameter estimate in ode_sys_opts
perturbed_parameter_list = sim_obj.change_parameter_values(
sim_obj.test_perturbations, ode_sys_opts)
# add perturbation jobs to back of jobs queue
experiment_id = [
'experiment_{}'.format(parameter_id) for parameter_id,
_ in enumerate(perturbed_parameter_list)
]
for j_experiment_id, j_perturbation in zip(
experiment_id, perturbed_parameter_list):
self.__add_next_task(task='perturbation_sim',
**{
'sim_obj':
sim_obj,
'ode_sys_opts':
j_perturbation,
'estimate_id':
(estimate_id, sample_id,
data_set_id),
'perturbation_id':
j_experiment_id,
'y0':
j_slave_y[-1, :]
})
elif task == 'perturbation_sim':
j_slave_t, j_slave_y, j_slave_f, estimate_id, sample_id, data_set_id, perturbation_id = output
j_slave_dyn = {
't': j_slave_t,
'y': j_slave_y,
'flux': j_slave_f,
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'perturbation_id': perturbation_id
}
# info on bistability
if j_slave_y[-1, 0] > j_slave_y[-1, 1]:
stability_id = 1
elif j_slave_y[-1, 0] < j_slave_y[-1, 1]:
stability_id = 2
else:
stability_id = 0
# get ss values
j_slave_ss = {
'y': j_slave_y[-1, :],
'flux': j_slave_f[-1, :],
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'perturbation_id': perturbation_id,
'ssid': stability_id
}
i_slave_result = {
'dynamic': j_slave_dyn,
'ss': j_slave_ss,
'initial': False,
'perturbation': True
}
# append perturbation results
results.append(i_slave_result)
return results
class ParallelProcess(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
""",
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, task, **kwargs):
"""
create tasks and add it to the work queue. Every task has specific arguments
"""
if task == 'ident':
data = {
'ident_fun': kwargs['ident_fun'],
'flux_id': kwargs['flux_id'],
'flux_choice': kwargs['flux_choice'],
'sample_id': kwargs['sample_id'],
'data_set_id': kwargs['data_set_id'],
'exp_data': kwargs['exp_data'],
'task': task
}
self.work_queue.add_work(data)
def run_all(self, task, **kwargs):
"""parallel application core"""
if task == 'ident':
original_df = kwargs['exp_df']
# remove experiment_id as index
reset_df = original_df.reset_index('experiment_id')
ident_fun = kwargs['ident_fun']
flux_id = kwargs['flux_id']
flux_choice = kwargs['flux_choice']
idx = pd.IndexSlice
all_df_indices = reset_df.index.unique().tolist()
# create tuple of indices
# import pdb; pdb.set_trace()
for j_index, sample_data_set_id in enumerate(all_df_indices):
j_exp_data_set = reset_df.loc[
idx[sample_data_set_id],
['acetate', 'pep', 'fdp', 'E', 'v1', 'v2', 'v3', 'v5'
]].values.tolist()
flat_data_list = [
i_element for i_data in j_exp_data_set
for i_element in i_data
]
self.__add_next_task(task=task,
**{
'exp_data': flat_data_list,
'ident_fun': ident_fun,
'flux_id': flux_id,
'flux_choice': flux_choice,
'sample_id': sample_data_set_id[0],
'data_set_id': sample_data_set_id[1]
})
# Keeep starting slaves as long as there is work to do
results = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
task, output = slave_return_data
if task == 'ident':
ident_info, slave_flux_id, slave_flux_choice, sample_id, data_set_id = output
i_slave_result = {
'flux_id': slave_flux_id,
'flux_choice': slave_flux_choice,
'sample_id': sample_id,
'data_set_id': data_set_id,
'ident_info': ident_info
}
results.append(i_slave_result)
print('Master: slave finished task %s returning: %s)' %
(task, str(data_set_id)))
return results
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done. There different type of work so
the slaves must be able to do different tasks
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, i, task=None):
"""
we create random tasks 1-3 and add it to the work queue
Every task has specific arguments
"""
if task is None:
task = random.randint(1, 3)
if task == 1:
args = i
data = (Tasks.TASK1, args)
elif task == 2:
args = (i, i * 2)
data = (Tasks.TASK2, args)
elif task == 3:
args = (i, 999, 'something')
data = (Tasks.TASK3, args)
self.work_queue.add_work(data)
def run(self, tasks=100):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(tasks):
self.__add_next_task(i)
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
#
# each task type has its own return type
#
task, data = slave_return_data
if task == Tasks.TASK1:
done, arg1 = data
elif task == Tasks.TASK2:
done, arg1, arg2, arg3 = data
elif task == Tasks.TASK3:
done, arg1, arg2 = data
if done:
print('Master: slave finished is task returning: %s)' %
str(data))
# sleep some time
time.sleep(0.3)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=None):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
#version 1:
#PB_fn = os.path.join(os.environ['DRONES_DIR'], 'obiwan_analysis/brickstat/FinishedBricks.txt')
#ntasks = len(np.loadtxt(PB_fn,dtype=n.str).transpose())
#print('total of %d tasks' % ntasks)
#version1 end
#version 2:
import glob
from astropy.table import vstack
paths = glob.glob(os.path.join(os.environ['NGC_tractor'],'*','*'))
final_tab = None
n=0
for path in paths:
brickname = os.path.basename(path)
self.work_queue.add_work(data=(n, brickname))
n+=1
#version 1:
#for i in range(ntasks):
# 'data' will be passed to the slave and can be anything
#self.work_queue.add_work(data=(i, i))
#version 1 end
#
# Keeep starting slaves as long as there is work to do
#
final_table = None
tab = None
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, tab = slave_return_data
print('No %d is done' % done)
if tab is not None:
if final_table is not None:
final_table = vstack([final_table, tab])
else:
final_table = tab
# sleep some time
time.sleep(0.3)
print('writing all the output to one table...')
final_table.write(os.path.join(os.environ['obiwan_out'],'subset','ngc_sim.fits'), format='fits',overwrite=True)
print('done!')