class App(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
self.neigbor_set = []
def get_neigbor_set(self):
"""
Get set of neighbors
"""
return self.neigbor_set
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, l: list):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(len(l)):
# 'data' will be passed to the slave and can be anything
self.work_queue.add_work(data=l[i])
#
# Keep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, child, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
if (child is not None) and (len(child) != 0):
self.neigbor_set.append(copy.deepcopy(child))
# sleep some time: this is a crucial detail discussed below!
time.sleep(0.03)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=100):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(tasks):
#
# the slave will be working on one out of 3 resources
#
resource_id = random.randint(1, 3)
data = ('Do something', i, resource_id)
self.work_queue.add_work(data, resource_id)
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
# sleep some time
time.sleep(0.3)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=None):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
task_list = np.loadtxt(BRICKSTAT_DIR + 'UnfinishedBricks.txt',
dtype=np.str)
if tasks is None:
tasks = len(task_list)
for i in range(tasks):
# 'data' will be passed to the slave and can be anything
self.work_queue.add_work(data=(task_list[i], i))
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if done:
print('Master: slave finished is task and says "%s"' %
message)
# sleep some time
time.sleep(0.3)
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
self.url_queue = [
'http://riweb.tibeica.com/crawl/'
] # 'http://fanacmilan.com/' 'http://riweb.tibeica.com/crawl/' 'http://www.greenworldmoldova.com/'
self.visited = {}
self.UTILS = Utils()
self.limit = 200
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=10):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
work_queue = [i for i in range(tasks)
] # let's pretend this is our work queue
#
# while we have work to do and not all slaves completed
#
while work_queue or not self.master.done():
#
# give work to do to each idle slave
#
for slave in self.master.get_ready_slaves():
if not work_queue:
break
task = work_queue.pop(0) # get next task in the queue
print('Master: slave %d is going to do task %d' %
(slave, task))
self.master.run(slave, data=('Do task', task))
#
# reclaim slaves that have finished working
# so that we can assign them more work
#
for slave in self.master.get_completed_slaves():
done, message = self.master.get_data(slave)
if done:
print('Master: slave %d finished is task and says "%s"' %
(slave, message))
else:
print('Master: slave %d failed to accomplish his task' %
slave)
# sleep some time
time.sleep(0.3)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, root_dir):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
# """
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for dirName, subdirList, fileList in os.walk(root_dir):
for fname in fileList:
self.work_queue.add_work(data=os.path.join(dirName, fname))
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if not done:
print(message)
class ParallelOde(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self,
ode_rhs_fun,
flux_fun,
t_final,
parameters,
ode_opts,
initial_value=(),
value_id=()):
"""
create tasks and add it to the work queue
Every task has specific arguments
"""
# set ode rhs function, initial condition and parameters
data = {
'ode_fun': ode_rhs_fun,
'y0': initial_value,
'id': value_id,
'ode_sys_opts': parameters,
'ode_opts': ode_opts,
't_final': t_final,
'flux_fun': flux_fun
}
# add data to work queue
self.work_queue.add_work(data)
def run_i_value(self, ode_rhs_fun, flux_fun, ode_sys_opts, initial_values,
t_final, ode_opts, experiment_id):
"""
This is the core where I keep starting slaves
as long as there is work to do
"""
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for j_experiment_id, j_value in zip(experiment_id, initial_values):
self.__add_next_task(ode_rhs_fun=ode_rhs_fun,
flux_fun=flux_fun,
t_final=t_final,
parameters=ode_sys_opts,
ode_opts=ode_opts,
initial_value=j_value,
value_id=j_experiment_id)
# Keeep starting slaves as long as there is work to do
all_boolean = []
all_tout = []
all_yout = []
all_y0_id = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
y0_id, done, time_course, y_result = slave_return_data
# import pdb; pdb.set_trace()
all_boolean.append(done)
all_tout.append(time_course)
all_yout.append(y_result)
all_y0_id.append(y0_id)
if done:
print('Master: slave finished its task returning: %s)' %
str(y0_id))
# sleep some time
# time.sleep(0.3)
results = {
'time': all_tout,
'y': all_yout,
'id': all_y0_id,
'boolean': all_boolean
}
return results
def run_i_parameter(self, ode_rhs_fun, flux_fun, ode_sys_opts,
initial_value, t_final, ode_opts, experiment_id):
"""
This is the core where I keep starting slaves
as long as there is work to do
"""
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for j_experiment_id, j_parameter in zip(experiment_id, ode_sys_opts):
self.__add_next_task(ode_rhs_fun=ode_rhs_fun,
flux_fun=flux_fun,
t_final=t_final,
parameters=j_parameter,
ode_opts=ode_opts,
initial_value=initial_value,
value_id=j_experiment_id)
# Keeep starting slaves as long as there is work to do
all_boolean = []
all_tout = []
all_yout = []
all_y0_id = []
all_flux = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
y0_id, done, time_course, y_result, flux = slave_return_data
# import pdb; pdb.set_trace()
all_boolean.append(done)
all_tout.append(time_course)
all_yout.append(y_result)
all_y0_id.append(y0_id)
all_flux.append(flux)
if done:
print('Master: slave finished its task returning: %s)' %
str(y0_id))
# sleep some time
# time.sleep(0.3)
results = {
'time': all_tout,
'y': all_yout,
'id': all_y0_id,
'boolean': all_boolean,
'flux': all_flux
}
return results
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating taahe Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, name_for_run = None, split_idx = 0, N_splits = 1, tasks=None):
"""
name for run list: elg_like_run,elg_ngc_run
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
#version 1:
#PB_fn = '/global/cscratch1/sd/huikong/obiwan_Aug/repos_for_docker/obiwan_code/py/obiwan/more/obiwan_run/brickstat/elg_200per_run/FinishedBricks.txt'
#bricknames = np.loadtxt(PB_fn,dtype=n.str).transpose()
#ntasks = len(bricknames)
#print('total of %d tasks' % ntasks)
#version1 end
#version 2:
import glob
from astropy.table import vstack
global BRICKPATH
global topdir_obiwan_out
print(BRICKPATH)
#paths = glob.glob(os.path.join(os.environ[name_for_run],'tractor','*','*'))
#paths = np.array_split(paths, N_splits)[split_idx]
bricknames = np.loadtxt(BRICKPATH,dtype=np.str)
bricknames = np.array_split(bricknames, N_splits)[split_idx]
final_sim = None
n=0
bricknames.sort()
for brickname in bricknames:
#brickname = os.path.basename(path)
self.work_queue.add_work(data=(n, brickname))
n+=1
#end of verion 2
#version 1:
#for i in range(ntasks):
# 'data' will be passed to the slave and can be anything
# self.work_queue.add_work(data=(i, bricknames[i]))
#version 1 end
#
# Keeep starting slaves as long as there is work to do
#
sim_table = None
tab = None
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, sim = slave_return_data
print('No %d is done' % done)
if sim is not None:
if final_sim is not None:
final_sim = vstack((final_sim,sim))
else:
final_sim=sim
# sleep some time
time.sleep(0.3)
print(topdir_obiwan_out,'subset','sim_%s_part%d_of_%d.fits' %(name_for_run, split_idx, N_splits))
print('writing all the output to one table...')
final_sim.write(os.path.join(topdir_obiwan_out,'subset','sim_%s_part%d_of_%d.fits' % (name_for_run, split_idx, N_splits)), format='fits',overwrite=True)
print('done!')
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
self.url_queue = [
'http://riweb.tibeica.com/crawl/'
] # 'http://fanacmilan.com/' 'http://riweb.tibeica.com/crawl/' 'http://www.greenworldmoldova.com/'
self.visited = {}
self.UTILS = Utils()
self.limit = 200
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# while we have work to do and not all slaves completed
#
while self.url_queue or not self.master.done():
#
# give work to do to each idle slave
#
for slave in self.master.get_ready_slaves():
if not self.url_queue:
break
current_url = self.url_queue.pop(
0) # get next url in the queue
if current_url in self.visited:
continue
print('Slave {0} is going to process url {1}'.format(
slave, current_url))
# check url in robots
rp = urllib.robotparser.RobotFileParser()
try:
url = urlparse(current_url)
rp.set_url(url.scheme + '://' + url.netloc + '/robots.txt')
self.read_robots(rp)
if not rp.can_fetch(self.UTILS.USER_AGENT, current_url):
continue
except:
continue
# set to visited current url
self.visited[url.scheme + '://' + url.netloc + url.path] = True
self.master.run(slave, data=current_url)
self.limit -= 1
print('Limit: {}'.format(self.limit))
#
# reclaim slaves that have finished working
# so that we can assign them more work
#
for slave in self.master.get_completed_slaves():
done, code, file_path, url = self.master.get_data(slave)
if done:
if code == 200:
self.get_links(file_path, url)
else:
new_location = file_path
if code == 301:
try:
self.visited.pop(url.geturl(), None)
except:
pass
if new_location not in self.url_queue and new_location not in self.visited:
self.url_queue.insert(0, new_location)
else:
print(
'Failed to process the url: {0} --- Response code: {1}'
.format(url.geturl(), code))
if self.limit <= 0:
self.terminate_slaves()
def get_links(self, html_file, url):
with open(html_file, 'r') as file:
soup = BeautifulSoup(file, features="html.parser")
meta_robots = soup.find("meta", attrs={"name": "robots"})
# if robots meta exists, check if the following is allowed
if meta_robots:
meta_robots_content = meta_robots['content']
if 'nofollow' in meta_robots_content:
return
for a in soup.find_all('a', href=True):
href = a['href'].strip()
if href == '' or href[0] == '#':
continue
if '#' in href:
href = ''.join(href.split('#')[:-1])
if 'https' in href or 'http' in href and href not in self.visited:
self.url_queue.append(href)
else:
new_url = urljoin(url.geturl(), href)
if new_url not in self.visited and new_url not in self.url_queue:
self.url_queue.append(new_url)
@timeout(0.7)
def read_robots(self, rp):
rp.read()
class ParallelValidate(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
""",
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, task, **kwargs):
"""
create tasks and add it to the work queue. Every task has specific arguments
"""
if task == 'initial_sim':
data = {
'sim_obj': kwargs['sim_obj'],
'ode_sys_opts': kwargs['ode_sys_opts'],
'id': kwargs['estimate_id'],
'y0': kwargs['y0'],
'task': task
}
elif task == 'perturbation_sim':
data = {
'sim_obj': kwargs['sim_obj'],
'ode_sys_opts': kwargs['ode_sys_opts'],
'id': kwargs['estimate_id'],
'y0': kwargs['y0'],
'perturbation_id': kwargs['perturbation_id'],
'task': task
}
self.work_queue.add_work(data)
def run_all(self, task, **kwargs):
"""parallel application core"""
if task == 'initial_sim':
estimates = kwargs['parameters']
estimate_id = kwargs['estimate_info']
sim_obj = kwargs['sim_obj']
for j_index, (j_estimate, j_estimate_id) in enumerate(
zip(estimates, estimate_id)):
self.__add_next_task(task=task,
**{
'sim_obj': sim_obj,
'ode_sys_opts': j_estimate,
'estimate_id': j_estimate_id,
'y0': sim_obj.wt_y0
})
# Keeep starting slaves as long as there is work to do
results = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
task, output = slave_return_data
if task == 'initial_sim':
j_slave_t, j_slave_y, j_slave_f, estimate_id, sample_id, data_set_id, sim_obj, ode_sys_opts = output
j_slave_dyn = {
't': j_slave_t,
'y': j_slave_y,
'flux': j_slave_f,
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id
}
# info on bistability
if j_slave_y[-1, 0] > j_slave_y[-1, 1]:
stability_id = 1
elif j_slave_y[-1, 0] < j_slave_y[-1, 1]:
stability_id = 2
else:
stability_id = 0
# get ss values
j_slave_ss = {
'y': j_slave_y[-1, :],
'flux': j_slave_f[-1, :],
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'ssid': stability_id
}
i_slave_result = {
'dynamic': j_slave_dyn,
'ss': j_slave_ss,
'initial': True,
'perturbation': False
}
# append initial sim results
results.append(i_slave_result)
# create list of perturbated parameter for given parameter estimate in ode_sys_opts
perturbed_parameter_list = sim_obj.change_parameter_values(
sim_obj.test_perturbations, ode_sys_opts)
# add perturbation jobs to back of jobs queue
experiment_id = [
'experiment_{}'.format(parameter_id) for parameter_id,
_ in enumerate(perturbed_parameter_list)
]
for j_experiment_id, j_perturbation in zip(
experiment_id, perturbed_parameter_list):
self.__add_next_task(task='perturbation_sim',
**{
'sim_obj':
sim_obj,
'ode_sys_opts':
j_perturbation,
'estimate_id':
(estimate_id, sample_id,
data_set_id),
'perturbation_id':
j_experiment_id,
'y0':
j_slave_y[-1, :]
})
elif task == 'perturbation_sim':
j_slave_t, j_slave_y, j_slave_f, estimate_id, sample_id, data_set_id, perturbation_id = output
j_slave_dyn = {
't': j_slave_t,
'y': j_slave_y,
'flux': j_slave_f,
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'perturbation_id': perturbation_id
}
# info on bistability
if j_slave_y[-1, 0] > j_slave_y[-1, 1]:
stability_id = 1
elif j_slave_y[-1, 0] < j_slave_y[-1, 1]:
stability_id = 2
else:
stability_id = 0
# get ss values
j_slave_ss = {
'y': j_slave_y[-1, :],
'flux': j_slave_f[-1, :],
'estimate_id': estimate_id,
'sample_id': sample_id,
'data_set_id': data_set_id,
'perturbation_id': perturbation_id,
'ssid': stability_id
}
i_slave_result = {
'dynamic': j_slave_dyn,
'ss': j_slave_ss,
'initial': False,
'perturbation': True
}
# append perturbation results
results.append(i_slave_result)
return results
class ParallelProcess(object):
"""Class running multiple ode simulations by assigning jobs to different slaves
until all the work is done"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
""",
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, task, **kwargs):
"""
create tasks and add it to the work queue. Every task has specific arguments
"""
if task == 'ident':
data = {
'ident_fun': kwargs['ident_fun'],
'flux_id': kwargs['flux_id'],
'flux_choice': kwargs['flux_choice'],
'sample_id': kwargs['sample_id'],
'data_set_id': kwargs['data_set_id'],
'exp_data': kwargs['exp_data'],
'task': task
}
self.work_queue.add_work(data)
def run_all(self, task, **kwargs):
"""parallel application core"""
if task == 'ident':
original_df = kwargs['exp_df']
# remove experiment_id as index
reset_df = original_df.reset_index('experiment_id')
ident_fun = kwargs['ident_fun']
flux_id = kwargs['flux_id']
flux_choice = kwargs['flux_choice']
idx = pd.IndexSlice
all_df_indices = reset_df.index.unique().tolist()
# create tuple of indices
# import pdb; pdb.set_trace()
for j_index, sample_data_set_id in enumerate(all_df_indices):
j_exp_data_set = reset_df.loc[
idx[sample_data_set_id],
['acetate', 'pep', 'fdp', 'E', 'v1', 'v2', 'v3', 'v5'
]].values.tolist()
flat_data_list = [
i_element for i_data in j_exp_data_set
for i_element in i_data
]
self.__add_next_task(task=task,
**{
'exp_data': flat_data_list,
'ident_fun': ident_fun,
'flux_id': flux_id,
'flux_choice': flux_choice,
'sample_id': sample_data_set_id[0],
'data_set_id': sample_data_set_id[1]
})
# Keeep starting slaves as long as there is work to do
results = []
while not self.work_queue.done():
# give more work to do to each idle slave (if any)
self.work_queue.do_work()
# reclaim returned data from completed slaves
for slave_return_data in self.work_queue.get_completed_work():
task, output = slave_return_data
if task == 'ident':
ident_info, slave_flux_id, slave_flux_choice, sample_id, data_set_id = output
i_slave_result = {
'flux_id': slave_flux_id,
'flux_choice': slave_flux_choice,
'sample_id': sample_id,
'data_set_id': data_set_id,
'ident_info': ident_info
}
results.append(i_slave_result)
print('Master: slave finished task %s returning: %s)' %
(task, str(data_set_id)))
return results
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done. There different type of work so
the slaves must be able to do different tasks
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def __add_next_task(self, i, task=None):
"""
we create random tasks 1-3 and add it to the work queue
Every task has specific arguments
"""
if task is None:
task = random.randint(1, 3)
if task == 1:
args = i
data = (Tasks.TASK1, args)
elif task == 2:
args = (i, i * 2)
data = (Tasks.TASK2, args)
elif task == 3:
args = (i, 999, 'something')
data = (Tasks.TASK3, args)
self.work_queue.add_work(data)
def run(self, tasks=100):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
for i in range(tasks):
self.__add_next_task(i)
#
# Keeep starting slaves as long as there is work to do
#
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
#
# each task type has its own return type
#
task, data = slave_return_data
if task == Tasks.TASK1:
done, arg1 = data
elif task == Tasks.TASK2:
done, arg1, arg2, arg3 = data
elif task == Tasks.TASK3:
done, arg1, arg2 = data
if done:
print('Master: slave finished is task returning: %s)' %
str(data))
# sleep some time
time.sleep(0.3)
class MyApp(object):
"""
This is my application that has a lot of work to do so it gives work to do
to its slaves until all the work is done
"""
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
self.master.terminate_slaves()
def run(self, tasks=None):
"""
This is the core of my application, keep starting slaves
as long as there is work to do
"""
#
# let's prepare our work queue. This can be built at initialization time
# but it can also be added later as more work become available
#
#version 1:
#PB_fn = os.path.join(os.environ['DRONES_DIR'], 'obiwan_analysis/brickstat/FinishedBricks.txt')
#ntasks = len(np.loadtxt(PB_fn,dtype=n.str).transpose())
#print('total of %d tasks' % ntasks)
#version1 end
#version 2:
import glob
from astropy.table import vstack
paths = glob.glob(os.path.join(os.environ['NGC_tractor'],'*','*'))
final_tab = None
n=0
for path in paths:
brickname = os.path.basename(path)
self.work_queue.add_work(data=(n, brickname))
n+=1
#version 1:
#for i in range(ntasks):
# 'data' will be passed to the slave and can be anything
#self.work_queue.add_work(data=(i, i))
#version 1 end
#
# Keeep starting slaves as long as there is work to do
#
final_table = None
tab = None
while not self.work_queue.done():
#
# give more work to do to each idle slave (if any)
#
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, tab = slave_return_data
print('No %d is done' % done)
if tab is not None:
if final_table is not None:
final_table = vstack([final_table, tab])
else:
final_table = tab
# sleep some time
time.sleep(0.3)
print('writing all the output to one table...')
final_table.write(os.path.join(os.environ['obiwan_out'],'subset','ngc_sim.fits'), format='fits',overwrite=True)
print('done!')
