def bind_local_sino(self, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
if ThisHost.is_master():
logger.info("On Master node, skip bind local sino.")
return
else:
logger.info(
"On Worker node, local data for worker {}.".format(task_index))
worker_sinos = self.load_local_sino(task_index)
self.worker_graphs[task_index].init_sino(worker_sinos)
def bind_local_data(self, data_info, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
if ThisHost.is_master():
logger.info("On Master node, skip bind local data.")
return
else:
logger.info(
"On Worker node, local data for worker {}.".format(task_index))
emap, lors = self.load_local_data(data_info, task_index)
self.worker_graphs[task_index].assign_efficiency_map(emap)
self.worker_graphs[task_index].assign_lors(lors)
def bind_local_matrix(self, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
if ThisHost.is_master():
logger.info("On Master node, skip bind local matrix.")
return
else:
logger.info(
"On Worker node, local data for worker {}.".format(task_index))
worker_matrix = self.load_local_matrix(task_index)
worker_matrix = sparse.coo_matrix(worker_matrix)
self.worker_graphs[task_index].init_matrix(worker_matrix)
def sino_shape(self, task_index=None):
if task_index is None:
task_index = ThisHost().host().task_index
if isinstance(self._sino_shape, (list, tuple)):
return self._sino_shape
else:
return self._sino_shape[task_index]
def main_summation(job, task):
cfg = {"master": ["localhost:2221"],
"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'master', 0)
master_host = Master.master_host()
this_host = ThisHost.host()
host0 = Host('worker', 0)
host1 = Host('worker', 1)
hmi = DistributeGraphInfo(None, None, None, master_host)
tm = TensorNumpyNDArray([1.0], None,
DistributeGraphInfo.from_graph_info(hmi, name='t0'))
t0c = tm.copy_to(host0)
t1c = tm.copy_to(host1)
m_sum = Summation(name='summation', graph_info=DistributeGraphInfo(
'summation', None, None, host0))([t0c, t1c])
make_distribute_session()
if task == 0:
ptensor(tm)
Server.join()
if task == 1:
ptensor(tm, 'tm')
ptensor(t0c, 't0c')
ptensor(t1c, 't1c')
ptensor(m_sum)
def main_basic(job, task):
cfg = {"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'worker', 0)
master_host = Master.master_host()
this_host = ThisHost.host()
host1 = Host(job, 1)
hmi = DistributeGraphInfo(None, None, None, master_host)
with tf.variable_scope('scope_test'):
t0 = TensorVariable(VariableInfo(None, [1], tf.float32),
hmi.update(name='v0'))
aop = tf.assign(t0.data, tf.constant([3.]))
t1 = TensorNumpyNDArray([1.0], None,
hmi.update(name='v1'))
t1c = t1.copy_to(host1)
t1p = Tensor(t1c.data + 1, t1c.data_info,
t1c.graph_info.update(name='t1_plus'))
make_distribute_session()
if task == 0:
ptensor(t1)
Server.join()
if task == 1:
ptensor(t1)
ptensor(t1c)
ptensor(t1p)
ptensor(t0)
ThisSession.run(aop)
ptensor(t0)
def lors_shapes(self, axis, task_index=None):
if task_index is None:
task_index = ThisHost().host().task_index
if isinstance(self._lors_shapes[axis], (list, tuple)):
return self._lors_shapes[axis]
else:
return self._lors_shapes[axis][task_index]
def sino_range(self, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
if self._sino_range is not None:
return self._maybe_broadcast_value(self._sino_range, task_index)
else:
return None
def sino_file(self, task_index=None):
if task_index is None:
task_index = ThisHost().host().task_index
if isinstance(self._sino_file, str):
return self._sino_file
else:
return self._sino_file[task_index]
def main_sync(job, task):
cfg = {"master": ["localhost:2221"],
"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'master', 0)
master_host = Master.master_host()
this_host = ThisHost.host()
host0 = Host('worker', 0)
host1 = Host('worker', 1)
def sleep(ips):
for i in range(5, 0, -1):
time.sleep(1)
return 0
# hmi = DistributeGraphInfo(None, None, None, master_host)
tm = TensorNumpyNDArray([1.0], None,
DistributeGraphInfo.from_graph_info(hmi, name='t0'))
tcs = []
# t0c = tm.copy_to(host0)
# t1c = tm.copy_to(host1)
# m_sum = Summation(name='summation', graph_info=DistributeGraphInfo(
# 'summation', None, None, host0))([t0c, t1c])
ops = tf.FIFOQueue(2, tf.bool, shapes=[],
name='barrier', shared_name='barrier')
# ptensor(tm)
if ThisHost.host() == master_host:
join = ops.dequeue_many(2)
else:
signal = ops.enqueue(False)
no = tf.constant('tmp')
ops = [tf.Print(no, data=[no], message='Done_{}'.format(i), name='p_{}'.format(i))
for i in range(3)]
# ops.enqueue()
make_distribute_session()
if ThisHost.host() == master_host:
ThisSession.run(join)
print('Joined.')
time.sleep(2)
ThisSession.run(ops[0])
# Server.join()
elif ThisHost.host() == host0:
ThisSession.run(signal)
ThisSession.run(ops[1])
elif ThisHost.host() == host1:
time.sleep(3)
ThisSession.run(signal)
ThisSession.run(ops[2])
def _maybe_broadcast_ints(self, value, task_index):
if task_index is None:
task_index = ThisHost().host().task_index
else:
task_index = int(task_index)
if len(value) <= task_index or isinstance(value[task_index], int):
return value
return value[task_index]
def lors_files(self, axis, task_index=None):
if task_index is None:
task_index = ThisHost().host().task_index
# print("lorfiles!!!!!!!!!!!!!!!!!!!!!!!!!:", self._lors_files, type(self._lors_files))
if isinstance(self._lors_files[axis], str):
return self._lors_files[axis]
else:
return self._lors_files[axis][task_index]
def main_sync_2(job, task):
cfg = {"master": ["localhost:2221"],
"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'master', 0)
master_host = Master.master_host()
this_host = ThisHost.host()
hosts = [Host('worker', i) for i in range(2)]
hmi = DistributeGraphInfo(None, None, None, master_host)
tm = TensorNumpyNDArray([0.0], None,
DistributeGraphInfo.from_graph_info(hmi, name='tm'))
t_local_var = []
t_local_copied = []
for h in hosts:
ta, tv = tm.copy_to(h, True)
t_local_copied.append(ta)
t_local_var.append(tv)
t_local_plus = [TensorTest.from_(t) for t in t_local_copied]
for i in range(1):
t_local_plus = [t.add_one() for t in t_local_plus]
t_write_back = []
for i in range(len(hosts)):
t_write_back.append(t_local_var[i].assign(t_local_plus[i]))
t_global_pluses = [t.copy_to(master_host) for t in t_local_var]
sm = Summation(name='summation', graph_info=hmi.update(name='summatin'))
t_res = sm(t_global_pluses)
br = Barrier('barrier', hosts)
# ops = tf.FIFOQueue(2, tf.bool, shapes=[],
# name='barrier', shared_name='barrier')
# join = ops.dequeue_many(2)
# signal = ops.enqueue(False)
make_distribute_session()
if ThisHost.host() == master_host:
# ThisSession.run(join)
# ThisSession.run(br)
br.run()
ptensor(t_res)
else:
time.sleep(5)
ptensor(t_local_plus[task])
ptensor(t_write_back[task])
# ThisSession.run(signal)
# ThisSession.run(br)
br.run()
Server.join()
def bind_local_data(self):
"""
bind the static effmap data
"""
return
map_file = self.work_directory + self.image_info.map_file
#matrix_file = self.work_directory + self.Inputinfo.sm
task_index = ThisHost.host().task_index
if ThisHost.is_master():
logger.info("On Master node, skip bind local data.")
return
else:
logger.info(
"On Worker node, local data for worker {}.".format(task_index))
emap = self.load_local_effmap(map_file)
self.worker_graphs[task_index].init_efficiency_map(emap)
self.bind_local_sino()
self.bind_local_matrix()
def _maybe_broadcast_value(self,
value,
task_index=None,
valid_type=(list, tuple)):
if task_index is None:
task_index = ThisHost.host().task_index
if isinstance(value, valid_type):
return value[task_index]
else:
return value
def lors_ranges(self, axis, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
if self._lors_ranges is not None:
return self._maybe_broadcast_value(self._lors_ranges[axis], task_index)
elif self._lors_steps is not None:
step = self._maybe_broadcast_value(
self._lors_steps[axis], task_index)
return [task_index * step, (task_index + 1) * step]
else:
return None
def main(job, task):
tf.logging.set_verbosity(0)
cfg = {"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'worker', 0)
# # if task == 1:
# # time.sleep(10)
# with tf.device(Master.master_host().device_prefix()):
# with tf.variable_scope('test'):
# t1 = tf.get_variable('var', [], tf.float32)
master_host = Master.master_host()
this_host = ThisHost.host()
host2 = Host(job, 1)
hmi = DistributeGraphInfo(None, None, None, master_host)
with tf.variable_scope('scope_test'):
t0 = TensorVariable(VariableInfo(None, [1], tf.float32),
DistributeGraphInfo.from_(hmi, name='t1'))
aop = tf.assign(t0.data, tf.constant([3.]))
t1 = TensorNumpyNDArray([1.0], None,
DistributeGraphInfo.from_(hmi, name='t1_copy'))
t1c = t1.copy_to(host2)
t1p = Tensor(t1c.data + 1, t1c.data_info, DistributeGraphInfo.from_(t1c.graph_info, name='t1_plus'))
# t2 = t0.copy_to(host2)
make_distribute_session()
if task == 0:
# ThisSession.run(tf.global_variables_initializer())
ptensor(t1)
Server.join()
if task == 1:
ptensor(t1)
ptensor(t1c)
ptensor(t1p)
# print(t2.run())
# print(t2.data)
# print(t0.run())
# print(t0)
ptensor(t0)
print(ThisSession.run(aop))
ptensor(t0)
def run(self):
KS = self.KEYS.STEPS
self.run_step_of_this_host(KS.INIT)
logger.info('STEP: {} done.'.format(KS.INIT))
nb_iterations = self.Reconinfo.nb_iterations
#nb_subsets = self.Reconinfo.nb_subsets
image_name = self.image_info.name
for i in tqdm(range(nb_iterations), ascii=True):
self.run_step_of_this_host(KS.RECON)
logger.info('STEP: {} done.'.format(KS.RECON))
self.run_step_of_this_host(KS.MERGE)
logger.info('STEP: {} done.'.format(KS.MERGE))
self.run_and_print_if_not_master(
self.worker_graphs[ThisHost.host().task].tensor(
self.worker_graphs[0].KEYS.TENSOR.RESULT))
self.run_and_print_if_is_master(self.master_graph.tensor('x'))
self.run_and_save_if_is_master(self.master_graph.tensor('x'),
image_name + '_{}.npy'.format(i))
logger.info('Recon {} steps done.'.format(nb_iterations))
def main_add_one(job, task):
cfg = {"worker": ["localhost:2222",
"localhost:2223"]}
make_distribute_host(cfg, job, task, None, 'worker', 0)
master_host = Master.master_host()
this_host = ThisHost.host()
host1 = Host(job, 1)
hmi = DistributeGraphInfo(None, None, None, master_host)
with tf.variable_scope('scope_test'):
t0 = TensorNumpyNDArray([1.0], None,
hmi.update(name='v0'))
t1 = TensorTest.from_(t0)
t2 = t1.add_one()
make_distribute_session()
if task == 0:
ptensor(t0)
ptensor(t1)
ptensor(t2)
Server.join()
if task == 1:
ptensor(t0)
ptensor(t1)
ptensor(t2)
def run_and_save_if_is_master(self, x, path):
if ThisHost.is_master():
if isinstance(x, Tensor):
x = x.data
result = ThisSession.run(x)
np.save(path, result)
def run_and_print_if_is_master(self, x):
if ThisHost.is_master():
if isinstance(x, Tensor):
x = x.data
result = ThisSession.run(x)
print(result)
def lors_steps(self, axis, task_index=None):
if task_index is None:
task_index = ThisHost.host().task_index
return self._lors_steps[axis]
def sino_steps(self, task_index = None):
if task_index is None:
task_index = ThisHost.host().task_index
return self._sino_steps
def matrix_steps(self, task_index = None):
if task_index is None:
task_index = ThisHost.host().task_index
return self._matrix_steps