def _receiver(self):
"""Continuously receives futures from workers until the shutdown
sentinel is set.
"""
log.debug('Manager._receiver()')
assert MPI.Is_thread_main() is False
assert threading.currentThread().getName() == "receiver"
while not self.shutItDown:
# are we waiting on any results?
while self.nPending:
stat = MPI.Status()
(tid, msg, val) = self.comm.recv(source=MPI.ANY_SOURCE,
tag=self.result_tag,
status=stat)
log.debug('Manager._receiver received task: %s' % tid)
# update future
ft = self.pending_futures.pop(tid)
if msg == 'exception':
ft._set_exception(*val)
else:
ft._set_result(val)
with self.lock:
self.dests.append(stat.Get_source())
self.nPending -= 1
# force context switching ( 1ms )
time.sleep(0.001)
def _dispatcher(self):
"""Continuously dispatches tasks to idle destinations until the
shutdown sentinel is set.
"""
log.debug('Manager._dispatcher()')
assert MPI.Is_thread_main() is False
assert threading.currentThread().getName() == "dispatcher"
while not self.shutItDown:
req = []
# do we have work and somewhere to send it?
while self.tasks and self.dests:
with self.lock:
task = self.tasks.popleft()
sendTo = self.dests.popleft()
self.nPending += 1
req.append(
self.comm.isend(task, dest=sendTo, tag=self.task_tag))
# make sure all sends completed
MPI.Request.Waitall(requests=req)
# force context switching ( 1ms )
time.sleep(0.001)
def testIsThreadMain(self):
try:
flag = MPI.Is_thread_main()
except NotImplementedError:
self.skipTest('mpi-is_thread_main')
name = threading.current_thread().name
main = (name == 'MainThread') or not HAVE_THREADING
self.assertEqual(flag, main)
if VERBOSE:
log = lambda m: sys.stderr.write(m + '\n')
log("%s: MPI.Is_thread_main() -> %s" % (name, flag))
def testIsThreadMain(self, main=True):
try:
flag = MPI.Is_thread_main()
except NotImplementedError:
return
self.assertEqual(flag, main)
if _VERBOSE:
thread = current_thread()
name = thread.getName()
log = lambda m: sys.stderr.write(m + '\n')
log("%s: MPI.Is_thread_main() -> %s" % (name, flag))
def _test_is(self, main=False):
try:
flag = MPI.Is_thread_main()
except NotImplementedError:
return
self.assertEqual(flag, main)
if _VERBOSE:
from sys import stderr
thread = current_thread()
name = thread.getName()
log = lambda m: stderr.write(m + '\n')
log("%s: MPI.Is_thread_main() -> %s" % (name, flag))
def __new__(cls):
"""Creates a Serial WorkManager if size is 1. Otherwise creates a
single Manager and size-1 Worker.
"""
log.debug('MPIWorkManager.__new__()')
assert MPI.Is_initialized()
assert MPI.Is_thread_main()
rank = MPI.COMM_WORLD.Get_rank()
size = MPI.COMM_WORLD.Get_size()
if size == 1:
return super().__new__(Serial)
elif rank == 0:
return super().__new__(Manager)
else:
return super().__new__(Worker)
if (rank == 0):
print("[HUGECTR][INFO] iter: {}; loss: {}".format(i, loss))
if (i % 1000 == 0 and i != 0):
sess.check_overflow()
sess.copy_weights_for_evaluation()
data_reader_eval = sess.get_data_reader_eval()
for _ in range(solver_config.max_eval_batches):
sess.eval()
metrics = sess.get_eval_metrics()
print("[HUGECTR][INFO] rank: {}, iter: {}, {}".format(
rank, i, metrics))
return
if __name__ == "__main__":
json_file = sys.argv[1]
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
thread = threading.Thread(target=session_impl_test,
args=(json_file, ),
name='[rank-%d train]' % rank)
current_thread = threading.currentThread()
print('[HUGECTR][INFO] %s is main thread: %s' %
(current_thread.name, MPI.Is_thread_main()))
print('[HUGECTR][INFO] before: rank %d ' % (rank))
# start the thread
thread.start()
# wait for terminate
thread.join()
print('[HUGECTR][INFO] after: rank %d ' % (rank))
sess.set_learning_rate(lr)
sess.train()
if (i%100 == 0):
loss = sess.get_current_loss()
if (rank == 0):
print("[HUGECTR][INFO] iter: {}; loss: {}".format(i, loss))
if (i%1000 == 0 and i != 0):
sess.check_overflow()
sess.copy_weights_for_evaluation()
data_reader_eval = sess.get_data_reader_eval()
for _ in range(solver_config.max_eval_batches):
sess.eval()
metrics = sess.get_eval_metrics()
print("[HUGECTR][INFO] rank: {}, iter: {}, {}".format(rank, i, metrics))
return
if __name__ == "__main__":
json_file = sys.argv[1]
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
thread = threading.Thread(target=session_impl_test, args = (json_file,), name='[rank-%d train]' % rank)
current_thread = threading.currentThread()
print('[HUGECTR][INFO] %s is main thread: %s' % (current_thread.name, MPI.Is_thread_main()))
print('[HUGECTR][INFO] before: rank %d '% (rank))
# start the thread
thread.start()
# wait for terminate
thread.join()
print('[HUGECTR][INFO] after: rank %d ' % (rank))
def recv():
current_thread = threading.currentThread()
print '%s is main thread: %s' % (current_thread.name, MPI.Is_thread_main())
comm.Recv(recv_buf, source=other, tag=11)
print '%s receives %d from rank %d' % (current_thread.name, recv_buf,
other)
def send():
current_thread = threading.currentThread()
print '%s is main thread: %s' % (current_thread.name, MPI.Is_thread_main())
print '%s sends %d to rank %d...' % (current_thread.name, rank, other)
comm.Send(np.array(rank), dest=other, tag=11)
current_thread = threading.currentThread()
print '%s is main thread: %s' % (current_thread.name, MPI.Is_thread_main())
print '%s sends %d to rank %d...' % (current_thread.name, rank, other)
comm.Send(np.array(rank), dest=other, tag=11)
def recv():
current_thread = threading.currentThread()
print '%s is main thread: %s' % (current_thread.name, MPI.Is_thread_main())
comm.Recv(recv_buf, source=other, tag=11)
print '%s receives %d from rank %d' % (current_thread.name, recv_buf,
other)
# create thread by using a function
send_thread = threading.Thread(target=send,
name='[rank-%d send_thread]' % rank)
recv_thread = threading.Thread(target=recv,
name='[rank-%d recv_thread]' % rank)
current_thread = threading.currentThread()
print '%s is main thread: %s' % (current_thread.name, MPI.Is_thread_main())
print 'before: rank %d has %d' % (rank, recv_buf)
# start the threads
send_thread.start()
recv_thread.start()
# wait for terminate
send_thread.join()
recv_thread.join()
print 'after: rank %d has %d' % (rank, recv_buf)
