def test_init():
if not initialized():
initialize()
from mpi4py import MPI
assert initialized() == True
assert MPI.Is_initialized() == True
assert finalized() == False
assert MPI.Is_finalized() == False
def myexit():
# Tell children to end
if is_master_process(): MPI_done()
## TODO: By telling children to exit, it may take a while for them to finish..
global out
if out is not None:
out.close() # close this -- meaning the subprocess is told to stop
if not MPI.Is_finalized():
MPI.Finalize()
def fetch_minibatch_data(self,
mode,
pop=False,
fetch_meta_data=False,
data_shape=None,
channels="last"):
# Return a dictionary object with keys 'image', 'label', and others as needed
# self._queueloaders['train'].fetch_data(keyword_label).dim() as an example
# Don't try to pull data if MPI is finalized, since IO is shutdown:
if MPI.Is_finalized():
return None
if self._count[mode] != 0:
if self._warning:
print(
"Calling fetch_minibatch_data without calling prepare_next. This will not give new data."
)
print(
"To quiet this wanring, call prepare_next before fetch_minibatch_data or call queueloader.no_warnings()"
)
if pop:
# This function will pop the data
while self._queueloaders[mode].is_reading():
time.sleep(0.01)
self._queueloaders[mode].pop_current_data()
else:
if self._warning:
print(
"Calling fetch_minibatch_data with pop = False. This will give you the same data as last time."
)
print("To quiet this warning, call queueloader.no_warnings()")
self._queueloaders[mode].next(store_entries=fetch_meta_data,
store_event_ids=fetch_meta_data)
this_data = {}
for key in self._data_keys[mode]:
this_data[key] = self._queueloaders[mode].fetch_data(
self._data_keys[mode][key]).data(shape=data_shape,
channels=channels)
# this_data[key] = numpy.reshape(this_data[key], self._dims[mode][key])
if fetch_meta_data:
this_data['entries'] = self._queueloaders[mode].fetch_entries()
this_data['event_ids'] = self._queueloaders[mode].fetch_event_ids()
self._count[mode] += 1
return this_data
def onexit():
if not MPI.Is_initialized() or MPI.Is_finalized():
return
# Get the current exception (if any)
exc = excepthook.exception
# If we are exiting normally then call MPI_Finalize
if (MPI.COMM_WORLD.size == 1 or exc is None
or isinstance(exc, KeyboardInterrupt)
or (isinstance(exc, SystemExit) and exc.code == 0)):
MPI.Finalize()
# Otherwise forcefully abort
else:
MPI.COMM_WORLD.Abort(1)
def coordinate_next_batch_indexes(self, mode, comm, root_rank=0):
'''
This function is a little naieve (sp?). But it will take the root rank,
use it to determine the next batch indexes, then scatter those indexes.
It will scatter it evenly to the entire comm that is passed in.
'''
# Don't coordinate if MPI is finalized:
if MPI.Is_finalized():
return None
comm_size = comm.Get_size()
if comm.Get_rank() == root_rank:
set_entries = self.get_next_batch_indexes(
mode, self._minibatch_size[mode])
# Create a buffer for the data to scatter
# Which is just a reference to the numpy array
sendbuff = None
if comm.Get_rank() == root_rank:
sendbuff = set_entries
local_size = int(self._minibatch_size[mode] / comm_size)
if (self._minibatch_size[mode] % comm_size != 0):
print(
'You have requested to scatter {} image(s) over {} ranks. This is not possible as the number are not divisible.'
.format(self._minibatch_size[mode], comm_size))
raise Exception(
"Please change either the minibatch size or the number or ranks to scatter to."
)
# The recvbuff must be properly sized:
recvbuff = numpy.empty((local_size), dtype=numpy.int32)
# Scatterv will scatter numpy arrays, and note the automatic lookup of
# dtypes from numpy to MPI. If you are getting crazy, undefined or NaN values,
# the dtype is a good start for investigating.
comm.Scatter(sendbuff, recvbuff, root=root_rank)
#print (self._rank, self._local_rank, 'done Scatterv!')
return recvbuff
def exit(self, status=0):
"""Exit the mpi4py processor with the given status.
@keyword status: The program exit status.
@type status: int
"""
# Execution on the slave.
if MPI.COMM_WORLD.rank != 0:
# Catch sys.exit being called on an executing slave.
if self.in_main_loop:
raise Exception('sys.exit unexpectedly called on slave!')
# Catch sys.exit
else:
sys.stderr.write('\n')
sys.stderr.write('***********************************************\n')
sys.stderr.write('\n')
sys.stderr.write('warning sys.exit called before mpi4py main loop\n')
sys.stderr.write('\n')
sys.stderr.write('***********************************************\n')
sys.stderr.write('\n')
MPI.COMM_WORLD.Abort()
# Execution on the master.
else:
# Slave clean up.
if MPI.Is_initialized() and not MPI.Is_finalized() and MPI.COMM_WORLD.rank == 0:
# Send the exit command to all slaves.
self._broadcast_command(Exit_command())
# Dump all results.
self._ditch_all_results()
# Exit the program with the given status.
sys.exit(status)
import os from mpi4py import rc assert rc.initialize is True assert rc.finalize is None assert rc.thread_level == 'multiple' os.environ['MPI4PY_RC_INITIALIZE'] = 'false' os.environ['MPI4PY_RC_FINALIZE'] = 'off' os.environ['MPI4PY_RC_THREAD_LEVEL'] = 'single' from mpi4py import MPI assert not MPI.Is_initialized() assert not MPI.Is_finalized() assert rc.initialize is False assert rc.finalize is False assert rc.thread_level == 'single'
def test_finalized_mpi4py(self):
# test mpi finalization (automatically when including mpi4py, but only just before the Python process terminates)
self.assertFalse(mpi.Is_finalized())
def testIsFinalized(self):
flag = MPI.Is_finalized()
self.assertTrue(type(flag) is bool)
self.assertFalse(flag)
def cleanup():
global init_by_devito
if init_by_devito and MPI.Is_initialized() and not MPI.Is_finalized():
MPI.Finalize()
# mpi4py.rc(initialize=False) # or use the following way # we must set the initialization and runtime configuration options # before the import of MPI module from mpi4py import rc # the default of rc.initialize is True and rc.finalize is None # change rc.initialize to False # rc.finalize will also be False when it value is None rc.initialize = False from mpi4py import MPI print 'Before call Init(), MPI.Is_initialized: %s' % MPI.Is_initialized() print 'Before call Init(), MPI.Is_finalized: %s' % MPI.Is_finalized() # now we have to call Init explicitly MPI.Init() print 'After call Init(), MPI.Is_initialized: %s' % MPI.Is_initialized() print 'After call Init(), MPI.Is_finalized: %s' % MPI.Is_finalized() comm = MPI.COMM_WORLD rank = comm.rank print 'I am Process %d...' % rank # have to call Finalize explicitly to terminate MPI MPI.Finalize() print 'After call Finalize(), MPI.Is_initialized: %s' % MPI.Is_initialized() print 'After call Finalize(), MPI.Is_finalized: %s' % MPI.Is_finalized()
def finalize_process_group():
if not MPI.Is_finalized():
MPI.Finalize()
def finalize():
if not MPI.Is_finalized():
MPI.Finalize()
def INNT():
# Initializing the MPI and testing if it's been initialized
MPI.Init()
print(MPI.Is_initialized())
print(MPI.Is_finalized())
# Get Parameters
generation, dataset, mutationChance, param, groupSize = getParameters()
# Initialize the fitness
fitnessParent = -1
# The fitness of the parent
fitnessChild = -1
# The fitness of the child
networkFitness = -1
# The fitness of the network
genBestFitness = -1
# Fitness of the generation
# Initialize the classes
net, ga, com, pd = initClasses(param, MPI, groupSize, networkFitness)
# Get the logger
# filename = 'output{}.log'.format(pd.rank)
filename = 'output.log'
logger = logging.getLogger()
handler = logging.FileHandler(filename)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
# Split the communicator
subGroup = pd.rank / groupSize
subComm = MPI.Comm.Split(MPI.COMM_WORLD, subGroup, pd.rank)
# initialize the networks
# one random network at every processor
# INITIALIZZEEEE ISLAND WITH SOME SPECIALITYYYYY
data = net.initNetwork()
# Islands differ in activation function
# Since there will be at min 2 subgroups
if pd.subGroup == 0:
data['activation'] = 'sigmoid'
elif pd.subGroup == 1:
data['activation'] = 'elu'
else:
data['activation'] = 'selu'
# Start running GA (Genetic Algorithm) generation
for g in range(generation):
if genBestFitness < 100:
# GET PARENT FITNESS/ACCURACY
# Every processor trains and evaluate the accuracy/fitness of the parent network
fitnessParent = ga.getFitness(data, dataset)
print('loop_1 done', g, pd.rank)
# BREED THE CHILD
# This to be done using MPI ISend
# Get the parent using Non Blocking exchange
child = ga.breeding(pd.rank, g, data, mutationChance,
pd.intraIslandExchange(data, subComm))
MPI.COMM_WORLD.Barrier()
# GET CHILD'S FITNESS/ACCURACY
# Every processor trains and evaluate the accuracy/fitness of the child network
fitnessChild = ga.getFitness(child, dataset)
'''
If the network fitness has improved over previous generation,
then pass on the features/hyperparameters
Pass on the better of the two (parent or child) from this generation to the next generation
Comparison done - of the previous value at the procecssor with the new computed value
'''
networkFitness, data = com.networkData(fitnessParent, fitnessChild,
data, child)
'''
Compare the fitness of the best networks of all the families
Compares the fitness of all the networks data that are with all the processors in the communication
Get the best fitness the generation
Kill the poorest performing of the population
Randomly initialize the poorest fitness population to keep the population constant
'''
genBestFitness, data = com.genFitness(data, param, MPI, groupSize)
# print(genBestFitness, data)
logger.debug(
'generation=%d, Rank=%d, processid=%s, group=ID%d, subRank=%d, parent=%s, child=%s, parentFitness=%0.4f, childFitness=%0.4f, networkFitness=%0.4f, genBestFitness=%0.4f',
g, pd.rank, socket.gethostname(), pd.subGroup,
subComm.Get_rank(), data, child, fitnessParent, fitnessChild,
networkFitness, genBestFitness)
'''
Do inter-island exchange after every 5 generations
In this all the ranks are sending the data to the previous ranks
'''
if g % 5 == 0:
pd.interIslandExchange(data, subComm)
print('loop_6 done', pd.rank)
MPI.COMM_WORLD.Barrier()
else:
# Broadcast the best results to all the processors
pd.broadcast(data, pd.rank)
print('best fitness achieved')
# And halt
MPI.Finalize()
MPI.Finalize()
def DNNT():
# Initializing the MPI and testing if it's been initialized
MPI.Init()
print(MPI.Is_initialized())
print(MPI.Is_finalized())
# Get Parameters
generation, dataset, mutationChance, param = getParameters()
# Initialize the fitness
fitnessParent = -1
# The fitness of the parent
fitnessChild = -1
# The fitness of the child
networkFitness = -1
# The fitness of the network
genBestFitness = -1
# Fitness of the generation
# Initialize the classes
net, ga, com, pd = initClasses(param, MPI, networkFitness)
# Get the logger
# filename = 'output{}.log'.format(pd.rank)
filename = 'output.log'
logger = logging.getLogger()
handler = logging.FileHandler(filename)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
# initialize the networks
# one random network at every processor
data = net.initNetwork()
# Start running GA (Genetic Algorithm) generation
for g in range(generation):
if genBestFitness < 100:
# GET PARENT FITNESS/ACCURACY
# Every processor trains and evaluate the accuracy/fitness of the parent network
fitnessParent = ga.getFitness(data, dataset)
# BREED THE CHILD
# This to be done using MPI ISend
# Get the parent using Non Blocking exchange
child = ga.breeding(data, mutationChance,
pd.nonBlockingExchange(data))
MPI.COMM_WORLD.Barrier()
# GET CHILD'S FITNESS/ACCURACY
# Every processor trains and evaluate the accuracy/fitness of the child network
fitnessChild = ga.getFitness(child, dataset)
'''
If the network fitness has improved over previous generation,
then pass on the features/hyperparameters
Pass on the better of the two (parent or child) from this generation to the next generation
Comparison done of the previous value at the procecssor with the new computed value
'''
networkFitness, data = com.networkData(fitnessParent, fitnessChild,
data, child)
logger.debug(
'generation=%d, Rank=%d, processid=%s, parent=%s, child=%s, '
'parentFitness=%0.4f, childFitness=%0.4f, networkFitness=%0.4f',
g, pd.rank, socket.gethostname(), data, child, fitnessParent,
fitnessChild, networkFitness)
'''
Compare the fitness of the best networks of all the families
Compares the fitness of all the networks data that are with all the processors in the communication
Get the best fitness the generation
Kill the poorest performing of the population
Randomly initialize the poorest fitness population to keep the population constant
'''
genBestFitness, data = com.genFitness(data, param, MPI)
print(genBestFitness, data)
else:
# Broadcast the best results to all the processors
pd.broadcast(data, pd.rank)
print('best fitness achieved')
# And halt
MPI.Finalize()
MPI.Finalize()
def finalize():
if not MPI.Is_finalized():
# print "finalizing..."
MPI.Finalize()
else:
pass
