def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list)
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
# loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0)
if rank == 0:
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
if completechunk.upper() in ['T', 'TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers, log = readArchiveXML(caseroot, tseries_input_rootdir,
tseries_output_rootdir, case,
options.standalone, completechunk,
debug, debugMsg)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = scomm.partition(specifiers,
func=partition.Duplicate(),
involved=True)
if rank == 0:
debugMsg("# of Specifiers: " + str(len(specifiers)),
header=True,
verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0, len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,
lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1, lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0
def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list);
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
generate_all = cesmEnv['TIMESERIES_GENERATE_ALL']
if completechunk.upper() in ['T','TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers,log = readArchiveXML(caseroot, tseries_input_rootdir, tseries_output_rootdir,
case, options.standalone, completechunk, generate_all,
debug, debugMsg, scomm, rank, size)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
if rank == 0:
debugMsg("# of Specifiers: "+str(len(specifiers)), header=True, verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0,len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1,lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0