def _run_convert(self,
infiles,
prefix,
suffix,
metadata,
ncfmt,
clevel,
serial,
verbosity,
wmode,
once,
print_diags=False):
if not (serial and self.rank > 0):
spec = Specifier(infiles=infiles,
ncfmt=ncfmt,
compression=clevel,
prefix=prefix,
suffix=suffix,
metadata=metadata)
rshpr = create_reshaper(spec,
serial=serial,
verbosity=verbosity,
wmode=wmode,
once=once)
rshpr.convert()
if print_diags:
rshpr.print_diagnostics()
MPI_COMM_WORLD.Barrier()
def testCreateReshaperType(self):
actual = create_reshaper(
self.spec, serial=False, verbosity=2, once=False)
expected = Slice2SeriesReshaper
msg = self._info_msg("type(create_reshaper)",
None, type(actual), expected)
self.assertEqual(type(actual), expected, msg)
def main(argv=None):
opts, specfile = cli(argv)
# Try importing the file
try:
spec = pickle.load(open(specfile, 'rb'))
except:
err_msg = "Specifier File '{}' could not be opened and read".format(
specfile)
raise RuntimeError(err_msg)
# Create the PyReshaper object
reshpr = reshaper.create_reshaper(spec,
serial=opts.serial,
verbosity=opts.verbosity,
wmode=opts.write_mode,
once=opts.once)
# Run the conversion (slice-to-series) process
reshpr.convert(output_limit=opts.limit,
rchunks=opts.rchunks,
wchunks=opts.wchunks)
# Print timing diagnostics
reshpr.print_diagnostics()
def main(options, scomm, rank, size):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# set the caseroot based on standalone or not
pp_caseroot = caseroot
if not options.standalone:
caseroot, pp_subdir = os.path.split(caseroot)
if rank == 0:
print('cesm_tseries_generator: caseroot = {0}'.format(caseroot))
# set the debug level
debug = options.debug[0]
# cesmEnv["id"] = "value" parsed from the CASEROOT/env_*.xml files
env_file_list = [
'env_case.xml', 'env_run.xml', 'env_build.xml', 'env_mach_pes.xml'
]
# check if the standalone option is set
if options.standalone:
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list)
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
# loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0)
if rank == 0:
specifiers = readArchiveXML(caseroot, cesmEnv['DOUT_S_ROOT'],
cesmEnv['CASE'], options.standalone, debug)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = scomm.partition(specifiers,
func=partition.Duplicate(),
involved=True)
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers,
serial=False,
verbosity=debug)
# Run the conversion (slice-to-series) process
reshpr.convert()
# Print timing diagnostics
reshpr.print_diagnostics()
# TO-DO check if DOUT_S_SAVE_HISTORY_FILES is true or false and
# delete history files accordingly
return 0
def testCreateReshaperType(self):
actual = create_reshaper(self.spec,
serial=True,
verbosity=2,
once=False)
expected = Slice2SeriesReshaper
msg = self._info_msg("type(create_reshaper)", None, type(actual),
expected)
self.assertEqual(type(actual), expected, msg)
def _test_create_reshaper(self, serial, verbosity, wmode):
self._test_header(("create_reshaper(serial={}, verbosity={}, "
"wmode={!r})").format(serial, verbosity, wmode))
if not (serial and self.rank > 0):
spec = Specifier(infiles=self.slices, ncfmt='netcdf',
compression=0, prefix='output.', suffix='.nc',
metadata=[])
rshpr = create_reshaper(spec, serial=serial, verbosity=verbosity,
wmode=wmode)
self._assertion("type(reshaper)", type(rshpr),
Slice2SeriesReshaper)
def _run_convert(self, infiles, prefix, suffix, metadata,
ncfmt, clevel, serial, verbosity, wmode, once,
print_diags=False):
if not (serial and self.rank > 0):
spec = Specifier(infiles=infiles, ncfmt=ncfmt, compression=clevel,
prefix=prefix, suffix=suffix, metadata=metadata)
rshpr = create_reshaper(spec, serial=serial,
verbosity=verbosity,
wmode=wmode, once=once)
rshpr.convert()
if print_diags:
rshpr.print_diagnostics()
MPI_COMM_WORLD.Barrier()
def _test_create_reshaper(self, serial, verbosity, wmode):
self._test_header(("create_reshaper(serial={0}, verbosity={1}, "
"wmode={2!r})").format(serial, verbosity, wmode))
if not (serial and self.rank > 0):
spec = Specifier(infiles=mkTestData.slices,
ncfmt='netcdf',
compression=0,
prefix='output.',
suffix='.nc',
metadata=[])
rshpr = create_reshaper(spec,
serial=serial,
verbosity=verbosity,
wmode=wmode)
self._assertion("type(reshaper)", type(rshpr), Reshaper)
def convert(self, print_diags=False):
if self.create_args.get('verbosity', 1) == 0:
oldout = sys.stdout
newout = StringIO()
sys.stdout = newout
spec = Specifier(**self.spec_args)
rshpr = create_reshaper(spec, **self.create_args)
assert type(rshpr) == Reshaper, 'type(reshaper) failure'
rshpr.convert(**self.convert_args)
if self.create_args.get('verbosity', 1) == 0:
actual = newout.getvalue()
assert actual == '', 'stdout should be empty'
sys.stdout = oldout
if print_diags:
rshpr.print_diagnostics()
def convert(self, print_diags=False):
if not (self.create_args.get('serial', False) and self.rank > 0):
if self.create_args.get('verbosity', 1) == 0:
oldout = sys.stdout
newout = StringIO()
sys.stdout = newout
spec = Specifier(**self.spec_args)
rshpr = create_reshaper(spec, **self.create_args)
self.assertEqual(type(rshpr), Reshaper, 'type(reshaper) failure')
rshpr.convert(**self.convert_args)
if self.create_args.get('verbosity', 1) == 0:
actual = newout.getvalue()
self.assertEqual(actual, '', 'stdout should be empty')
sys.stdout = oldout
if print_diags:
rshpr.print_diagnostics()
MPI_COMM_WORLD.Barrier()
def convert(self, print_diags=False):
if not (self.create_args.get('serial', False) and self.rank > 0):
if self.create_args.get('verbosity', 1) == 0:
oldout = sys.stdout
newout = StringIO()
sys.stdout = newout
spec = Specifier(**self.spec_args)
rshpr = create_reshaper(spec, **self.create_args)
self.assertEqual(type(rshpr), Reshaper, 'type(reshaper) failure')
rshpr.convert(**self.convert_args)
if self.create_args.get('verbosity', 1) == 0:
actual = newout.getvalue()
self.assertEqual(actual, '', 'stdout should be empty')
sys.stdout = oldout
if print_diags:
rshpr.print_diagnostics()
MPI_COMM_WORLD.Barrier()
def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list)
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
# loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0)
if rank == 0:
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
if completechunk.upper() in ['T', 'TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers, log = readArchiveXML(caseroot, tseries_input_rootdir,
tseries_output_rootdir, case,
options.standalone, completechunk,
debug, debugMsg)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = scomm.partition(specifiers,
func=partition.Duplicate(),
involved=True)
if rank == 0:
debugMsg("# of Specifiers: " + str(len(specifiers)),
header=True,
verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0, len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,
lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1, lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i],
serial=False,
verbosity=debug,
simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0
# load the pyreshaper modules
from pyreshaper import specification, reshaper
specifiers = list()
# loading the specifiers from the env_archive.xml only needs to run on the master task (rank=0)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank == 0:
specifiers = readArchiveXML(cesmEnv)
comm.Barrier()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = comm.bcast(specifiers, root=0)
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers, serial=False, verbosity=2)
# Set the output limit - the limit on the number of time-series files per processor to write.
# A limit of 0 means write all output files.
reshpr.output_limit = 0
# Run the conversion (slice-to-series) process
reshpr.convert()
# Print timing diagnostics
reshpr.print_diagnostics()
# TO-DO check if DOUT_S_SAVE_HISTORY_FILES is true or false
# Print a success statement to stdout
# load the pyreshaper modules
from pyreshaper import specification, reshaper
specifiers = list()
# loading the specifiers from the env_archive.xml only needs to run on the master task (rank=0)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank == 0:
specifiers = readArchiveXML(cesmEnv)
comm.Barrier()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
specifiers = comm.bcast(specifiers, root=0)
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers, serial=False, verbosity=2)
# Set the output limit - the limit on the number of time-series files per processor to write.
# A limit of 0 means write all output files.
reshpr.output_limit = 0
# Run the conversion (slice-to-series) process
reshpr.convert()
# Print timing diagnostics
reshpr.print_diagnostics()
# check if DOUT_S_SAVE_HISTORY_FILES is true or false
# Print a success statement to stdout
def main(options, scomm, rank, size, debug, debugMsg):
"""
"""
# initialize the CASEROOT environment dictionary
cesmEnv = dict()
# CASEROOT is given on the command line as required option --caseroot
caseroot = options.caseroot[0]
# get the XML variables loaded into a hash
env_file_list = ['env_postprocess.xml']
cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list);
# initialize the specifiers list to contain the list of specifier classes
specifiers = list()
tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR']
tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR']
case = cesmEnv['CASE']
completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK']
generate_all = cesmEnv['TIMESERIES_GENERATE_ALL']
if completechunk.upper() in ['T','TRUE']:
completechunk = 1
else:
completechunk = 0
specifiers,log = readArchiveXML(caseroot, tseries_input_rootdir, tseries_output_rootdir,
case, options.standalone, completechunk, generate_all,
debug, debugMsg, scomm, rank, size)
scomm.sync()
# specifiers is a list of pyreshaper specification objects ready to pass to the reshaper
if rank == 0:
debugMsg("# of Specifiers: "+str(len(specifiers)), header=True, verbosity=1)
if len(specifiers) > 0:
# setup subcommunicators to do streams and chunks in parallel
# everyone participates except for root
inter_comm, lsubcomms = divide_comm(scomm, len(specifiers))
color = inter_comm.get_color()
lsize = inter_comm.get_size()
lrank = inter_comm.get_rank()
GWORK_TAG = 10 # global comm mpi tag
LWORK_TAG = 20 # local comm mpi tag
# global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done.
if (rank == 0):
for i in range(0,len(specifiers)): # hand out all specifiers
scomm.ration(data=i, tag=GWORK_TAG)
for i in range(0,lsubcomms): # complete, signal this to all subcomms
scomm.ration(data=-99, tag=GWORK_TAG)
# subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends
# this information to all ranks within subcomm
elif (lrank == 0):
i = -999
while i != -99:
i = scomm.ration(tag=GWORK_TAG) # recv from global
for x in range(1,lsize):
inter_comm.ration(i, LWORK_TAG) # send to local ranks
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
# all subcomm ranks - recv the specifier to work on and call the reshaper
else:
i = -999
while i != -99:
i = inter_comm.ration(tag=LWORK_TAG) # recv from local root
if i != -99:
# create the PyReshaper object - uncomment when multiple specifiers is allowed
reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm)
# Run the conversion (slice-to-series) process
reshpr.convert()
inter_comm.sync()
if rank == 0:
# Update system log with the dates that were just converted
debugMsg('before chunking.write_log', header=True, verbosity=1)
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log)
debugMsg('after chunking.write_log', header=True, verbosity=1)
scomm.sync()
return 0
print('Creating directory {}'.format(dirs.OUTPUT))
os.makedirs(dirs.OUTPUT)
# Converted time-series file prefix & suffix
prefix = 'FAMIPC5.cam.'
output_prefix = os.path.join(dirs.OUTPUT, prefix)
output_suffix = conversion_utils.parse_output_suffix(input_fnames)
# --- Create PyReshaper specifier object ---------------------------------------
specifier = specification.create_specifier()
# Define specifier input needed perform the conversion
specifier.input_file_list = input_files
specifier.netcdf_format = "netcdf4"
specifier.compression_level = 1
specifier.output_file_prefix = output_prefix
specifier.output_file_suffix = output_suffix
specifier.time_variant_metadata = ["time", "time_bounds"]
# specifier.exclude_list = ['HKSAT','ZLAKE']
# Create the PyReshaper object
rshpr = reshaper.create_reshaper(specifier,
serial=False,
verbosity=1,
wmode='s')
# --- Run the conversion (slice-to-series) process -----------------------------
rshpr.convert()
# --- Print timing diagnostics -------------------------------------------------
rshpr.print_diagnostics()
