def _run_convert(self, infiles, prefix, suffix, metadata, ncfmt, clevel, serial, verbosity, wmode, once, print_diags=False): if not (serial and self.rank > 0): spec = Specifier(infiles=infiles, ncfmt=ncfmt, compression=clevel, prefix=prefix, suffix=suffix, metadata=metadata) rshpr = create_reshaper(spec, serial=serial, verbosity=verbosity, wmode=wmode, once=once) rshpr.convert() if print_diags: rshpr.print_diagnostics() MPI_COMM_WORLD.Barrier()
def testCreateReshaperType(self): actual = create_reshaper( self.spec, serial=False, verbosity=2, once=False) expected = Slice2SeriesReshaper msg = self._info_msg("type(create_reshaper)", None, type(actual), expected) self.assertEqual(type(actual), expected, msg)
def main(argv=None): opts, specfile = cli(argv) # Try importing the file try: spec = pickle.load(open(specfile, 'rb')) except: err_msg = "Specifier File '{}' could not be opened and read".format( specfile) raise RuntimeError(err_msg) # Create the PyReshaper object reshpr = reshaper.create_reshaper(spec, serial=opts.serial, verbosity=opts.verbosity, wmode=opts.write_mode, once=opts.once) # Run the conversion (slice-to-series) process reshpr.convert(output_limit=opts.limit, rchunks=opts.rchunks, wchunks=opts.wchunks) # Print timing diagnostics reshpr.print_diagnostics()
def main(options, scomm, rank, size): """ """ # initialize the CASEROOT environment dictionary cesmEnv = dict() # CASEROOT is given on the command line as required option --caseroot caseroot = options.caseroot[0] # set the caseroot based on standalone or not pp_caseroot = caseroot if not options.standalone: caseroot, pp_subdir = os.path.split(caseroot) if rank == 0: print('cesm_tseries_generator: caseroot = {0}'.format(caseroot)) # set the debug level debug = options.debug[0] # cesmEnv["id"] = "value" parsed from the CASEROOT/env_*.xml files env_file_list = [ 'env_case.xml', 'env_run.xml', 'env_build.xml', 'env_mach_pes.xml' ] # check if the standalone option is set if options.standalone: env_file_list = ['env_postprocess.xml'] cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list) # initialize the specifiers list to contain the list of specifier classes specifiers = list() # loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0) if rank == 0: specifiers = readArchiveXML(caseroot, cesmEnv['DOUT_S_ROOT'], cesmEnv['CASE'], options.standalone, debug) scomm.sync() # specifiers is a list of pyreshaper specification objects ready to pass to the reshaper specifiers = scomm.partition(specifiers, func=partition.Duplicate(), involved=True) # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers, serial=False, verbosity=debug) # Run the conversion (slice-to-series) process reshpr.convert() # Print timing diagnostics reshpr.print_diagnostics() # TO-DO check if DOUT_S_SAVE_HISTORY_FILES is true or false and # delete history files accordingly return 0
def testCreateReshaperType(self): actual = create_reshaper(self.spec, serial=True, verbosity=2, once=False) expected = Slice2SeriesReshaper msg = self._info_msg("type(create_reshaper)", None, type(actual), expected) self.assertEqual(type(actual), expected, msg)
def _test_create_reshaper(self, serial, verbosity, wmode): self._test_header(("create_reshaper(serial={}, verbosity={}, " "wmode={!r})").format(serial, verbosity, wmode)) if not (serial and self.rank > 0): spec = Specifier(infiles=self.slices, ncfmt='netcdf', compression=0, prefix='output.', suffix='.nc', metadata=[]) rshpr = create_reshaper(spec, serial=serial, verbosity=verbosity, wmode=wmode) self._assertion("type(reshaper)", type(rshpr), Slice2SeriesReshaper)
def _test_create_reshaper(self, serial, verbosity, wmode): self._test_header(("create_reshaper(serial={0}, verbosity={1}, " "wmode={2!r})").format(serial, verbosity, wmode)) if not (serial and self.rank > 0): spec = Specifier(infiles=mkTestData.slices, ncfmt='netcdf', compression=0, prefix='output.', suffix='.nc', metadata=[]) rshpr = create_reshaper(spec, serial=serial, verbosity=verbosity, wmode=wmode) self._assertion("type(reshaper)", type(rshpr), Reshaper)
def convert(self, print_diags=False): if self.create_args.get('verbosity', 1) == 0: oldout = sys.stdout newout = StringIO() sys.stdout = newout spec = Specifier(**self.spec_args) rshpr = create_reshaper(spec, **self.create_args) assert type(rshpr) == Reshaper, 'type(reshaper) failure' rshpr.convert(**self.convert_args) if self.create_args.get('verbosity', 1) == 0: actual = newout.getvalue() assert actual == '', 'stdout should be empty' sys.stdout = oldout if print_diags: rshpr.print_diagnostics()
def convert(self, print_diags=False): if not (self.create_args.get('serial', False) and self.rank > 0): if self.create_args.get('verbosity', 1) == 0: oldout = sys.stdout newout = StringIO() sys.stdout = newout spec = Specifier(**self.spec_args) rshpr = create_reshaper(spec, **self.create_args) self.assertEqual(type(rshpr), Reshaper, 'type(reshaper) failure') rshpr.convert(**self.convert_args) if self.create_args.get('verbosity', 1) == 0: actual = newout.getvalue() self.assertEqual(actual, '', 'stdout should be empty') sys.stdout = oldout if print_diags: rshpr.print_diagnostics() MPI_COMM_WORLD.Barrier()
def main(options, scomm, rank, size, debug, debugMsg): """ """ # initialize the CASEROOT environment dictionary cesmEnv = dict() # CASEROOT is given on the command line as required option --caseroot caseroot = options.caseroot[0] # get the XML variables loaded into a hash env_file_list = ['env_postprocess.xml'] cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list) # initialize the specifiers list to contain the list of specifier classes specifiers = list() # loading the specifiers from the env_timeseries.xml only needs to run on the master task (rank=0) if rank == 0: tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR'] tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR'] case = cesmEnv['CASE'] completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK'] if completechunk.upper() in ['T', 'TRUE']: completechunk = 1 else: completechunk = 0 specifiers, log = readArchiveXML(caseroot, tseries_input_rootdir, tseries_output_rootdir, case, options.standalone, completechunk, debug, debugMsg) scomm.sync() # specifiers is a list of pyreshaper specification objects ready to pass to the reshaper specifiers = scomm.partition(specifiers, func=partition.Duplicate(), involved=True) if rank == 0: debugMsg("# of Specifiers: " + str(len(specifiers)), header=True, verbosity=1) if len(specifiers) > 0: # setup subcommunicators to do streams and chunks in parallel # everyone participates except for root inter_comm, lsubcomms = divide_comm(scomm, len(specifiers)) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() GWORK_TAG = 10 # global comm mpi tag LWORK_TAG = 20 # local comm mpi tag # global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done. if (rank == 0): for i in range(0, len(specifiers)): # hand out all specifiers scomm.ration(data=i, tag=GWORK_TAG) for i in range(0, lsubcomms): # complete, signal this to all subcomms scomm.ration(data=-99, tag=GWORK_TAG) # subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends # this information to all ranks within subcomm elif (lrank == 0): i = -999 while i != -99: i = scomm.ration(tag=GWORK_TAG) # recv from global for x in range(1, lsize): inter_comm.ration(i, LWORK_TAG) # send to local ranks if i != -99: # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm) # Run the conversion (slice-to-series) process reshpr.convert() inter_comm.sync() # all subcomm ranks - recv the specifier to work on and call the reshaper else: i = -999 while i != -99: i = inter_comm.ration(tag=LWORK_TAG) # recv from local root if i != -99: # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm) # Run the conversion (slice-to-series) process reshpr.convert() inter_comm.sync() if rank == 0: # Update system log with the dates that were just converted debugMsg('before chunking.write_log', header=True, verbosity=1) chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log) debugMsg('after chunking.write_log', header=True, verbosity=1) scomm.sync() return 0
# load the pyreshaper modules from pyreshaper import specification, reshaper specifiers = list() # loading the specifiers from the env_archive.xml only needs to run on the master task (rank=0) comm = MPI.COMM_WORLD rank = comm.Get_rank() if rank == 0: specifiers = readArchiveXML(cesmEnv) comm.Barrier() # specifiers is a list of pyreshaper specification objects ready to pass to the reshaper specifiers = comm.bcast(specifiers, root=0) # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers, serial=False, verbosity=2) # Set the output limit - the limit on the number of time-series files per processor to write. # A limit of 0 means write all output files. reshpr.output_limit = 0 # Run the conversion (slice-to-series) process reshpr.convert() # Print timing diagnostics reshpr.print_diagnostics() # TO-DO check if DOUT_S_SAVE_HISTORY_FILES is true or false # Print a success statement to stdout
# load the pyreshaper modules from pyreshaper import specification, reshaper specifiers = list() # loading the specifiers from the env_archive.xml only needs to run on the master task (rank=0) comm = MPI.COMM_WORLD rank = comm.Get_rank() if rank == 0: specifiers = readArchiveXML(cesmEnv) comm.Barrier() # specifiers is a list of pyreshaper specification objects ready to pass to the reshaper specifiers = comm.bcast(specifiers, root=0) # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers, serial=False, verbosity=2) # Set the output limit - the limit on the number of time-series files per processor to write. # A limit of 0 means write all output files. reshpr.output_limit = 0 # Run the conversion (slice-to-series) process reshpr.convert() # Print timing diagnostics reshpr.print_diagnostics() # check if DOUT_S_SAVE_HISTORY_FILES is true or false # Print a success statement to stdout
def main(options, scomm, rank, size, debug, debugMsg): """ """ # initialize the CASEROOT environment dictionary cesmEnv = dict() # CASEROOT is given on the command line as required option --caseroot caseroot = options.caseroot[0] # get the XML variables loaded into a hash env_file_list = ['env_postprocess.xml'] cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list); # initialize the specifiers list to contain the list of specifier classes specifiers = list() tseries_input_rootdir = cesmEnv['TIMESERIES_INPUT_ROOTDIR'] tseries_output_rootdir = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR'] case = cesmEnv['CASE'] completechunk = cesmEnv['TIMESERIES_COMPLETECHUNK'] generate_all = cesmEnv['TIMESERIES_GENERATE_ALL'] if completechunk.upper() in ['T','TRUE']: completechunk = 1 else: completechunk = 0 specifiers,log = readArchiveXML(caseroot, tseries_input_rootdir, tseries_output_rootdir, case, options.standalone, completechunk, generate_all, debug, debugMsg, scomm, rank, size) scomm.sync() # specifiers is a list of pyreshaper specification objects ready to pass to the reshaper if rank == 0: debugMsg("# of Specifiers: "+str(len(specifiers)), header=True, verbosity=1) if len(specifiers) > 0: # setup subcommunicators to do streams and chunks in parallel # everyone participates except for root inter_comm, lsubcomms = divide_comm(scomm, len(specifiers)) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() GWORK_TAG = 10 # global comm mpi tag LWORK_TAG = 20 # local comm mpi tag # global root - hands out specifiers to work on. When complete, it must tell each subcomm all work is done. if (rank == 0): for i in range(0,len(specifiers)): # hand out all specifiers scomm.ration(data=i, tag=GWORK_TAG) for i in range(0,lsubcomms): # complete, signal this to all subcomms scomm.ration(data=-99, tag=GWORK_TAG) # subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends # this information to all ranks within subcomm elif (lrank == 0): i = -999 while i != -99: i = scomm.ration(tag=GWORK_TAG) # recv from global for x in range(1,lsize): inter_comm.ration(i, LWORK_TAG) # send to local ranks if i != -99: # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm) # Run the conversion (slice-to-series) process reshpr.convert() inter_comm.sync() # all subcomm ranks - recv the specifier to work on and call the reshaper else: i = -999 while i != -99: i = inter_comm.ration(tag=LWORK_TAG) # recv from local root if i != -99: # create the PyReshaper object - uncomment when multiple specifiers is allowed reshpr = reshaper.create_reshaper(specifiers[i], serial=False, verbosity=debug, simplecomm=inter_comm) # Run the conversion (slice-to-series) process reshpr.convert() inter_comm.sync() if rank == 0: # Update system log with the dates that were just converted debugMsg('before chunking.write_log', header=True, verbosity=1) chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log) debugMsg('after chunking.write_log', header=True, verbosity=1) scomm.sync() return 0
print('Creating directory {}'.format(dirs.OUTPUT)) os.makedirs(dirs.OUTPUT) # Converted time-series file prefix & suffix prefix = 'FAMIPC5.cam.' output_prefix = os.path.join(dirs.OUTPUT, prefix) output_suffix = conversion_utils.parse_output_suffix(input_fnames) # --- Create PyReshaper specifier object --------------------------------------- specifier = specification.create_specifier() # Define specifier input needed perform the conversion specifier.input_file_list = input_files specifier.netcdf_format = "netcdf4" specifier.compression_level = 1 specifier.output_file_prefix = output_prefix specifier.output_file_suffix = output_suffix specifier.time_variant_metadata = ["time", "time_bounds"] # specifier.exclude_list = ['HKSAT','ZLAKE'] # Create the PyReshaper object rshpr = reshaper.create_reshaper(specifier, serial=False, verbosity=1, wmode='s') # --- Run the conversion (slice-to-series) process ----------------------------- rshpr.convert() # --- Print timing diagnostics ------------------------------------------------- rshpr.print_diagnostics()