def test_pyAveragerSpecifier_series_split():
pyAveSpecifier_series = specification.create_specifier(
in_directory='tests/data/series_files/ice/',
out_directory='tests/output/Preproc/',
prefix='test_data.cice.h',
suffix='nc',
date_pattern='yyyymm-yyyymm',
hist_type='series',
avg_list=average,
weighted=wght,
split=True,
split_files='nh,sh',
split_orig_size='nj=384,ni=320',
ncformat=ncfrmt,
serial=serial,
ice_obs_file=ice_obs_file,
reg_file=reg_file,
year0=-99,
year1=-99,
clobber=clobber,
ncl_location=ncl_location,
)
assert type(
pyAveSpecifier_series) == pyaverager.specification.pyAveragerSpecifier
def test_pyAveragerSpecifier_slice():
pyAveSpecifier_slice = specification.create_specifier(
in_directory='tests/data/slice_files/ice/',
out_directory='tests/output/Preproc/',
prefix='test_data.cice.h',
suffix='nc',
date_pattern='yyyy-mm',
hist_type='slice',
avg_list=average,
weighted=wght,
ncformat=ncfrmt,
serial=serial,
ice_obs_file=ice_obs_file,
reg_file=reg_file,
year0=year0,
year1=year1,
clobber=clobber,
ncl_location=ncl_location,
)
assert type(
pyAveSpecifier_slice) == pyaverager.specification.pyAveragerSpecifier
clobber = True
suffix = 'nc'
date_pattern = 'yyyymm-yyyymm'
#### End user modify ####
pyAveSpecifier = specification.create_specifier(
in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
ncformat=ncfrmt,
varlist=var_list,
serial=serial,
clobber=clobber,
mean_diff_rms_obs_dir=mean_diff_rms_obs_dir,
region_nc_var=region_nc_var,
regions=regions,
region_wgt_var=region_wgt_var,
obs_dir=obs_dir,
obs_file=obs_file,
reg_obs_file_suffix=reg_obs_file_suffix,
vertical_levels=vertical_levels)
PyAverager.run_pyAverager(pyAveSpecifier)
htype = 'slice'
average = [
'dep_ann:1850:1859', 'djf:1850:1858', 'dep_mam:1850:1859',
'dep_jja:1850:1859', 'dep_son:1850:1859', 'jan:1850:1859', 'feb:1850:1859',
'mar:1850:1859', 'apr:1850:1859', 'may:1850:1859', 'jun:1850:1859',
'jul:1850:1859', 'aug:1850:1859', 'sep:1850:1859', 'oct:1850:1859',
'nov:1850:1859', 'dec:1850:1859'
]
wght = True
ncfrmt = 'netcdf'
serial = False
suffix = 'nc'
clobber = True
date_pattern = 'yyyymm-yyyymm'
#### End user modify ####
pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
ncformat=ncfrmt,
serial=serial,
clobber=clobber)
PyAverager.run_pyAverager(pyAveSpecifier)
ice_obs_file = '/glade/p/work/mickelso/PyAvg-IceDiag-obs/gx1v6_grid.nc'
reg_file = '/glade/p/work/mickelso/PyAvg-IceDiag-obs/REGION_MASK.nc'
year0 = 1
year1 = 10
ncl_location = '/glade/scratch/mickelso/pyAverager_trunk/trunk/pyaverager'
#### End user modify ####
pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
split=spl,
split_files=split_fn,
split_orig_size=split_size,
ncformat=ncfrmt,
serial=serial,
ice_obs_file=ice_obs_file,
reg_file=reg_file,
year0=year0,
year1=year1,
clobber=clobber,
ncl_location=ncl_location)
PreProc.run_pre_proc(pyAveSpecifier)
PyAverager.run_pyAverager(pyAveSpecifier)
def callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile,
out_dir, case_prefix, averageList, varList, envDict, stream,
grid_file, year0, year1, split, split_size, main_comm,
debugMsg):
"""setup the pyAverager specifier class with specifications to create
the climatology files in parallel.
Arguments:
avg_start_year (integer) - starting year for diagnostics
avg_stop_year (integer) - ending year for diagnositcs
in_dir (string) - input directory with either history time slice or variable time series files
htype (string) - 'series' or 'slice' depending on input history file type
out_dir (string) - output directory for climatology files
case_prefix (string) - input filename prefix
averageList (list) - list of averages to be created
varList (list) - list of variables. Note: an empty list implies all variables.
main_comm (object) - simple MPI communicator object
"""
wght = False
ncfrmt = 'netcdf'
serial = False
clobber = True
if htype == 'series':
date_pattern = 'yyyymm-yyyymm'
else:
date_pattern = 'yyyy-mm'
suffix = 'nc'
if split:
split_fn = 'nh,sh'
else:
split_fn = ''
ice_obs_file = grid_file #GRIDFILE
reg_file = envDict['REGION_MASK_FILE']
ncl_location = envDict['NCLPATH']
if main_comm.is_manager():
debugMsg('calling specification.create_specifier with following args',
header=True)
debugMsg('... in_directory = {0}'.format(in_dir), header=True)
debugMsg('... out_directory = {0}'.format(out_dir), header=True)
debugMsg('... prefix = {0}'.format(case_prefix), header=True)
debugMsg('... suffix = {0}'.format(suffix), header=True)
debugMsg('... date_pattern = {0}'.format(date_pattern), header=True)
debugMsg('... hist_type = {0}'.format(htype), header=True)
debugMsg('... avg_list = {0}'.format(averageList), header=True)
debugMsg('... weighted = {0}'.format(wght), header=True)
debugMsg('... ncformat = {0}'.format(ncfrmt), header=True)
debugMsg('... varlist = {0}'.format(varList), header=True)
debugMsg('... serial = {0}'.format(serial), header=True)
debugMsg('... clobber = {0}'.format(clobber), header=True)
debugMsg('... ice_obs_file = {0}'.format(ice_obs_file), header=True)
debugMsg('... reg_file = {0}'.format(reg_file), header=True)
debugMsg('... ncl_location = {0}'.format(ncl_location), header=True)
main_comm.sync()
try:
pyAveSpecifier = specification.create_specifier(
in_directory=in_dir,
out_directory=out_dir,
prefix=case_prefix,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=averageList,
weighted=wght,
ncformat=ncfrmt,
varlist=varList,
serial=serial,
clobber=clobber,
ice_obs_file=ice_obs_file,
reg_file=reg_file,
split=split,
split_files=split_fn,
split_orig_size=split_size,
ncl_location=ncl_location)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
try:
if main_comm.is_manager():
debugMsg("calling run_pyAverager")
main_comm.sync()
PyAverager.run_pyAverager(pyAveSpecifier)
pyAveSpecifier.year0 = year0
pyAveSpecifier.year1 = year1
pyAveSpecifier.split = split
pyAveSpecifier.split_files = split_fn
pyAveSpecifier.split_orig_size = split_size
debugMsg("calling run_pre_proc")
PreProc.run_pre_proc(pyAveSpecifier)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
#### User modify ####
in_dir='/glade/p/tdd/asap/data/b40.20th.track1.1deg.006/atm/hist/'
out_dir= os.environ.get('RESULTS_DIR')+'/atm/slice'
pref= 'b40.20th.track1.1deg.006.cam2.h0'
htype= 'slice'
average= ['ann:1850:1859','djf:1850:1858','mam:1850:1859','jja:1850:1859','son:1850:1859','zonalavg:1850:1859']
wght= True
ncfrmt = 'netcdf'
serial=False
suffix = 'nc'
clobber = True
date_pattern= 'yyyymm-yyyymm'
#### End user modify ####
pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
ncformat=ncfrmt,
serial=serial,
clobber=clobber)
PyAverager.run_pyAverager(pyAveSpecifier)
date_pattern= 'yyyymm-yyyymm'
#### End user modify ####
scomm.sync()
pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
ncformat=ncfrmt,
varlist=var_list,
serial=serial,
clobber=clobber,
mean_diff_rms_obs_dir=mean_diff_rms_obs_dir,
region_nc_var=region_nc_var,
regions=regions,
region_wgt_var=region_wgt_var,
obs_dir=obs_dir,
obs_file=obs_file,
reg_obs_file_suffix=reg_obs_file_suffix,
main_comm=scomm)
scomm.sync()
PyAverager.run_pyAverager(pyAveSpecifier)
def callPyAverager(in_dir, htype, tavgdir, case_prefix, averageList, varList,
diag_obs_root, netcdf_format, nlev, timeseries_obspath,
main_comm, debugMsg):
"""setup the pyAverager specifier class with specifications to create
the climatology files in parallel.
Arguments:
in_dir (string) - input directory with either history time slice or variable time series files
htype (string) - 'series' or 'slice' depending on input history file type
tavgdir (string) - output directory for climatology files
case_prefix (string) - input filename prefix
averageList (list) - list of averages to be created
varList (list) - list of variables. Note: an empty list implies all variables.
tseries (boolean) - TRUE if TIMESERIES plots are specified.
diag_obs_root (string) - OCNDIAG_DIAGOBSROOT machine dependent path to input data root
netcdf_format (string) - OCNDIAG_netcdf_format one of ['netcdf', 'netcdf4', 'netcdf4c', 'netcdfLarge']
nlev (integer) - Number of ocean vertical levels
timeseries_obspath (string) - timeseries observation files path
main_comm (object) - simple MPI communicator object
"""
# the following are used for timeseries averages and ignored otherwise
## mean_diff_rms_obs_dir = '{0}/omwg/timeseries_obs'.format(diag_obs_root)
mean_diff_rms_obs_dir = timeseries_obspath
region_nc_var = 'REGION_MASK'
regions={1:'Sou',2:'Pac',3:'Ind',6:'Atl',8:'Lab',9:'Gin',10:'Arc',11:'Hud',0:'Glo'}
region_wgt_var = 'TAREA'
## obs_dir = '{0}/omwg/timeseries_obs'.format(diag_obs_root)
obs_dir = timeseries_obspath
obs_file = 'obs.nc'
reg_obs_file_suffix = '_hor_mean_obs.nc'
wght = False
valid_netcdf_formats = ['netcdf', 'netcdf4', 'netcdf4c', 'netcdfLarge']
ncfrmt = 'netcdf'
if netcdf_format in valid_netcdf_formats:
ncfrmt = netcdf_format
serial = False
clobber = True
date_pattern = 'yyyymm-yyyymm'
suffix = 'nc'
main_comm.sync();
if main_comm.is_manager():
debugMsg('calling specification.create_specifier with following args', header=True)
debugMsg('... in_directory = {0}'.format(in_dir), header=True)
debugMsg('... out_directory = {0}'.format(tavgdir), header=True)
debugMsg('... prefix = {0}'.format(case_prefix), header=True)
debugMsg('... suffix = {0}'.format(suffix), header=True)
debugMsg('... date_pattern = {0}'.format(date_pattern), header=True)
debugMsg('... hist_type = {0}'.format(htype), header=True)
debugMsg('... avg_list = {0}'.format(averageList), header=True)
debugMsg('... weighted = {0}'.format(wght), header=True)
debugMsg('... ncformat = {0}'.format(ncfrmt), header=True)
debugMsg('... varlist = {0}'.format(varList), header=True)
debugMsg('... serial = {0}'.format(serial), header=True)
debugMsg('... clobber = {0}'.format(clobber), header=True)
debugMsg('... mean_diff_rms_obs_dir = {0}'.format(mean_diff_rms_obs_dir), header=True)
debugMsg('... region_nc_var = {0}'.format(region_nc_var), header=True)
debugMsg('... regions = {0}'.format(regions), header=True)
debugMsg('... region_wgt_var = {0}'.format(region_wgt_var), header=True)
debugMsg('... obs_dir = {0}'.format(obs_dir), header=True)
debugMsg('... obs_file = {0}'.format(obs_file), header=True)
debugMsg('... reg_obs_file_suffix = {0}'.format(reg_obs_file_suffix), header=True)
debugMsg('... nlev = {0}'.format(nlev), header=True)
main_comm.sync();
try:
pyAveSpecifier = specification.create_specifier(
in_directory = in_dir,
out_directory = tavgdir,
prefix = case_prefix,
suffix=suffix,
date_pattern=date_pattern,
hist_type = htype,
avg_list = averageList,
weighted = wght,
ncformat = ncfrmt,
varlist = varList,
serial = serial,
clobber = clobber,
mean_diff_rms_obs_dir = mean_diff_rms_obs_dir,
region_nc_var = region_nc_var,
regions = regions,
region_wgt_var = region_wgt_var,
obs_dir = obs_dir,
obs_file = obs_file,
reg_obs_file_suffix = reg_obs_file_suffix,
vertical_levels = nlev,
main_comm = main_comm)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
try:
if main_comm.is_manager():
debugMsg("calling run_pyAverager")
PyAverager.run_pyAverager(pyAveSpecifier)
main_comm.sync();
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
reg_file ='/glade/p/work/mickelso/PyAvg-IceDiag-obs/REGION_MASK.nc'
year0 = 1
year1 = 10
ncl_location = '/glade/scratch/mickelso/pyAverager_trunk/trunk/pyaverager'
#### End user modify ####
pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
out_directory=out_dir,
prefix=pref,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=average,
weighted=wght,
split=spl,
split_files=split_fn,
split_orig_size=split_size,
ncformat=ncfrmt,
serial=serial,
ice_obs_file=ice_obs_file,
reg_file=reg_file,
year0=year0,
year1=year1,
clobber=clobber,
ncl_location=ncl_location)
PreProc.run_pre_proc(pyAveSpecifier)
PyAverager.run_pyAverager(pyAveSpecifier)
def callPyAverager(start_year, stop_year, in_dir, htype, key_infile, out_dir,
case_prefix, averageList, varList, envDict, stream,
main_comm, debugMsg):
"""setup the pyAverager specifier class with specifications to create
the climatology files in parallel.
Arguments:
start_year (integer) - starting year for diagnostics
stop_year (integer) - ending year for diagnositcs
in_dir (string) - input directory with either history time slice or variable time series files
htype (string) - 'series' or 'slice' depending on input history file type
out_dir (string) - output directory for climatology files
case_prefix (string) - input filename prefix
averageList (list) - list of averages to be created
varList (list) - list of variables. Note: an empty list implies all variables.
"""
wght = envDict['weight_months']
if wght == 'True':
wght = True
else:
wght = False
valid_netcdf_formats = ['netcdf', 'netcdf4', 'netcdf4c', 'netcdfLarge']
ncfrmt = 'netcdf'
if envDict['netcdf_format'] in valid_netcdf_formats:
ncfrmt = envDict['netcdf_format']
serial = False
clobber = True
collapse_dim = 'lon'
if htype == 'series':
date_pattern = 'yyyymm-yyyymm'
else:
date_pattern = 'yyyy-mm'
suffix = 'nc'
main_comm.sync()
varList = []
if envDict['strip_off_vars'].lower() in ['t', 'true']:
varList = get_variable_list(envDict, in_dir, case_prefix, key_infile,
htype, stream)
main_comm.sync()
if main_comm.is_manager():
debugMsg('calling specification.create_specifier with following args',
header=True)
debugMsg('... in_directory = {0}'.format(in_dir), header=True)
debugMsg('... out_directory = {0}'.format(out_dir), header=True)
debugMsg('... prefix = {0}'.format(case_prefix), header=True)
debugMsg('... suffix = {0}'.format(suffix), header=True)
debugMsg('... date_pattern = {0}'.format(date_pattern), header=True)
debugMsg('... hist_type = {0}'.format(htype), header=True)
debugMsg('... avg_list = {0}'.format(averageList), header=True)
debugMsg('... collapse_dim = {0}'.format(collapse_dim), header=True)
debugMsg('... weighted = {0}'.format(wght), header=True)
debugMsg('... ncformat = {0}'.format(ncfrmt), header=True)
debugMsg('... varlist = {0}'.format(varList), header=True)
debugMsg('... serial = {0}'.format(serial), header=True)
debugMsg('... clobber = {0}'.format(clobber), header=True)
try:
pyAveSpecifier = specification.create_specifier(
in_directory=in_dir,
out_directory=out_dir,
prefix=case_prefix,
suffix=suffix,
date_pattern=date_pattern,
hist_type=htype,
avg_list=averageList,
collapse_dim=collapse_dim,
weighted=wght,
ncformat=ncfrmt,
varlist=varList,
serial=serial,
clobber=clobber,
main_comm=main_comm)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
try:
if main_comm.is_manager():
debugMsg("calling run_pyAverager", header=True)
PyAverager.run_pyAverager(pyAveSpecifier)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
def callPyAverager(start_year, stop_year, in_dir, htype, key_infile, out_dir, case_prefix, averageList, varList, envDict, stream, main_comm, debugMsg):
"""setup the pyAverager specifier class with specifications to create
the climatology files in parallel.
Arguments:
start_year (integer) - starting year for diagnostics
stop_year (integer) - ending year for diagnositcs
in_dir (string) - input directory with either history time slice or variable time series files
htype (string) - 'series' or 'slice' depending on input history file type
out_dir (string) - output directory for climatology files
case_prefix (string) - input filename prefix
averageList (list) - list of averages to be created
varList (list) - list of variables. Note: an empty list implies all variables.
"""
wght = envDict['weight_months']
if wght == 'True':
wght = True
else:
wght = False
valid_netcdf_formats = ['netcdf', 'netcdf4', 'netcdf4c', 'netcdfLarge']
ncfrmt = 'netcdf'
if envDict['netcdf_format'] in valid_netcdf_formats:
ncfrmt = envDict['netcdf_format']
serial = False
clobber = True
collapse_dim = 'lon'
if htype == 'series':
date_pattern = 'yyyymm-yyyymm'
else:
date_pattern = 'yyyy-mm'
suffix = 'nc'
main_comm.sync()
varList = []
if envDict['strip_off_vars'].lower() in ['t','true']:
varList = get_variable_list(envDict,in_dir,case_prefix,key_infile,htype,stream)
main_comm.sync()
if main_comm.is_manager():
debugMsg('calling specification.create_specifier with following args', header=True)
debugMsg('... in_directory = {0}'.format(in_dir), header=True)
debugMsg('... out_directory = {0}'.format(out_dir), header=True)
debugMsg('... prefix = {0}'.format(case_prefix), header=True)
debugMsg('... suffix = {0}'.format(suffix), header=True)
debugMsg('... date_pattern = {0}'.format(date_pattern), header=True)
debugMsg('... hist_type = {0}'.format(htype), header=True)
debugMsg('... avg_list = {0}'.format(averageList), header=True)
debugMsg('... collapse_dim = {0}'.format(collapse_dim), header=True)
debugMsg('... weighted = {0}'.format(wght), header=True)
debugMsg('... ncformat = {0}'.format(ncfrmt), header=True)
debugMsg('... varlist = {0}'.format(varList), header=True)
debugMsg('... serial = {0}'.format(serial), header=True)
debugMsg('... clobber = {0}'.format(clobber), header=True)
try:
pyAveSpecifier = specification.create_specifier(
in_directory = in_dir,
out_directory = out_dir,
prefix = case_prefix,
suffix=suffix,
date_pattern=date_pattern,
hist_type = htype,
avg_list = averageList,
collapse_dim = collapse_dim,
weighted = wght,
ncformat = ncfrmt,
varlist = varList,
serial = serial,
clobber = clobber,
main_comm = main_comm)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
try:
if main_comm.is_manager():
debugMsg("calling run_pyAverager", header=True)
PyAverager.run_pyAverager(pyAveSpecifier)
except Exception as error:
print(str(error))
traceback.print_exc()
sys.exit(1)
