def test_my_stdev(dset, group):
"""
Test computation of multi-year means
Parameters
----------
dset : str
dset to compute means from
group : str | NoneType
group to collect datasets into
"""
my_std = manual_stdev(H5_FILES, dset)
my_out = os.path.join(TEMP_DIR, "{}-MY.h5".format(dset))
with MultiYear(my_out, mode='w', group=group) as my:
my.collect(H5_FILES, dset)
dset_std = my.stdev(dset)
compare_arrays(my_std, dset_std, "Computed STDEV")
with MultiYear(my_out, mode='r', group=group) as my:
dset_std = my.stdev(dset)
compare_arrays(my_std, dset_std, "Saved STDEV")
if PURGE_OUT:
os.remove(my_out)
def test_my_collection(dset, group):
"""
Collect the desired dset
Parameters
----------
dset : str
dset to collect from H5_Files
group : str | NoneType
group to collect datasets into
"""
my_out = os.path.join(TEMP_DIR, "{}-MY.h5".format(dset))
my_dsets = [
'meta',
]
my_dsets.extend(['{}-{}'.format(dset, year) for year in YEARS])
if 'profile' in dset:
MultiYear.collect_profiles(my_out, H5_FILES, dset, group=group)
my_dsets.extend(["time_index-{}".format(year) for year in YEARS])
else:
MultiYear.collect_means(my_out, H5_FILES, dset, group=group)
my_dsets.extend(
["{}-{}".format(dset, val) for val in ['means', 'stdev']])
if group is not None:
my_dsets = ['{}/{}'.format(group, ds) for ds in my_dsets]
with Outputs(my_out, mode='r') as f:
out_dsets = f.datasets
msg = "Missing datasets after collection"
assert np.in1d(my_dsets, out_dsets).all(), msg
if PURGE_OUT:
os.remove(my_out)
def test_my_stdev(dset, group):
"""
Test computation of multi-year means
Parameters
----------
dset : str
dset to compute means from
group : str | NoneType
group to collect datasets into
"""
my_std = manual_stdev(H5_FILES, dset)
with tempfile.TemporaryDirectory() as temp:
my_out = os.path.join(temp, "{}-MY.h5".format(dset))
with MultiYear(my_out, mode='w', group=group) as my:
my.collect(H5_FILES, dset)
dset_std = my.stdev(dset)
compare_arrays(my_std, dset_std, "Computed STDEV")
with MultiYear(my_out, mode='r', group=group) as my:
dset_std = my.stdev(dset)
compare_arrays(my_std, dset_std, "Saved STDEV")
def multi_year_groups(ctx, group_params, verbose):
"""Run multi year collection and means for multiple groups."""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize loggers for multiple modules
log_dir = os.path.dirname(my_file)
init_mult(name,
log_dir,
modules=[__name__, 'reV.handlers.multi_year'],
verbose=verbose,
node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to collection method: "{}" : "{}" '
'with type "{}"'.format(key, val, type(val)))
logger.info('Multi-year collection is being run with job name "{}". '
'Target output path is: {}'.format(name, my_file))
ts = time.time()
for group_name, group in json.loads(group_params).items():
logger.info('- Collecting datasets "{}" from "{}" into "{}/"'.format(
group['dsets'], group['source_files'], group_name))
t0 = time.time()
for dset in group['dsets']:
if MultiYear.is_profile(group['source_files'], dset):
MultiYear.collect_profiles(my_file,
group['source_files'],
dset,
group=group['group'])
else:
MultiYear.collect_means(my_file,
group['source_files'],
dset,
group=group['group'])
runtime = (time.time() - t0) / 60
logger.info('- {} collection completed in: {:.2f} min.'.format(
group_name, runtime))
runtime = (time.time() - ts) / 60
logger.info(
'Multi-year collection completed in : {:.2f} min.'.format(runtime))
# add job to reV status file.
status = {
'dirout': os.path.dirname(my_file),
'fout': os.path.basename(my_file),
'job_status': 'successful',
'runtime': runtime
}
Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def multi_year(ctx, group, source_files, dsets, verbose):
"""Run multi year collection and means on local worker."""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize loggers for multiple modules
log_dir = os.path.dirname(my_file)
init_mult(name,
log_dir,
modules=[__name__, 'reV.handlers.multi_year'],
verbose=verbose,
node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to collection method: "{}" : "{}" '
'with type "{}"'.format(key, val, type(val)))
logger.info('Multi-year collection is being run for "{}" '
'with job name "{}" on {}. Target output path is: {}'.format(
dsets, name, source_files, my_file))
t0 = time.time()
for dset in dsets:
if MultiYear.is_profile(source_files, dset):
MultiYear.collect_profiles(my_file,
source_files,
dset,
group=group)
else:
MultiYear.collect_means(my_file, source_files, dset, group=group)
runtime = (time.time() - t0) / 60
logger.info(
'Multi-year collection completed in: {:.2f} min.'.format(runtime))
# add job to reV status file.
status = {
'dirout': os.path.dirname(my_file),
'fout': os.path.basename(my_file),
'job_status': 'successful',
'runtime': runtime,
'finput': source_files
}
Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def test_update(dset, group):
"""
Test computation of multi-year means
Parameters
----------
dset : str
dset to compute means from
group : str | NoneType
group to collect datasets into
"""
my_out = os.path.join(TEMP_DIR, "{}-MY.h5".format(dset))
# Collect 2012 and compute 'means'
files = H5_FILES[:1]
MultiYear.collect_means(my_out, files, dset, group=group)
my_means = manual_means(files, dset)
my_std = manual_stdev(files, dset)
with MultiYear(my_out, mode='r', group=group) as my:
dset_means = my.means(dset)
dset_std = my.stdev(dset)
print(group, my.datasets)
compare_arrays(my_means, dset_means, "2012 Means")
compare_arrays(my_std, dset_std, "2012 STDEV")
# Add 2013
files = H5_FILES
MultiYear.collect_means(my_out, files, dset, group=group)
my_means = manual_means(files, dset)
my_std = manual_stdev(files, dset)
with MultiYear(my_out, mode='r', group=group) as my:
dset_means = my.means(dset)
dset_std = my.stdev(dset)
compare_arrays(my_means, dset_means, "Updated Means")
compare_arrays(my_std, dset_std, "Updated STDEV")
if PURGE_OUT:
os.remove(my_out)
def test_update(dset, group):
"""
Test computation of multi-year means
Parameters
----------
dset : str
dset to compute means from
group : str | NoneType
group to collect datasets into
"""
with tempfile.TemporaryDirectory() as temp:
my_out = os.path.join(temp, "{}-MY.h5".format(dset))
# Collect 2012 and compute 'means'
files = H5_FILES[:1]
MultiYear.collect_means(my_out, files, dset, group=group)
my_means = manual_means(files, dset)
my_std = manual_stdev(files, dset)
with MultiYear(my_out, mode='r', group=group) as my:
dset_means = my.means(dset)
dset_std = my.stdev(dset)
compare_arrays(my_means, dset_means, "2012 Means")
compare_arrays(my_std, dset_std, "2012 STDEV")
# Add 2013
files = H5_FILES
MultiYear.collect_means(my_out, files, dset, group=group)
my_means = manual_means(files, dset)
my_std = manual_stdev(files, dset)
with MultiYear(my_out, mode='r', group=group) as my:
dset_means = my.means(dset)
dset_std = my.stdev(dset)
compare_arrays(my_means, dset_means, "Updated Means")
compare_arrays(my_std, dset_std, "Updated STDEV")