def iterate_valid_targets(config, impacts=None, verbose=True):
root = config['results-root']
verbose = verbose or config.get('verbose', False)
do_montecarlo = config.get('do-montecarlo', False)
do_rcp_only = config.get('only-rcp', None)
do_iam_only = config.get('only-iam', None)
do_targetsubdirs = config.get('targetsubdirs', None)
checks = config.get('checks', None)
allmodels = config['only-models'] if config.get('only-models',
'all') != 'all' else None
if do_targetsubdirs:
iterator = results.iterate_targetdirs(root, do_targetsubdirs)
elif do_montecarlo == 'both':
iterator = results.iterate_both(root)
elif do_montecarlo:
iterator = results.iterate_montecarlo(root)
else:
iterator = results.iterate_batch(root, 'median')
# Logic for a given directory
#if root[-1] == '/':
# root = root[0:-1]
#iterator = results.iterate_batch(*os.path.split(root))
observations = 0
message_on_none = "No target directories."
for batch, rcp, model, iam, ssp, targetdir in iterator:
message_on_none = "No valid target directories."
if checks is not None and not results.directory_contains(
targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if do_iam_only and iam != do_iam_only:
print targetdir, "not", do_iam_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
if impacts is None:
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
else:
# Check that at least one of the impacts is here
for impact in impacts:
if impact + ".nc4" in os.listdir(targetdir):
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
break
if observations == 0:
print message_on_none
def iterate_valid_targets(config, impacts, verbose=True):
root = config['results-root']
do_montecarlo = config['do-montecarlo']
do_adaptation = config['do-adaptation'] == True
batches = range(config['batches']) if isinstance(
config['batches'], int) else config['batches']
allmodels = config['only-models'] if config.get('only-models',
'all') != 'all' else None
checks = config['checks']
do_rcp_only = config['only-rcp']
do_realization_only = config['only-realization']
batch_presuffix = config.get('batch-presuffix', False)
suffix = get_suffix(config)
if do_adaptation:
prefix = 'batch-adapt-'
if batch_presuffix:
prefix += batch_presuffix + '-'
batches = map(lambda i: prefix + str(i), batches)
if do_montecarlo:
iterator = results.iterate_montecarlo(root, batches=batches)
else:
if root[-1] == '/':
root = root[0:-1]
iterator = results.iterate_batch(*os.path.split(root))
for (batch, rcp, model, realization, pvals, targetdir) in iterator:
if checks is not None and not results.directory_contains(
targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if do_realization_only and realization != do_realization_only:
print targetdir, "not", do_realization_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
if impacts is None:
yield (batch, rcp, model, realization, pvals, targetdir)
continue
# Check that at least one of the impacts is here
for impact in impacts:
if impact + suffix + ".tar.gz" in os.listdir(targetdir):
yield (batch, rcp, model, realization, pvals, targetdir)
break
def iterate_valid_targets(config, impacts, verbose=True):
root = config['results-root']
do_montecarlo = config['do-montecarlo']
do_adaptation = config['do-adaptation'] == True
batches = range(config['batches']) if isinstance(config['batches'], int) else config['batches']
allmodels = config['only-models'] if config.get('only-models', 'all') != 'all' else None
checks = config['checks']
do_rcp_only = config['only-rcp']
do_realization_only = config['only-realization']
batch_presuffix = config.get('batch-presuffix', False)
suffix = get_suffix(config)
if do_adaptation:
prefix = 'batch-adapt-'
if batch_presuffix:
prefix += batch_presuffix + '-'
batches = map(lambda i: prefix + str(i), batches)
if do_montecarlo:
iterator = results.iterate_montecarlo(root, batches=batches)
else:
if root[-1] == '/':
root = root[0:-1]
iterator = results.iterate_batch(*os.path.split(root))
for (batch, rcp, model, realization, pvals, targetdir) in iterator:
if checks is not None and not results.directory_contains(targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if do_realization_only and realization != do_realization_only:
print targetdir, "not", do_realization_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
# Check that at least one of the impacts is here
for impact in impacts:
if impact + suffix + ".tar.gz" in os.listdir(targetdir):
yield (batch, rcp, model, realization, pvals, targetdir)
continue
def iterate_valid_targets(config, impacts=None, verbose=True):
root = config['results-root']
do_montecarlo = config['do-montecarlo']
do_rcp_only = config.get('only-rcp', None)
checks = config.get('checks', None)
allmodels = config['only-models'] if config.get('only-models',
'all') != 'all' else None
if do_montecarlo:
iterator = results.iterate_montecarlo(root)
else:
iterator = results.iterate_batch(root, 'median')
# Logic for a given directory
#if root[-1] == '/':
# root = root[0:-1]
#iterator = results.iterate_batch(*os.path.split(root))
for batch, rcp, model, iam, ssp, targetdir in iterator:
if checks is not None and not results.directory_contains(
targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
if impacts is None:
yield batch, rcp, model, iam, ssp, targetdir
else:
# Check that at least one of the impacts is here
for impact in impacts:
if impact + ".nc4" in os.listdir(targetdir):
yield batch, rcp, model, iam, ssp, targetdir
break
def iterate_valid_targets(config, impacts=None, verbose=True):
root = config['results-root']
do_montecarlo = config['do-montecarlo']
do_rcp_only = config['only-rcp']
allmodels = config['only-models'] if config.get('only-models', 'all') != 'all' else None
if do_montecarlo:
iterator = results.iterate_montecarlo(root)
else:
iterator = results.iterate_byp(root)
# Logic for a given directory
#if root[-1] == '/':
# root = root[0:-1]
#iterator = results.iterate_batch(*os.path.split(root))
for batch, rcp, model, iam, ssp, targetdir in iterator:
if checks is not None and not results.directory_contains(targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
if impacts is None:
yield batch, rcp, model, iam, ssp, targetdir
else:
# Check that at least one of the impacts is here
for impact in impacts:
if impact + ".nc4" in os.listdir(targetdir):
yield batch, rcp, model, iam, ssp, targetdir
break
# Go through all median result sets for a baseline
for (pdir, rcp, model, realization, pvals,
targetdir) in results.iterate_byp("/home/jrising/impacts"):
if pdir != 'pmed' or rcp != only_rcp or realization != only_realization:
continue
if impact + suffix + ".tar.gz" not in os.listdir(targetdir):
continue
collect_result(impact, pdir, rcp, model, realization, targetdir,
hold_impact)
# Go through all Monte Carlo result sets to understand the variance
for (batch, rcp, model, realization, pvals,
targetdir) in results.iterate_montecarlo("/home/jrising/impacts",
batches=batches):
# Make sure this is a valid result set
if rcp != only_rcp or realization != only_realization:
continue
if not results.directory_contains(targetdir, checks):
continue
if batch not in batches:
continue
if impact + suffix + ".tar.gz" not in os.listdir(targetdir):
continue
# Filter the result into the hold_model if we are at th base model
if model == base_model:
models = [
'global_interaction_gmfd', 'global_interaction_no_popshare_gmfd'
] # 'global_interaction_best', 'global_interaction_no_popshare_best',
variations = ['', '_comatose', '_dumb', '-histclim', '-costs']
aggregated = ['-aggregated', '-levels']
print "Mortality Median 2:"
iterator = results.iterate_batch(
"/shares/gcp/outputs/mortality/impacts-pharaoh2", 'median')
complete, missingness, incomplete, toaggregate, output4 = count_results(
iterator, mortality.allmodels.check_doit)
print complete, missingness, incomplete, toaggregate
print "Mortality Monte Carlo 2:"
iterator = results.iterate_montecarlo(
"/shares/gcp/outputs/mortality/impacts-pharaoh2")
complete, missingness, incomplete, toaggregate, output3 = count_results(
iterator, mortality.allmodels.check_doit)
print complete, missingness, incomplete, toaggregate
print "Mortality Median:"
iterator = results.iterate_batch(
"/shares/gcp/outputs/mortality/impacts-pharaoh", 'median')
complete, missingness, incomplete, toaggregate, output4 = count_results(
iterator, mortality.allmodels.check_doit)
print complete, missingness, incomplete, toaggregate
print "Mortality Monte Carlo:"
iterator = results.iterate_montecarlo(
"/shares/gcp/outputs/mortality/impacts-pharaoh")
# Go through all median result sets for a baseline
for (pdir, rcp, model, realization, pvals, targetdir) in results.iterate_byp("/home/jrising/impacts"):
if pdir != 'pmed':
continue
if impact + suffix + ".tar.gz" not in os.listdir(targetdir):
continue
# Collect the result into the hold_model_impact and hold_realization_impact data structures
if model == base_model:
collect_result(impact, pdir, rcp, model, realization, targetdir, hold_model_impact)
if realization == base_realization:
collect_result(impact, pdir, rcp, model, realization, targetdir, hold_realization_impact)
# Go through all Monte Carlo result sets to understand the variance
for (batch, rcp, model, realization, pvals, targetdir) in results.iterate_montecarlo("/home/jrising/impacts", batches=batches):
# Make sure this is a valid result set
if not results.directory_contains(targetdir, checks):
continue
if batch not in batches:
continue
if impact + suffix + ".tar.gz" not in os.listdir(targetdir):
continue
# Filter the result into the hold_model_realization data structure
if model == base_model and realization == base_realization:
collect_result(impact, batch, rcp, model, realization, targetdir, hold_model_realization)
# Filter the result into the hold_nothing data structure
collect_result(impact, batch, rcp, model, realization, targetdir, hold_nothing)
for cohort in ['young', 'older', 'oldest', 'combined']
]
variations = ['', '-noadapt', '-incadapt', '-histclim', '-costs']
aggregated = ['-aggregated', '-levels']
print("4th-order Polynomial Median:")
iterator = results.iterate_batch(
"/shares/gcp/outputs/mortality/impacts-harvester", 'median')
complete, missingness, incomplete, toaggregate, output4 = count_results(
iterator, mortality.allmodels.check_doit)
print((complete, missingness, incomplete, toaggregate))
print(output4)
print("4th-order Polynomial Monte Carlo:")
iterator = results.iterate_montecarlo(
"/shares/gcp/outputs/mortality/impacts-harvester")
complete, missingness, incomplete, toaggregate, output3 = count_results(
iterator, mortality.allmodels.check_doit)
print((complete, missingness, incomplete, toaggregate))
models = [
'global_interaction_Tmean-CSpline-LS-AgeSpec-' + cohort
for cohort in ['young', 'older', 'oldest', 'combined']
]
variations = ['', '-noadapt', '-incadapt', '-histclim', '-costs']
aggregated = ['-aggregated', '-levels']
print("Cubic Spline Median:")
iterator = results.iterate_batch(
"/shares/gcp/outputs/mortality/impacts-subterran", 'median')
complete, missingness, incomplete, toaggregate, output2 = count_results(
def iterate_valid_targets(root, config, impacts=None, verbose=True):
verbose = verbose or config.get('verbose', False)
do_montecarlo = config.get('do-montecarlo', False)
do_rcp_only = config.get('only-rcp', None)
do_iam_only = config.get('only-iam', None)
do_targetsubdirs = config.get('targetsubdirs', None)
do_batchdir = config.get('batchdir', 'median')
checks = config.get('checks', None)
dirtree = config.get('dirtree', 'normal')
allmodels = config['only-models'] if config.get('only-models',
'all') != 'all' else None
if dirtree == 'climate-only':
def get_iterator():
for alldirs in results.recurse_directories(root, 2):
yield ['pest', alldirs[0], alldirs[1], 'NA', 'NA', alldirs[2]]
iterator = get_iterator()
elif do_targetsubdirs:
iterator = results.iterate_targetdirs(root, do_targetsubdirs)
elif do_montecarlo == 'both':
iterator = results.iterate_both(root)
elif do_montecarlo:
iterator = results.iterate_montecarlo(root)
else:
iterator = results.iterate_batch(root, do_batchdir)
# Logic for a given directory
#if root[-1] == '/':
# root = root[0:-1]
#iterator = results.iterate_batch(*os.path.split(root))
observations = 0
message_on_none = "No target directories."
for batch, rcp, model, iam, ssp, targetdir in iterator:
message_on_none = "No valid target directories."
if checks is not None and not results.directory_contains(
targetdir, checks):
if verbose:
print targetdir, "missing", checks
continue
if do_rcp_only and rcp != do_rcp_only:
print targetdir, "not", do_rcp_only
continue
if do_iam_only and iam != do_iam_only:
print targetdir, "not", do_iam_only
continue
if allmodels is not None and model not in allmodels:
print targetdir, "not in", allmodels
continue
if impacts is None:
if is_parallel_deltamethod(config):
dmpath = get_deltamethod_path(targetdir, config)
if isinstance(dmpath, dict):
allthere = True
for name in dmpath:
if not os.path.isdir(dmpath[name]):
print "deltamethod", dmpath[name], "missing 1"
allthere = False
break
if allthere:
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
elif os.path.isdir(dmpath):
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
elif verbose:
print "deltamethod", get_deltamethod_path(
targetdir, config), "missing 2"
else:
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
else:
# Check that at least one of the impacts is here
for impact in impacts:
if impact + ".nc4" in os.listdir(multipath(targetdir, impact)):
if is_parallel_deltamethod(config):
if isinstance(targetdir, dict):
dmpath = os.path.join(
multipath(
get_deltamethod_path(targetdir, config),
impact), impact + ".nc4")
if not os.path.isfile(dmpath):
print "deltamethod", dmpath, "missing 3"
continue
elif not os.path.isfile(
os.path.join(targetdir, impact + ".nc4")):
print "deltamethod", dmpath, "missing 4"
continue
observations += 1
yield batch, rcp, model, iam, ssp, targetdir
break
if observations == 0:
print message_on_none
