def print_arguments(config_dict):
Logger.Indent("-> started with parameters:", 4, 7)
params = []
for key, value in config_dict.iteritems():
params.append("-> %-15s = %s" % (key, value))
params.sort()
for param in params:
Logger.Indent(param, 8, 29)
def merge_multiple_variables(self, files):
"""
creates a list of splitted files if more than one variable is included in the list of files.
otherwise the list is returned unchanged.
"""
# count the different variables
variables = []
for onefile in files:
if onefile.dict["parts"]["variable"] not in variables:
variables.append(onefile.dict["parts"]["variable"])
variables.sort()
# separate the files by variable
files_by_var = []
all_times = []
all_ensemble = []
for var in variables:
file_list = {}
for onefile in files:
if onefile.dict["parts"]["variable"] == var:
time_part = onefile.dict["parts"]["time"]
ensemble_part = ""
if "ensemble" in onefile.dict["parts"]:
ensemble_part = onefile.dict["parts"]["ensemble"]
if "rcm_ensemble" in onefile.dict["parts"]:
ensemble_part += "-" + onefile.dict["parts"][
"rcm_ensemble"]
file_list[time_part + ensemble_part] = onefile
if time_part not in all_times:
all_times.append(time_part)
if ensemble_part not in all_ensemble:
all_ensemble.append(ensemble_part)
files_by_var.append(file_list)
# create merged files from all time steps
new_files = []
for time_part in all_times:
for ensemble_part in all_ensemble:
file_list = []
for ivar in range(len(variables)):
key = time_part + ensemble_part
if key in files_by_var[ivar]:
file_list.append(files_by_var[ivar][key])
if len(file_list) == len(variables):
if len(file_list) == 1:
new_files.append(file_list[0])
else:
new_files.append(splittedFile(file_list))
else:
Logger.Indent(
"-> %s not available for all variables in %s" %
(time_part, ensemble_part), 8, 11)
for onefile in file_list:
Logger.Indent("-> found only: %s" % onefile.to_path(),
12, 27)
return sorted(new_files, key=splittedFile.get_fake_path)
def check_ensemble_completeness(self, files):
"""
count the files for all ensemble member. remove members that have less members then others
"""
# do nothing if we don't have an ensemble
if "ensemble" not in files[0].dict["parts"]:
return files
# first step: group all files by their ensemble membership
ensembles = {}
for onefile in files:
ensemble_part = onefile.dict["parts"]["ensemble"]
if "rcm_ensemble" in onefile.dict["parts"]:
ensemble_part += "-" + onefile.dict["parts"]["rcm_ensemble"]
if ensemble_part not in ensembles:
ensembles[ensemble_part] = [onefile]
else:
ensembles[ensemble_part].append(onefile)
# find the maximum number of files per member
max_files = 0
for ensemble_part, filelist in ensembles.iteritems():
if len(filelist) > max_files:
max_files = len(filelist)
# copy all ensemble members with the maximal number of files to the result list
result = []
for ensemble_part, filelist in ensembles.iteritems():
if len(filelist) == max_files:
result.extend(filelist)
else:
Logger.Indent(
"-> removed ensemble member %s, not enough files found!" %
ensemble_part, 8, 11)
# sort the result list
return sorted(result, key=splittedFile.get_fake_path)
def apply_workarounds_for_path(filelist):
"""
some decadal experiments are not located in the same location then other experiments that belong together.
the result ist that
the files are grouped together in unexpected ways. here the path are replaced by predefined aliases.
"""
# check if all file have the same root
roots = []
for onefile in filelist:
if not onefile.dict['root_dir'] in roots:
roots.append(onefile.dict['root_dir'])
has_multiple_roots = len(roots) > 1
# check all files
for i in range(len(filelist)):
# some baseline0 model runs are located in a different folder
# this only matters if the root of the files is not always the same
if has_multiple_roots:
match = re.match(
"/miklip/integration/data4miklip/projectdata/baseline0/output1/MPI-M/MPI-ESM-LR/decadal(\d\d\d\d)/(day|mon|6hr)/atmos/"
+ filelist[i].dict["parts"]["variable"] + "/" +
filelist[i].dict["parts"]["ensemble"], filelist[i].to_path())
if match is not None:
newfile = splittedFile([filelist[i]])
replacement = "/miklip/integration/data4miklip/model/global/miklip/baseline0/output1/MPI-M/MPI-ESM-LR/decadal%s/%s/atmos/%s/%s/v20111122/%s" % (
match.group(1), match.group(2),
filelist[i].dict["parts"]["cmor_table"],
filelist[i].dict["parts"]["ensemble"],
filelist[i].dict["parts"]["variable"])
newpath = filelist[i].to_path().replace(
match.group(0), replacement)
newfile.set_fake_path(newpath)
Logger.Indent(
"-> workaround: %s => %s" %
(filelist[i].to_path(), newpath), 8, 11)
filelist[i] = newfile
