def test_expr():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.spectral_operator)
command.add_operator(cdoapi.cdo_command.expression_operator,
"var91=sq(var130)")
commstr = command.create_command()
assert commstr == "-expr,'var91=sq(var130)' -sp2gpl -selcode,130"
def find_sp_variable(task, autofilter):
global ifs_gridpoint_file_, ifs_spectral_file_, surface_pressure, ln_surface_pressure
if autofilter:
if grib_filter.spvar is None:
log.error("Could not find surface pressure in model output...")
return
log.info("Found surface pressure in file %s" % grib_filter.spvar[2])
setattr(task, cmor_task.filter_output_key, [grib_filter.spvar[2]])
if grib_filter.spvar[0] == 152:
task.source = cmor_source.ifs_source.read("var134=exp(var152)")
task.source.grid_ = 1 if grib_filter.spvar[
2] == ifs_spectral_file_ else 0
return
log.info("Looking for surface pressure variable in input files...")
command = cdoapi.cdo_command()
code_string = command.show_code(ifs_spectral_file_)
codes = [cmor_source.grib_code(int(c)) for c in code_string[0].split()]
if surface_pressure in codes:
log.info("Found surface pressure in spectral file")
setattr(task, cmor_task.filter_output_key, [ifs_spectral_file_])
task.source.grid_ = 1
return
if ln_surface_pressure in codes:
log.info("Found lnsp in spectral file")
setattr(task, cmor_task.filter_output_key, [ifs_spectral_file_])
task.source = cmor_source.ifs_source.read("var134=exp(var152)")
return
log.info(
"Did not find sp or lnsp in spectral file: assuming gridpoint file contains sp"
)
setattr(task, cmor_task.filter_output_key, [ifs_gridpoint_file_])
task.source.grid_ = 0
def execute(tasks, cleanup=True, autofilter=True):
global log, start_date_, ifs_grid_descr_
supported_tasks = [
t for t in filter_tasks(tasks)
if t.status == cmor_task.status_initialized
]
log.info("Executing %d IFS tasks..." % len(supported_tasks))
mask_tasks = get_mask_tasks(supported_tasks)
surf_pressure_tasks = get_sp_tasks(supported_tasks, autofilter)
regular_tasks = [
t for t in supported_tasks if t not in surf_pressure_tasks
]
tasks_todo = mask_tasks + surf_pressure_tasks + regular_tasks
grid_descr_file = None
if autofilter:
tasks_todo = grib_filter.execute(tasks_todo, start_date_.month)
for t in tasks_todo:
if getattr(t.source, "grid_", None) == cmor_source.ifs_grid.point:
filepaths = getattr(t, cmor_task.filter_output_key, [])
if any(filepaths):
grid_descr_file = filepaths[0]
break
else:
for task in tasks_todo:
grid = getattr(task.source, "grid_")
if grid == cmor_source.ifs_grid.point:
setattr(task, cmor_task.filter_output_key,
[ifs_gridpoint_file_])
elif grid == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key,
[ifs_spectral_file_])
else:
log.error(
"Task ifs source has unknown grid for %s in table %s" %
(task.target.variable, task.target.table))
task.set_failed()
setattr(task, cmor_task.output_frequency_key, output_frequency_)
grid_descr_file = ifs_gridpoint_file_
log.info("Fetching grid description from %s ..." % grid_descr_file)
ifs_grid_descr_ = cdoapi.cdo_command().get_grid_descr(
grid_descr_file) if os.path.exists(grid_descr_file) else {}
processed_tasks = []
try:
log.info("Post-processing tasks...")
processed_tasks = postprocess([
t for t in tasks_todo if t.status == cmor_task.status_initialized
])
for task in [t for t in processed_tasks if t in mask_tasks]:
read_mask(task.target.variable,
getattr(task, cmor_task.output_path_key))
cmorize([t for t in processed_tasks if t in supported_tasks])
except Exception:
if cleanup:
clean_tmp_data(processed_tasks, True)
processed_tasks = []
raise
finally:
if cleanup:
clean_tmp_data(processed_tasks, False)
def test_expr(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.spectral_operator)
command.add_operator(cdoapi.cdo_command.expression_operator,
"var91=sq(var130)")
commstr = command.create_command()
nose.tools.eq_("-expr,'var91=sq(var130)' -sp2gpl -selcode,130",
commstr)
def test_expr_monmean():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.post_expr_operator,
"var91=sq(var130)")
command.add_operator(cdoapi.cdo_command.month +
cdoapi.cdo_command.mean)
commstr = command.create_command()
assert commstr == "-expr,'var91=sq(var130)' -monmean -selcode,130"
def test_expr_monmean(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.post_expr_operator,
"var91=sq(var130)")
command.add_operator(cdoapi.cdo_command.month +
cdoapi.cdo_command.mean)
commstr = command.create_command()
nose.tools.eq_("-expr,'var91=sq(var130)' -monmean -selcode,130",
commstr)
def test_add_sellevel():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.select_z_operator,
cdoapi.cdo_command.pressure)
command.add_operator(cdoapi.cdo_command.gridtype_operator,
cdoapi.cdo_command.regular_grid_type)
command.add_operator(cdoapi.cdo_command.select_lev_operator, 500, 350,
10)
commstr = command.create_command()
assert commstr == "-setgridtype,regular -sellevel,500,350,10 -selzaxis,pressure -selcode,130"
def create_grid_from_file(filepath):
global log
command = cdoapi.cdo_command()
grid_descr = command.get_grid_descr(filepath)
gridtype = grid_descr.get("gridtype", "unknown")
if gridtype != "gaussian":
log.error("Cannot read other grids then regular gaussian grids, current grid type read from file %s was % s" % (
filepath, gridtype))
return None
xvals = read_coordinate_vals(grid_descr, 'x', 360)
yvals = read_coordinate_vals(grid_descr, 'y', 180)
return create_gauss_grid(xvals, yvals)
def test_add_daymin(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.year + cdoapi.cdo_command.mean)
command.add_operator(cdoapi.cdo_command.day + cdoapi.cdo_command.max)
command.add_operator(cdoapi.cdo_command.select_z_operator,
cdoapi.cdo_command.pressure)
command.add_operator(cdoapi.cdo_command.gridtype_operator,
cdoapi.cdo_command.regular_grid_type)
command.add_operator(cdoapi.cdo_command.select_lev_operator, 500, 350,
10)
commstr = command.create_command()
nose.tools.eq_(
"-yearmean -daymax -setgridtype,regular -sellevel,500,350,10 -selzaxis,pressure -selcode,130",
commstr)
def test_add_monmean(self):
command = cdoapi.cdo_command(130)
command.add_operator(
cdoapi.cdo_command.mean_time_operators[cdoapi.cdo_command.month])
command.add_operator(cdoapi.cdo_command.select_z_operator,
cdoapi.cdo_command.pressure)
command.add_operator(cdoapi.cdo_command.gridtype_operator,
cdoapi.cdo_command.regular_grid_type)
command.add_operator(cdoapi.cdo_command.select_lev_operator, 500, 350,
10)
commstr = command.create_command()
nose.tools.eq_(
"-setgridtype,regular -monmean -sellevel,500,350,10 -selzaxis,pressure -selcode,130",
commstr)
def create_grid_from_grib(filepath):
global log
command = cdoapi.cdo_command()
grid_descr = command.get_grid_descr(filepath)
gridtype = grid_descr.get("gridtype", "unknown")
if gridtype != "gaussian":
log.error(
"Cannot read other grids then regular gaussian grids, current grid type read from file %s was % s"
% (filepath, gridtype))
return None
xsize = grid_descr.get("xsize", 0)
xfirst = grid_descr.get("xfirst", 0)
yvals = grid_descr.get("yvals", numpy.array([]))
if not (xsize > 0 and len(yvals) > 0):
log.error("Invalid grid detected in post-processed data: %s" %
str(grid_descr))
return None
return create_gauss_grid(xsize, xfirst, yvals)
def test_add_specmapping():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.spectral_operator)
commstr = command.create_command()
assert commstr == "-sp2gpl -selcode,130"
def test_add_gridmapping():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.gridtype_operator,
cdoapi.cdo_command.regular_grid_type)
commstr = command.create_command()
assert commstr == "-setgridtype,regular -selcode,130"
def test_select_code(self):
command = cdoapi.cdo_command(130)
commstr = command.create_command()
nose.tools.eq_("-selcode,130", commstr)
def test_add_gridmapping(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.gridtype_operator,
cdoapi.cdo_command.regular_grid_type)
commstr = command.create_command()
nose.tools.eq_("-setgridtype,regular -selcode,130", commstr)
def test_add_specmapping(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.spectral_operator)
commstr = command.create_command()
nose.tools.eq_("-sp2gpl -selcode,130", commstr)
def test_select_codes(self):
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.select_code_operator, 131, 132)
commstr = command.create_command()
nose.tools.eq_("-selcode,130,131,132", commstr)
def test_select_codes():
command = cdoapi.cdo_command(130)
command.add_operator(cdoapi.cdo_command.select_code_operator, 131, 132)
commstr = command.create_command()
assert commstr == "-selcode,130,131,132"
def execute_tasks(tasks, filter_files=True, multi_threaded=False, once=False):
valid_tasks, varstasks = validate_tasks(tasks)
if not any(valid_tasks):
return []
task2files, task2freqs, fxkeys, keys2files = cluster_files(
valid_tasks, varstasks)
grids = [cmor_source.ifs_grid.point, cmor_source.ifs_grid.spec]
if filter_files:
keys_gp, timestamp_gp = set(), -1
keys_sp, timestamp_sp = set(), -1
filehandles = open_files(keys2files)
fxkeys2files = {k: keys2files[k] for k in fxkeys}
if any(gridpoint_files):
gridpoint_start_date = sorted(gridpoint_files.keys())[0]
first_gridpoint_file = gridpoint_files[gridpoint_start_date][0]
if ini_gridpoint_file != first_gridpoint_file and ini_gridpoint_file is not None:
with open(str(ini_gridpoint_file), 'r') as fin:
keys_gp, timestamp_gp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files,
grids[0], gridpoint_start_date, filehandles)
elif ini_gridpoint_file is not None:
with open(str(ini_gridpoint_file), 'r') as fin:
keys_gp, timestamp_gp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files, grids[0],
None, filehandles)
if any(spectral_files):
spectral_start_date = sorted(spectral_files.keys())[0]
first_spectral_file = spectral_files[spectral_start_date][0]
if ini_spectral_file != first_spectral_file and ini_spectral_file is not None:
with open(str(ini_spectral_file), 'r') as fin:
keys_sp, timestamp_sp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files,
grids[1], spectral_start_date, filehandles)
elif ini_spectral_file is not None:
with open(str(ini_spectral_file), 'r') as fin:
keys_sp, timestamp_sp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files, grids[1],
None, filehandles)
if multi_threaded:
threads = []
for file_list, grid, keys, timestamp in zip(
[gridpoint_files, spectral_files], grids, [keys_gp, keys_sp],
[timestamp_gp, timestamp_sp]):
thread = threading.Thread(target=filter_grib_files,
args=(file_list, keys2files, grid,
filehandles, 0, 0, once, keys,
timestamp))
threads.append(thread)
thread.start()
threads[0].join()
threads[1].join()
else:
for file_list, grid, keys, timestamp in zip(
[gridpoint_files, spectral_files], grids, [keys_gp, keys_sp],
[timestamp_gp, timestamp_sp]):
filter_grib_files(file_list,
keys2files,
grid,
filehandles,
month=0,
year=0,
once=once,
prev_keys=keys,
prev_timestamp=timestamp)
for handle in filehandles.values():
handle.close()
for task in task2files:
if task.status != cmor_task.status_failed:
file_list = task2files[task]
filter_output = os.path.join(temp_dir, file_list[0])
if len(file_list) > 1:
filter_output = os.path.join(temp_dir, '_'.join(file_list))
if not os.path.isfile(filter_output):
cdoapi.cdo_command().merge(
[os.path.join(temp_dir, f) for f in file_list],
filter_output)
setattr(task, cmor_task.filter_output_key, [filter_output])
for task in task2freqs:
if task.status != cmor_task.status_failed:
setattr(task, cmor_task.output_frequency_key, task2freqs[task])
return valid_tasks
def test_select_code():
command = cdoapi.cdo_command(130)
commstr = command.create_command()
assert commstr == "-selcode,130"
