def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read("214.128", "sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param in [165, 166] newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def test_pressure_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("ua", "Amon") src = cmor_source.ifs_source.read("131.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "131.128.210.6") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.filter_output_key), [filepath]) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 131) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 600) % 2400) time = newtime os.remove(filepath)
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "79.128.1.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param == 79 newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read( "var214=sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "165.128.105_166.128.105") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, "path"), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) ok_(param in [165, 166]) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "79.128.1") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.output_path_key), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 79) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def execute(tasks, cleanup=True, autofilter=True): global log, start_date_, ifs_grid_descr_ supported_tasks = [ t for t in filter_tasks(tasks) if t.status == cmor_task.status_initialized ] log.info("Executing %d IFS tasks..." % len(supported_tasks)) mask_tasks = get_mask_tasks(supported_tasks) surf_pressure_tasks = get_sp_tasks(supported_tasks, autofilter) regular_tasks = [ t for t in supported_tasks if t not in surf_pressure_tasks ] tasks_todo = mask_tasks + surf_pressure_tasks + regular_tasks grid_descr_file = None if autofilter: tasks_todo = grib_filter.execute(tasks_todo, start_date_.month) for t in tasks_todo: if getattr(t.source, "grid_", None) == cmor_source.ifs_grid.point: filepaths = getattr(t, cmor_task.filter_output_key, []) if any(filepaths): grid_descr_file = filepaths[0] break else: for task in tasks_todo: grid = getattr(task.source, "grid_") if grid == cmor_source.ifs_grid.point: setattr(task, cmor_task.filter_output_key, [ifs_gridpoint_file_]) elif grid == cmor_source.ifs_grid.spec: setattr(task, cmor_task.filter_output_key, [ifs_spectral_file_]) else: log.error( "Task ifs source has unknown grid for %s in table %s" % (task.target.variable, task.target.table)) task.set_failed() setattr(task, cmor_task.output_frequency_key, output_frequency_) grid_descr_file = ifs_gridpoint_file_ log.info("Fetching grid description from %s ..." % grid_descr_file) ifs_grid_descr_ = cdoapi.cdo_command().get_grid_descr( grid_descr_file) if os.path.exists(grid_descr_file) else {} processed_tasks = [] try: log.info("Post-processing tasks...") processed_tasks = postprocess([ t for t in tasks_todo if t.status == cmor_task.status_initialized ]) for task in [t for t in processed_tasks if t in mask_tasks]: read_mask(task.target.variable, getattr(task, cmor_task.output_path_key)) cmorize([t for t in processed_tasks if t in supported_tasks]) except Exception: if cleanup: clean_tmp_data(processed_tasks, True) processed_tasks = [] raise finally: if cleanup: clean_tmp_data(processed_tasks, False)
def execute(tasks, nthreads=1): global log, start_date_, auto_filter_ supported_tasks = [ t for t in filter_tasks(tasks) if t.status == cmor_task.status_initialized ] log.info("Executing %d IFS tasks..." % len(supported_tasks)) mask_tasks = get_mask_tasks(supported_tasks) fx_tasks = [ t for t in supported_tasks if cmor_target.get_freq(t.target) == 0 ] surf_pressure_tasks = get_sp_tasks(supported_tasks) regular_tasks = [ t for t in supported_tasks if t not in surf_pressure_tasks and cmor_target.get_freq(t.target) != 0 ] # No fx filtering needed, cdo can handle this file if ifs_init_gridpoint_file_.endswith("+000000"): tasks_to_filter = surf_pressure_tasks + regular_tasks tasks_no_filter = fx_tasks + mask_tasks for task in tasks_no_filter: # dirty hack for orography being in ICMGG+000000 file... if task.target.variable in ["orog", "areacella"]: task.source.grid_ = cmor_source.ifs_grid.point if task.source.grid_id() == cmor_source.ifs_grid.spec: setattr(task, cmor_task.filter_output_key, [ifs_init_spectral_file_]) else: setattr(task, cmor_task.filter_output_key, [ifs_init_gridpoint_file_]) setattr(task, cmor_task.output_frequency_key, 0) else: tasks_to_filter = mask_tasks + fx_tasks + surf_pressure_tasks + regular_tasks tasks_no_filter = [] if auto_filter_: tasks_todo = tasks_no_filter + grib_filter.execute( tasks_to_filter, filter_files=do_post_process(), multi_threaded=(nthreads > 1)) else: tasks_todo = tasks_no_filter for task in tasks_to_filter: if task.source.grid_id() == cmor_source.ifs_grid.point: setattr(task, cmor_task.filter_output_key, ifs_gridpoint_files_.values()) tasks_todo.append(task) elif task.source.grid_id() == cmor_source.ifs_grid.spec: setattr(task, cmor_task.filter_output_key, ifs_spectral_files_.values()) tasks_todo.append(task) else: log.error( "Task ifs source has unknown grid for %s in table %s" % (task.target.variable, task.target.table)) task.set_failed() for task in tasks_todo: setattr(task, cmor_task.output_frequency_key, get_output_freq(task)) # First post-process surface pressure and mask tasks for task in list( set(tasks_todo).intersection(mask_tasks + surf_pressure_tasks)): postproc.post_process(task, temp_dir_, do_post_process()) for task in list(set(tasks_todo).intersection(mask_tasks)): read_mask(task.target.variable, getattr(task, cmor_task.output_path_key)) proctasks = list(set(tasks_todo).intersection(regular_tasks + fx_tasks)) if nthreads == 1: for task in proctasks: cmor_worker(task) else: pool = multiprocessing.Pool(processes=nthreads) pool.map(cmor_worker, proctasks) for task in proctasks: setattr(task, cmor_task.output_path_key, postproc.get_output_path(task, temp_dir_)) if cleanup_tmpdir(): clean_tmp_data(tasks_todo)
def execute(tasks, nthreads=1): global log, start_date_, auto_filter_ supported_tasks = [t for t in filter_tasks(tasks) if t.status == cmor_task.status_initialized] log.info("Executing %d IFS tasks..." % len(supported_tasks)) mask_tasks = get_mask_tasks(supported_tasks) fx_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) == 0] regular_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) != 0] script_tasks = [t for t in supported_tasks if validate_script_task(t) is not None] # Scripts in charge of their own filtering, can create a group of variables at once script_tasks_no_filter = [t for t in script_tasks if validate_script_task(t) == "false"] # Scripts creating single variable, filtering done by ece2cmor3 script_tasks_filter = list(set(script_tasks) - set(script_tasks_no_filter)) req_ps_tasks, extra_ps_tasks = get_sp_tasks(supported_tasks) # No fx filtering needed, cdo can handle this file if ifs_init_gridpoint_file_.endswith("+000000"): tasks_to_filter = extra_ps_tasks + regular_tasks + script_tasks_filter tasks_no_filter = fx_tasks + mask_tasks for task in tasks_no_filter: # dirty hack for orography being in ICMGG+000000 file... if task.target.variable in ["orog", "areacella"]: task.source.grid_ = cmor_source.ifs_grid.point if task.source.grid_id() == cmor_source.ifs_grid.spec: setattr(task, cmor_task.filter_output_key, [ifs_init_spectral_file_]) else: setattr(task, cmor_task.filter_output_key, [ifs_init_gridpoint_file_]) setattr(task, cmor_task.output_frequency_key, 0) else: tasks_to_filter = mask_tasks + fx_tasks + extra_ps_tasks + regular_tasks + script_tasks_filter tasks_no_filter = [] np = nthreads # Launch no-filter scripts jobs = [] tasks_per_script = cmor_utils.group(script_tasks_no_filter, lambda tsk: getattr(tsk, cmor_task.postproc_script_key)) for s, tasklist in tasks_per_script.items(): log.info("Launching script %s to process variables %s" % (s, ','.join([t.target.variable + " in " + t.target.table for t in tasklist]))) script_args = (s, str(scripts[s]["src"]), tasklist) if np == 1: script_worker(*script_args) else: p = multiprocessing.Process(name=s, target=script_worker, args=script_args) p.start() jobs.append(p) np -= 1 # Do filtering if auto_filter_: tasks_todo = tasks_no_filter + grib_filter.execute(tasks_to_filter, filter_files=do_post_process(), multi_threaded=(nthreads > 1)) else: tasks_todo = tasks_no_filter for task in tasks_to_filter: if task.source.grid_id() == cmor_source.ifs_grid.point: setattr(task, cmor_task.filter_output_key, ifs_gridpoint_files_.values()) tasks_todo.append(task) elif task.source.grid_id() == cmor_source.ifs_grid.spec: setattr(task, cmor_task.filter_output_key, ifs_spectral_files_.values()) tasks_todo.append(task) else: log.error("Task ifs source has unknown grid for %s in table %s" % (task.target.variable, task.target.table)) task.set_failed() for task in tasks_todo: setattr(task, cmor_task.output_frequency_key, get_output_freq(task)) # First post-process surface pressure and mask tasks for task in list(set(tasks_todo).intersection(mask_tasks + req_ps_tasks + extra_ps_tasks)): postproc.post_process(task, temp_dir_, do_post_process()) for task in list(set(tasks_todo).intersection(mask_tasks)): read_mask(task.target.variable, getattr(task, cmor_task.output_path_key)) proctasks = list(set(tasks_todo).intersection(regular_tasks + fx_tasks)) if np == 1: for task in proctasks: cmor_worker(task) else: pool = multiprocessing.Pool(processes=np) pool.map(cmor_worker, proctasks) for task in proctasks: setattr(task, cmor_task.output_path_key, postproc.get_output_path(task, temp_dir_)) for job in jobs: job.join() if cleanup_tmpdir(): clean_tmp_data(tasks_todo)