def filter_grib_files(file_list, keys2files, grid, handles=None, month=0, year=0, once=False): dates = sorted(file_list.keys()) for i in range(len(dates)): date = dates[i] if month != 0 and year != 0 and (date.month, date.year) != (month, year): continue prev_grib_file, cur_grib_file = file_list[date] prev_chained = i > 0 and (prev_grib_file == file_list[dates[i - 1]][1]) if prev_grib_file is not None and not prev_chained: with open(prev_grib_file, 'r') as fin: proc_initial_month(date.month, grib_file.create_grib_file(fin), keys2files, grid, handles, once) next_chained = i < len(dates) - 1 and (cur_grib_file == file_list[dates[i + 1]][0]) with open(cur_grib_file, 'r') as fin: log.info("Filtering grib file %s..." % cur_grib_file) if next_chained: proc_grib_file(grib_file.create_grib_file(fin), keys2files, grid, handles, once) else: proc_final_month(date.month, grib_file.create_grib_file(fin), keys2files, grid, handles, once)
def initialize(gpfiles, shfiles, tmpdir): global gridpoint_files, spectral_files, temp_dir, varsfreq, accum_codes grib_file.initialize() gridpoint_files = { d: (get_prev_file(gpfiles[d]), gpfiles[d]) for d in gpfiles.keys() } spectral_files = { d: (get_prev_file(shfiles[d]), shfiles[d]) for d in shfiles.keys() } temp_dir = tmpdir accum_codes = load_accum_codes( os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "grib_codes.json")) gpdate = sorted( gridpoint_files.keys())[0] if any(gridpoint_files) else None shdate = sorted(spectral_files.keys())[0] if any(spectral_files) else None gpfile = gridpoint_files[gpdate][1] if any(gridpoint_files) else None shfile = spectral_files[shdate][1] if any(spectral_files) else None if gpfile is not None: with open(gpfile) as gpf: varsfreq.update( inspect_day(grib_file.create_grib_file(gpf), grid=cmor_source.ifs_grid.point)) update_sp_key(gpfile) if shfile is not None: with open(shfile) as shf: varsfreq.update( inspect_day(grib_file.create_grib_file(shf), grid=cmor_source.ifs_grid.spec)) update_sp_key(shfile)
def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read("214.128", "sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param in [165, 166] newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "79.128.1.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param == 79 newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def initialize(gpfile, shfile, tmpdir): global gridpoint_file, prev_gridpoint_file, spectral_file, prev_spectral_file, temp_dir, varsfreq, accum_codes gridpoint_file = gpfile spectral_file = shfile temp_dir = tmpdir accum_codes = load_accum_codes( os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "grib_codes.json")) prev_gridpoint_file, prev_spectral_file = get_prev_files(gridpoint_file) with open(gpfile) as gpf, open(shfile) as shf: varsfreq.update( inspect_day(grib_file.create_grib_file(gpf), grid=cmor_source.ifs_grid.point)) varsfreq.update( inspect_day(grib_file.create_grib_file(shf), grid=cmor_source.ifs_grid.spec))
def test_pressure_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("ua", "Amon") src = cmor_source.ifs_source.read("131.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "131.128.210.6") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.filter_output_key), [filepath]) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 131) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 600) % 2400) time = newtime os.remove(filepath)
def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read( "var214=sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "165.128.105_166.128.105") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, "path"), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) ok_(param in [165, 166]) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "79.128.1") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.output_path_key), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 79) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def filter_grib_files(file_list, keys2files, grid, handles=None, month=0, year=0, once=False, prev_keys=(), prev_timestamp=-1): dates = sorted(file_list.keys()) cache = None if once else build_fast_forward_cache(keys2files, grid) keys, timestamp = prev_keys, prev_timestamp for i in range(len(dates)): date = dates[i] if month != 0 and year != 0 and (date.month, date.year) != (month, year): continue prev_grib_file, cur_grib_file = file_list[date] prev_chained = i > 0 and (os.path.realpath(prev_grib_file) == os.path.realpath( file_list[dates[i - 1]][1])) if prev_grib_file is not None and not prev_chained: with open(prev_grib_file, 'r') as fin: log.info("Filtering grib file %s..." % os.path.abspath(prev_grib_file)) keys, timestamp = proc_initial_month( date.month, grib_file.create_grib_file(fin), keys2files, grid, handles, keys, timestamp, once) next_chained = i < len(dates) - 1 and (os.path.realpath(cur_grib_file) == os.path.realpath( file_list[dates[i + 1]][0])) with open(cur_grib_file, 'r') as fin: log.info("Filtering grib file %s..." % os.path.abspath(cur_grib_file)) if next_chained: keys, timestamp = proc_grib_file( grib_file.create_grib_file(fin), keys2files, grid, handles, keys, timestamp, once, cache) else: proc_final_month(date.month, grib_file.create_grib_file(fin), keys2files, grid, handles, keys, timestamp, once, cache)
def download_sftlf(task): filtered_grib = getattr(task, cmor_task.filter_output_key)[0] Nj = 0 with open(filtered_grib, 'r') as gfile: grib = grib_file.create_grib_file(gfile) while Nj == 0: grib.read_next(headers_only=True) Nj = int(grib.get_field("Nj")) fname = "fx-sftlf-EC-Earth3-T" + str(Nj - 1) + ".nc" fullpath = os.path.join(os.path.dirname(__file__), "resources", "b2share-data", fname) if not os.path.isfile(fullpath): if not cmor_utils.get_from_b2share(fname, fullpath): log.fatal("The file %s could not be downloaded, please install manually at %s" % (fname, fullpath)) return None return fullpath
def proc_mon(month, cur_grib_file, prev_grib_file, handles=None): if prev_grib_file: with open(prev_grib_file, 'r') as fin: proc_prev_month(month, grib_file.create_grib_file(fin), handles) with open(cur_grib_file, 'r') as fin: proc_next_month(month, grib_file.create_grib_file(fin), handles)
def execute_tasks(tasks, filter_files=True, multi_threaded=False, once=False): valid_tasks, varstasks = validate_tasks(tasks) if not any(valid_tasks): return [] task2files, task2freqs, fxkeys, keys2files = cluster_files( valid_tasks, varstasks) grids = [cmor_source.ifs_grid.point, cmor_source.ifs_grid.spec] if filter_files: keys_gp, timestamp_gp = set(), -1 keys_sp, timestamp_sp = set(), -1 filehandles = open_files(keys2files) fxkeys2files = {k: keys2files[k] for k in fxkeys} if any(gridpoint_files): gridpoint_start_date = sorted(gridpoint_files.keys())[0] first_gridpoint_file = gridpoint_files[gridpoint_start_date][0] if ini_gridpoint_file != first_gridpoint_file and ini_gridpoint_file is not None: with open(str(ini_gridpoint_file), 'r') as fin: keys_gp, timestamp_gp = filter_fx_variables( grib_file.create_grib_file(fin), fxkeys2files, grids[0], gridpoint_start_date, filehandles) elif ini_gridpoint_file is not None: with open(str(ini_gridpoint_file), 'r') as fin: keys_gp, timestamp_gp = filter_fx_variables( grib_file.create_grib_file(fin), fxkeys2files, grids[0], None, filehandles) if any(spectral_files): spectral_start_date = sorted(spectral_files.keys())[0] first_spectral_file = spectral_files[spectral_start_date][0] if ini_spectral_file != first_spectral_file and ini_spectral_file is not None: with open(str(ini_spectral_file), 'r') as fin: keys_sp, timestamp_sp = filter_fx_variables( grib_file.create_grib_file(fin), fxkeys2files, grids[1], spectral_start_date, filehandles) elif ini_spectral_file is not None: with open(str(ini_spectral_file), 'r') as fin: keys_sp, timestamp_sp = filter_fx_variables( grib_file.create_grib_file(fin), fxkeys2files, grids[1], None, filehandles) if multi_threaded: threads = [] for file_list, grid, keys, timestamp in zip( [gridpoint_files, spectral_files], grids, [keys_gp, keys_sp], [timestamp_gp, timestamp_sp]): thread = threading.Thread(target=filter_grib_files, args=(file_list, keys2files, grid, filehandles, 0, 0, once, keys, timestamp)) threads.append(thread) thread.start() threads[0].join() threads[1].join() else: for file_list, grid, keys, timestamp in zip( [gridpoint_files, spectral_files], grids, [keys_gp, keys_sp], [timestamp_gp, timestamp_sp]): filter_grib_files(file_list, keys2files, grid, filehandles, month=0, year=0, once=once, prev_keys=keys, prev_timestamp=timestamp) for handle in filehandles.values(): handle.close() for task in task2files: if task.status != cmor_task.status_failed: file_list = task2files[task] filter_output = os.path.join(temp_dir, file_list[0]) if len(file_list) > 1: filter_output = os.path.join(temp_dir, '_'.join(file_list)) if not os.path.isfile(filter_output): cdoapi.cdo_command().merge( [os.path.join(temp_dir, f) for f in file_list], filter_output) setattr(task, cmor_task.filter_output_key, [filter_output]) for task in task2freqs: if task.status != cmor_task.status_failed: setattr(task, cmor_task.output_frequency_key, task2freqs[task]) return valid_tasks