def run_processor(task, work_dir, config) -> bool: """ :param task: { 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'model': 'gamil', 'tasks': [ { "input_file_path": "", "ps_file_path": "", "output_file_path": "", "var_name": "U", "levels": [1000, 925, 850, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 10], "interp_type": "linear", "extrap": "false", }, ], } :param work_dir: :param config: :return: """ logger = get_logger() model = task["model"] try: vinterp_module = importlib.import_module(".models.{model}.vinterp".format(model=model), __package__) return vinterp_module.run_task(task, work_dir, config) except ImportError: logger.error("model is not supported: {model}".format(model=model)) return False
def run_processor(task, work_dir, config) -> bool: """ :param task: { 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'action': 'filter', 'input_file':'', 'output_file': '', 'var_name': 'OLR', 'method': 'butterworth', # or 'lanczos' 'low_pass': 0.01, 'high_pass': 0.05, } :param work_dir: :param config: :return: """ logger = get_logger() method = task['method'] input_file_path = str(Path(Path(work_dir), task['input_file'])) output_file_path = str(Path(Path(work_dir), task['output_file'])) esmdiag_env = os.environ.copy() esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"] logger.info("run ncl script...") ncl_script = Path(config["esmdiag"]["root"], 'ncl_scripts', 'filter_{method}.ncl'.format(method=method)) if not ncl_script.exists(): logger.error( 'filter method is not supported: {method}'.format(method=method)) ncl_command = [ 'ncl -Q ' '{ncl_script} ' 'var_path=\\"{var_path}\\" ' 'var_name=\\"{var_name}\\" ' 'fca={fca} ' 'fcb={fcb} ' 'out_path=\\"{out_path}\\"'.format(var_path=input_file_path, var_name=task['var_name'], fca=task['low_pass'], fcb=task['high_pass'], out_path=output_file_path, ncl_script=ncl_script) ] ncl_result = subprocess.run( ncl_command, env=esmdiag_env, # start_new_session=True, shell=True, ) logger.info("run ncl script...done") return True
def run_processor(task, work_dir, config) -> bool: """ :param task: { 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'action': 'anomaly', 'input_file':'', 'output_file': '', } :param work_dir: :param config: :return: """ logger = get_logger() input_file_path = str(Path(Path(work_dir), task['input_file'])) output_file_path = str(Path(Path(work_dir), task['output_file'])) from ploto.processor.cdo_processor.general import run_cdo step1_task = { 'type': 'ploto.processor.cdo_processor', 'operator': 'general', 'params': ['timmean', input_file_path, output_file_path + ".step1"], } run_cdo(step1_task, work_dir, config) step2_task = { 'type': 'ploto.processor.cdo_processor', 'operator': 'general', 'params': [ 'sub', input_file_path, output_file_path + ".step1", output_file_path ], } run_cdo(step2_task, work_dir, config) Path(output_file_path + ".step1").unlink() return True
def run_task(task, work_dir, config): logger = get_logger() metric = task['metric'] figure = task["figure"] try: figure_module = importlib.import_module( "ploto_esmdiag.plotter.esmdiag_plotter.metrics.{metric}.figures.{figure}".format( metric=metric, figure=figure ), __package__) except ImportError: logger.error("can't found metric {metric} figure {figure}".format( metric=metric, figure=figure)) return False ncl_script_path = getattr(figure_module, "NCL_SCRIPT_PATH") from ploto_esmdiag.plotter.esmdiag_plotter.figure_task import draw_figures draw_figures(ncl_script_path, task, work_dir, config) return True
import os from ploto_esmvaltool.fetcher.esmvalcore_fetcher import get_selected_files from ploto.logger import get_logger from esmvalcore._recipe import ( _special_name_to_dataset, _get_dataset_info, get_reference_levels, get_cmor_levels ) from esmvalcore.preprocessor import MULTI_MODEL_FUNCTIONS logger = get_logger() def update_variable_settings( variable: typing.Dict, settings: typing.Dict, variables: typing.List, config: typing.Dict, ): """ Update operation block settings for variable (combined variable). Parameters ---------- variable settings
def run_processor(task, work_dir, config) -> bool: """ :param task: { 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'action': 'detrend', 'dim': 0, 'input_file': 'GAMIL.gamil_wu_run11.PRC.daily.anomaly.area_avg_IO.1992-01-01:1993-12-31.nc', 'var_name': 'PRC', 'output_file': 'GAMIL.gamil_wu_run11.PRC.daily.anomaly.area_avg_IO.detrended.1992-01-01:1993-12-31.nc' 'common_config': { 'model_info': { 'id': "FGOALS-g3", 'atm_id': "GAMIL", 'ocn_id': "LICOM", 'ice_id': "CICE", }, 'case_info': { 'id': "gamil_wu_run11", }, 'date': { 'start': "1979-01-01", 'end': "1980-12-01" } } } :param work_dir: :param config: { 'esmdiag': { 'root': '/home/hujk/ploto/ploto/vendor/esmdiag' } } :return: """ logger = get_logger() input_file_path = str(Path(Path(work_dir), task['input_file'])) output_file_path = str(Path(Path(work_dir), task['output_file'])) esmdiag_env = os.environ.copy() esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"] ncl_script = Path(config["esmdiag"]["root"], 'ncl_scripts', 'detrend.ncl') if not ncl_script.exists(): logger.error('ncl script is not found: {path}'.format(path=str(ncl_script))) ncl_command = ( 'ncl -Q ' '{ncl_script} ' 'model_id=\\"{model_id}\\" ' 'case_id=\\"{case_id}\\" ' 'dim={dim} ' 'var_name=\\"{var_name}\\" ' 'var_path=\\"{var_path}\\" ' 'out_path=\\"{out_path}\\"'.format( model_id=task['common_config']['model_info']['atm_id'], case_id=task['common_config']['case_info']['id'], dim=task['dim'], var_path=input_file_path, var_name=task['var_name'], out_path=output_file_path, ncl_script=ncl_script)) logger.info("run ncl script...") logger.info("=> {ncl_command}".format(ncl_command=ncl_command)) ncl_result = subprocess.run( [ncl_command], env=esmdiag_env, # start_new_session=True, shell=True, ) logger.info("run ncl script...done") return True
def run_task(task, work_dir, config) -> bool: """ :param task: { 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'action: 'vinterp', 'model': 'gamil', 'tasks': [ { "input_file_path": "", "ps_file_path": "", "output_file_path": "", "var_name": "U", "levels": [1000, 925, 850, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 10], "interp_type": "linear", "extrap": "false" }, ] } :param work_dir: :param config: :return: """ logger = get_logger() vinterp_ncl_script = Path(Path(__file__).parent, "vinterp.ncl") for vinterp_task in task["tasks"]: input_file_path = str( Path(Path(work_dir), vinterp_task["input_file_path"])) output_file_path = str( Path(Path(work_dir), vinterp_task["output_file_path"])) ps_file_path = str(Path(Path(work_dir), vinterp_task["ps_file_path"])) var_name = vinterp_task["var_name"] levels = vinterp_task["levels"] interp_type = vinterp_task["interp_type"] if interp_type == "linear": interp_type = 1 elif interp_type == "log": interp_type = 2 else: logger.error("interp type is not supported:", interp_type) return False extrap = vinterp_task["extrap"] esmdiag_env = os.environ.copy() esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"] logger.info( "run vinterp.ncl for {var_name}...".format(var_name=var_name)) ncl_command = [ 'ncl -Q ' 'ps_path=\\"{ps_path}\\" ' 'var_path=\\"{var_path}\\" ' 'var_name=\\"{var_name}\\" ' 'plevs=\\(/{plevs}/\\) ' 'interp_type={interp_type} ' 'extrap={extrap} ' 'out_path=\\"{out_path}\\" ' '{ncl_script}'.format(ps_path=ps_file_path, var_path=input_file_path, var_name=var_name, plevs=",".join( [str(level) for level in levels]), interp_type=interp_type, extrap=extrap, out_path=output_file_path, ncl_script=vinterp_ncl_script) ] logger.info(' '.join(ncl_command)) ncl_result = subprocess.run( ncl_command, env=esmdiag_env, # start_new_session=True, shell=True, ) logger.info( "run vinterp.ncl for {var_name}...done".format(var_name=var_name)) return True
def run_plotter(plotter_task, work_dir, config): """ run ncldraw plotter :param plotter_task: a dict config of plotter task. { 'type': 'ploto.plotter.ncldraw_plotter', 'task_files': [ { 'file_path': 'task.xml', 'file_content': 'task file content' } ], 'resource_files':[ { 'file_path': 'fill.xml', # resource file path, NOTE: this is a relative path currently. 'file_content': 'content', # resource file content } ], 'time_level': '2017071400084', # YYYYMMDDHH + HHH, used by ncldraw program. 'image_path': 'image.png', # image file path } :param work_dir: :param config: :return: """ logger = get_logger() logger.info('prepare plot...') # resource file if 'resource_files' in plotter_task: for a_resource_file_config in plotter_task['resource_files']: generate_resource_file(a_resource_file_config, config) # task file cur_index = 0 for a_task_file_config in plotter_task['task_files']: logger.info('task {cur_index}:'.format(cur_index=cur_index)) changed_task_file_path = generate_task_file(a_task_file_config, work_dir) param = { 'task_file': changed_task_file_path, 'time_level': plotter_task['time_level'] } logger.info('running ncldraw...') command = [ 'ncldraw {task_file} {time_level}'.format( task_file=param['task_file'], time_level=param['time_level']) ] # print(command) ncldraw_result = subprocess.run(command, shell=True) # logger.debug(ncldraw_result.stdout) # logger.debug(ncldraw_result.stderr) logger.info('task {cur_index}: done'.format(cur_index=cur_index)) logger.info('running ncldraw...done')
def draw_figures(ncl_script, task, work_dir, config): """ :param ncl_script: :param task: { common: { model_info: { id: "FGOALS-g3", atm_id: "GAMIL", ocn_id: "LICOM", ice_id: "CICE", }, case_info: { id: "piControl-bugfix-licom-80368d", }, date: { start: "0030-01-01", end: "0060-12-31" } }, } :param work_dir: :param config: :return: """ logger = get_logger() os.chdir(work_dir) common_config = task["common"] esmdiag_env = os.environ.copy() esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"] # logger.info(esmdiag_env) logger.info("run ncl script...") ncl_command = [ # '/bin/bash', # '-i', '-c', 'ncl -Q ' 'model_id=\\"{model_id}\\" ' 'model_atm_id=\\"{model_atm_id}\\" ' 'model_ocn_id=\\"{model_ocn_id}\\" ' 'model_ice_id=\\"{model_ice_id}\\" ' 'case_id=\\"{case_id}\\" ' 'start_date=\\"{start_date}\\" ' 'end_date=\\"{end_date}\\" ' '{ncl_script}'.format( model_id=common_config["model_info"]["id"], model_atm_id=common_config["model_info"]["atm_id"], model_ocn_id=common_config["model_info"]["ocn_id"], model_ice_id=common_config["model_info"]["ice_id"], case_id=common_config["case_info"]["id"], start_date=common_config["date"]["start"], end_date=common_config["date"]["end"], ncl_script=ncl_script) ] logger.info( "ncl command: {ncl_command}".format(ncl_command=' '.join(ncl_command))) ncl_result = subprocess.run( ncl_command, env=esmdiag_env, # start_new_session=True, shell=True, ) logger.info("run ncl script...done")