def run_miles_block(): steps = [] steps.extend(get_fetcher_steps()) steps.extend(get_processor_steps()) steps.extend(get_plotter_steps()) config = zmnam_config.config run_ploto({"data": {"steps": steps}}, config)
def test_precip(): print("test begin...") from ploto_server.common.esmdiag.metrics.climo.figures.precip import generate_figure_task task = generate_figure_task( figure_config={ 'name': 'precip', 'task_id': str(uuid.uuid4()) }, common_config={ 'model_info': { 'id': "FGOALS-g3", 'atm_id': "GAMIL", 'ocn_id': "LICOM", 'ice_id': "CICE", }, 'case_info': { 'id': "gamil_wu_run11", }, 'date': { 'start': "1981-01-01", 'end': "1981-12-01" } }, server_config={ 'esmdiag': { 'web': { 'url': 'http://192.168.212.201:8088', 'api': { 'task_status': '/api/task/status' }, 'plot_base_dir': '/home/hujk/clusterfs/wangdp/ploto/plot_base', }, }, }, ) print(json.dumps(task, indent=2)) from ploto.run import run_ploto run_ploto(message={'data': task}, config={ 'base': { 'run_base_dir': '/home/hujk/clusterfs/wangdp/ploto/run_base', 'python_exe': '/home/hujk/.pyenv/versions/ploto-env/bin/python3' }, 'edp_fetcher': { 'edp_module_path': "/home/hujk/pyProject/" }, 'esmdiag': { 'root': '/home/hujk/ploto/ploto/vendor/esmdiag' } })
def run_dry_days(): steps = [] steps.extend(get_fetcher_tasks()) steps.extend(get_processor_tasks()) steps.extend(get_combine_metadata_tasks()) steps.append(get_plotter_task()) config = diurnal_config.config run_ploto({"data": {"steps": steps}}, config)
def run_climwip(): steps = [] steps.extend(get_fetcher_steps()) steps.extend(get_processor_steps()) steps.extend(get_plotter_steps()) config = climwip_config.config Path(config["base"]["run_base_dir"]).mkdir(parents=True, exist_ok=True) run_ploto({ "data": { "steps": steps } }, config)
def run_dry_days(): steps = [] exp_datasets = dry_days_recipe.exp_datasets variables = dry_days_recipe.variables tasks = [ { "exp_dataset": d, "variable": v, } for d, v in itertools.product(exp_datasets, variables) ] steps.extend([get_fetcher(**task) for task in tasks]) datasets = [ { **d, "recipe_dataset_index": index, "alias": d["dataset"] } for index, d in enumerate(exp_datasets) ] for d, v in itertools.product(datasets, variables): steps.append(get_processor(d, v)) plot_task = generate_default_plot_task() steps.extend([ { "step_type": "plotter", "type": "ploto_esmvaltool.plotter.esmvaltool_diag_plotter", **plot_task, "config": dry_days_config.plot_config, "input_files": [ "{work_dir}" + f"/processor/preproc/{d['dataset']}/{v['variable_group']}/metadata.yml" for d in datasets ], } for v in variables ]) config = dry_days_config.config run_ploto({ "data": { "steps": steps } }, config)
def run_meteor_draw_task(plot_task: dict): task = { 'steps': [ { 'step_type': 'plotter', 'type': 'ploto_gidat.plotter.meteor_draw_plotter', 'plot_task': plot_task, }, ], } message = { 'app': 'ploto', 'type': 'ploto-gidat', 'timestamp': time.time(), 'data': task } config = current_app.config["server_config"]["ploto"] from ploto.run import run_ploto run_ploto(message, config)
def run_example(): work_dir = "./dist/cases/case1/run" esmvaltool_config_file = "./dist/config.yml" # load recipe.yml recipe_file_path = "./dist/recipe.yml" with open(recipe_file_path, "r") as f: raw_recipe = yaml.safe_load(f) steps = [] steps.extend(get_processor_tasks(esmvaltool_config_file, raw_recipe)) steps.extend(get_plotter_task(esmvaltool_config_file, raw_recipe)) config = { "esmvaltool_python_plotter": {}, "esmvaltool": { "executables": { "py": "/home/hujk/.pyenv/versions/anaconda3-2019.10/envs/esmvaltool/bin/python3" }, "recipes": { "base": "/home/hujk/ploto/esmvaltool/study/esmvaltool/ESMValTool/esmvaltool/recipes", }, "diag_scripts": { "base": "/home/hujk/ploto/esmvaltool/study/esmvaltool/ESMValTool/esmvaltool/diag_scripts", }, }, "base": { "run_base_dir": "/home/hujk/ploto/ploto-esmvaltool/dist/cases/case1/run" } } run_ploto({"data": {"steps": steps}}, config)
def test_esmdiag_edp(): """ test A climo/precip ESMDiag task using data from EDP(earth data platform) """ common_config = { 'model_info': { 'id': "FGOALS-g3", 'atm_id': "GAMIL", 'ocn_id': "LICOM", 'ice_id': "CICE", }, 'case_info': { 'id': "gamil_wu_run11", }, 'date': { 'start': "1981-01-01", 'end': "1981-12-01" } } task = { 'steps': [ { 'step_type': 'fetcher', 'common': common_config, 'type': 'ploto_esmdiag.fetcher.edp_fetcher', 'query_param': { 'type': 'nc', 'output_dir': './data', 'file_prefix': 'GAMIL.gamil_wu_run11.step1', 'date_range': ['19810101', '19820101'], 'field_names': [ 'PRECT', 'PRECC', 'PRECL', 'PS' ], 'datedif': 'h0' }, }, { 'step_type': 'fetcher', 'common': common_config, 'type': 'ploto_esmdiag.fetcher.edp_fetcher', 'query_param': { 'type': 'nc', 'output_dir': './data', 'file_prefix': 'GAMIL.gamil_wu_run11.step2', 'date_range': ['19820101', '19820101'], 'field_names': [ 'gw' ], 'datedif': 'h0' } }, { 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'PS', 'startdate': '1981-01-01', 'enddate': '1981-12-01' }, 'input_files': [ './data/GAMIL.gamil_wu_run11.step1.*.nc' ], 'output_file': './GAMIL.gamil_wu_run11.PS.monthly.1981-01-01:1981-12-01.nc', }, { 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'PRECT', 'startdate': '1981-01-01', 'enddate': '1981-12-01' }, 'input_files': [ './data/GAMIL.gamil_wu_run11.step1.*.nc' ], 'output_file': './GAMIL.gamil_wu_run11.PRECT.monthly.1981-01-01:1981-12-01.nc', }, { 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'PRECC', 'startdate': '1981-01-01', 'enddate': '1981-12-01' }, 'input_files': [ './data/GAMIL.gamil_wu_run11.step1.*.nc' ], 'output_file': './GAMIL.gamil_wu_run11.PRECC.monthly.1981-01-01:1981-12-01.nc', }, { 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'PRECL', 'startdate': '1981-01-01', 'enddate': '1981-12-01' }, 'input_files': [ './data/GAMIL.gamil_wu_run11.step1.*.nc' ], 'output_file': './GAMIL.gamil_wu_run11.PRECL.monthly.1981-01-01:1981-12-01.nc', }, { 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'gw', }, 'input_files': [ './data/GAMIL.gamil_wu_run11.step2.*.nc' ], 'output_file': './FGOALS-g3.gamil_wu_run11.gw.nc', }, { 'step_type': 'plotter', 'type': 'ploto_esmdiag.plotter.esmdiag_plotter', 'metric': 'climo', 'figure': 'precip', 'common': common_config, }, ], } message = { 'app': 'ploto', 'type': 'ploto', 'timestamp': time.time(), 'data': task } config = { 'base': { 'run_base_dir': '/home/hujk/clusterfs/wangdp/ploto/run_base', 'python_exe': '/home/hujk/.pyenv/versions/ploto-env/bin/python3' }, 'edp_fetcher': { 'edp_module_path': "/home/hujk/pyProject/" }, 'esmdiag': { 'root': '/home/hujk/ploto/ploto/vendor/esmdiag' } } from ploto.run import run_ploto run_ploto(message, config)
def receive_giset_plot_example(): """ Run plot task in server. POST DATA: { "data_source": { "system_name": "grapes_gfs_gmf", "username": "******", "user_id": "u0184", "data_type": "storage", "routing_key": "GFSNEW.niuxingy", "test_ID": "TG2000617" }, "start_time": "2020070100000", "step": "6", "hh_list": [ "00", "12" ], "end_time": "2020070200000", "fcstlen": "96", "plot_task": { // ...skip... } } RETURN DATAļ¼ { "status": "ok", } """ request_data = request.json data_source = request_data["data_source"] start_valid_time = request_data["start_time"] end_valid_time = request_data["end_time"] forecast_length = int(request_data["fcstlen"]) forecast_step = int(request_data["step"]) start_hours = [int(f) for f in request_data["hh_list"]] plot_task_template = request_data['plot_task'] start_time = pd.to_datetime(start_valid_time[:10], format="%Y%m%d%H") end_time = pd.to_datetime(end_valid_time[:10], format="%Y%m%d%H") tasks = generate_meteor_draw_tasks( data_source, start_time, end_time, start_hours, forecast_length, forecast_step, plot_task_template, ) config = current_app.config["server_config"]["ploto"] for task in tasks: message = { 'app': 'ploto', 'type': 'ploto-gidat', 'timestamp': time.time(), 'data': task } run_ploto(message, config) return jsonify({ 'status': 'ok', })
def test_anomaly(): print("test begin...") common_config = { 'model_info': { 'id': "FGOALS-g3", 'atm_id': "GAMIL", 'ocn_id': "LICOM", 'ice_id': "CICE", }, 'case_info': { 'id': "gamil_wu_run11", }, 'date': { 'start': "1982-01-01", 'end': "1991-12-31" }, } start_date = datetime.datetime.strptime(common_config['date']['start'], "%Y-%m-%d") end_date = datetime.datetime.strptime(common_config['date']['end'], "%Y-%m-%d") date_range = [ (start_date - datetime.timedelta(days=30)).strftime("%Y%m%d"), (end_date + datetime.timedelta(days=30)).strftime("%Y%m%d") ] steps = [] steps.append({ 'step_type': 'fetcher', 'common': common_config, 'type': 'ploto_esmdiag.fetcher.edp_fetcher', 'query_param': { 'type': 'nc', 'output_dir': './data', 'file_prefix': 'step1.', 'date_range': date_range, 'field_names': ['U'], 'datedif': 'h1' }, }) time_range_string = "{start_date}:{end_date}".format( start_date=common_config['date']['start'], end_date=common_config['date']['end'], ) output_file_pattern = "{file_prefix}.{name}.daily.{time_range}.nc" steps.append({ 'step_type': 'processor', 'type': 'ploto.processor.cdo_processor', 'operator': 'select', 'params': { 'name': 'U', 'startdate': common_config['date']['start'], 'enddate': common_config['date']['end'] }, 'input_files': ['./data/{step1_file_prefix}.*.nc'.format(step1_file_prefix='step1.')], 'output_file': output_file_pattern.format( file_prefix='step2.', time_range=time_range_string, name='U', ), }) steps.append({ 'step_type': 'processor', 'type': 'ploto_esmdiag.processor.esmdiag_data_processor', 'action': 'anomaly', 'input_file': output_file_pattern.format(file_prefix='step2.', time_range=time_range_string, name='U'), 'output_file': "{file_prefix}.{name}.daily.anomaly.{time_range}.nc".format( file_prefix='step3.', time_range=time_range_string, name='U'), }) from ploto.run import run_ploto run_ploto(message={'data': { 'steps': steps }}, config={ 'base': { 'run_base_dir': '/home/hujk/clusterfs/wangdp/ploto/run_base', 'python_exe': '/home/hujk/.pyenv/versions/ploto-env/bin/python3' }, 'edp_fetcher': { 'edp_module_path': "/home/hujk/clusterfs/wangdp/pyProject" }, 'esmdiag': { 'root': '/home/hujk/ploto/ploto/vendor/esmdiag' } })