def run_processor(task, work_dir, config) -> bool:
"""
:param task:
{
'type': 'ploto_esmdiag.processor.esmdiag_data_processor',
'model': 'gamil',
'tasks': [
{
"input_file_path": "",
"ps_file_path": "",
"output_file_path": "",
"var_name": "U",
"levels": [1000, 925, 850, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 10],
"interp_type": "linear",
"extrap": "false",
},
],
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
model = task["model"]
try:
vinterp_module = importlib.import_module(".models.{model}.vinterp".format(model=model), __package__)
return vinterp_module.run_task(task, work_dir, config)
except ImportError:
logger.error("model is not supported: {model}".format(model=model))
return False
def run_processor(task, work_dir, config) -> bool:
"""
:param task:
{
'type': 'ploto_esmdiag.processor.esmdiag_data_processor',
'action': 'filter',
'input_file':'',
'output_file': '',
'var_name': 'OLR',
'method': 'butterworth', # or 'lanczos'
'low_pass': 0.01,
'high_pass': 0.05,
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
method = task['method']
input_file_path = str(Path(Path(work_dir), task['input_file']))
output_file_path = str(Path(Path(work_dir), task['output_file']))
esmdiag_env = os.environ.copy()
esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"]
logger.info("run ncl script...")
ncl_script = Path(config["esmdiag"]["root"], 'ncl_scripts',
'filter_{method}.ncl'.format(method=method))
if not ncl_script.exists():
logger.error(
'filter method is not supported: {method}'.format(method=method))
ncl_command = [
'ncl -Q '
'{ncl_script} '
'var_path=\\"{var_path}\\" '
'var_name=\\"{var_name}\\" '
'fca={fca} '
'fcb={fcb} '
'out_path=\\"{out_path}\\"'.format(var_path=input_file_path,
var_name=task['var_name'],
fca=task['low_pass'],
fcb=task['high_pass'],
out_path=output_file_path,
ncl_script=ncl_script)
]
ncl_result = subprocess.run(
ncl_command,
env=esmdiag_env,
# start_new_session=True,
shell=True,
)
logger.info("run ncl script...done")
return True
def run_processor(task, work_dir, config) -> bool:
"""
:param task:
{
'type': 'ploto_esmdiag.processor.esmdiag_data_processor',
'action': 'anomaly',
'input_file':'',
'output_file': '',
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
input_file_path = str(Path(Path(work_dir), task['input_file']))
output_file_path = str(Path(Path(work_dir), task['output_file']))
from ploto.processor.cdo_processor.general import run_cdo
step1_task = {
'type': 'ploto.processor.cdo_processor',
'operator': 'general',
'params': ['timmean', input_file_path, output_file_path + ".step1"],
}
run_cdo(step1_task, work_dir, config)
step2_task = {
'type':
'ploto.processor.cdo_processor',
'operator':
'general',
'params': [
'sub', input_file_path, output_file_path + ".step1",
output_file_path
],
}
run_cdo(step2_task, work_dir, config)
Path(output_file_path + ".step1").unlink()
return True
def run_task(task, work_dir, config):
logger = get_logger()
metric = task['metric']
figure = task["figure"]
try:
figure_module = importlib.import_module(
"ploto_esmdiag.plotter.esmdiag_plotter.metrics.{metric}.figures.{figure}".format(
metric=metric,
figure=figure
), __package__)
except ImportError:
logger.error("can't found metric {metric} figure {figure}".format(
metric=metric,
figure=figure))
return False
ncl_script_path = getattr(figure_module, "NCL_SCRIPT_PATH")
from ploto_esmdiag.plotter.esmdiag_plotter.figure_task import draw_figures
draw_figures(ncl_script_path, task, work_dir, config)
return True
import os
from ploto_esmvaltool.fetcher.esmvalcore_fetcher import get_selected_files
from ploto.logger import get_logger
from esmvalcore._recipe import (
_special_name_to_dataset,
_get_dataset_info,
get_reference_levels,
get_cmor_levels
)
from esmvalcore.preprocessor import MULTI_MODEL_FUNCTIONS
logger = get_logger()
def update_variable_settings(
variable: typing.Dict,
settings: typing.Dict,
variables: typing.List,
config: typing.Dict,
):
"""
Update operation block settings for variable (combined variable).
Parameters
----------
variable
settings
def run_processor(task, work_dir, config) -> bool:
"""
:param task:
{
'type': 'ploto_esmdiag.processor.esmdiag_data_processor',
'action': 'detrend',
'dim': 0,
'input_file': 'GAMIL.gamil_wu_run11.PRC.daily.anomaly.area_avg_IO.1992-01-01:1993-12-31.nc',
'var_name': 'PRC',
'output_file': 'GAMIL.gamil_wu_run11.PRC.daily.anomaly.area_avg_IO.detrended.1992-01-01:1993-12-31.nc'
'common_config': {
'model_info': {
'id': "FGOALS-g3",
'atm_id': "GAMIL",
'ocn_id': "LICOM",
'ice_id': "CICE",
},
'case_info': {
'id': "gamil_wu_run11",
},
'date': {
'start': "1979-01-01",
'end': "1980-12-01"
}
}
}
:param work_dir:
:param config:
{
'esmdiag': {
'root': '/home/hujk/ploto/ploto/vendor/esmdiag'
}
}
:return:
"""
logger = get_logger()
input_file_path = str(Path(Path(work_dir), task['input_file']))
output_file_path = str(Path(Path(work_dir), task['output_file']))
esmdiag_env = os.environ.copy()
esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"]
ncl_script = Path(config["esmdiag"]["root"], 'ncl_scripts', 'detrend.ncl')
if not ncl_script.exists():
logger.error('ncl script is not found: {path}'.format(path=str(ncl_script)))
ncl_command = (
'ncl -Q '
'{ncl_script} '
'model_id=\\"{model_id}\\" '
'case_id=\\"{case_id}\\" '
'dim={dim} '
'var_name=\\"{var_name}\\" '
'var_path=\\"{var_path}\\" '
'out_path=\\"{out_path}\\"'.format(
model_id=task['common_config']['model_info']['atm_id'],
case_id=task['common_config']['case_info']['id'],
dim=task['dim'],
var_path=input_file_path,
var_name=task['var_name'],
out_path=output_file_path,
ncl_script=ncl_script))
logger.info("run ncl script...")
logger.info("=> {ncl_command}".format(ncl_command=ncl_command))
ncl_result = subprocess.run(
[ncl_command],
env=esmdiag_env,
# start_new_session=True,
shell=True,
)
logger.info("run ncl script...done")
return True
def run_task(task, work_dir, config) -> bool:
"""
:param task:
{
'type': 'ploto_esmdiag.processor.esmdiag_data_processor',
'action: 'vinterp',
'model': 'gamil',
'tasks': [
{
"input_file_path": "",
"ps_file_path": "",
"output_file_path": "",
"var_name": "U",
"levels": [1000, 925, 850, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 10],
"interp_type": "linear",
"extrap": "false"
},
]
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
vinterp_ncl_script = Path(Path(__file__).parent, "vinterp.ncl")
for vinterp_task in task["tasks"]:
input_file_path = str(
Path(Path(work_dir), vinterp_task["input_file_path"]))
output_file_path = str(
Path(Path(work_dir), vinterp_task["output_file_path"]))
ps_file_path = str(Path(Path(work_dir), vinterp_task["ps_file_path"]))
var_name = vinterp_task["var_name"]
levels = vinterp_task["levels"]
interp_type = vinterp_task["interp_type"]
if interp_type == "linear":
interp_type = 1
elif interp_type == "log":
interp_type = 2
else:
logger.error("interp type is not supported:", interp_type)
return False
extrap = vinterp_task["extrap"]
esmdiag_env = os.environ.copy()
esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"]
logger.info(
"run vinterp.ncl for {var_name}...".format(var_name=var_name))
ncl_command = [
'ncl -Q '
'ps_path=\\"{ps_path}\\" '
'var_path=\\"{var_path}\\" '
'var_name=\\"{var_name}\\" '
'plevs=\\(/{plevs}/\\) '
'interp_type={interp_type} '
'extrap={extrap} '
'out_path=\\"{out_path}\\" '
'{ncl_script}'.format(ps_path=ps_file_path,
var_path=input_file_path,
var_name=var_name,
plevs=",".join(
[str(level) for level in levels]),
interp_type=interp_type,
extrap=extrap,
out_path=output_file_path,
ncl_script=vinterp_ncl_script)
]
logger.info(' '.join(ncl_command))
ncl_result = subprocess.run(
ncl_command,
env=esmdiag_env,
# start_new_session=True,
shell=True,
)
logger.info(
"run vinterp.ncl for {var_name}...done".format(var_name=var_name))
return True
def run_plotter(plotter_task, work_dir, config):
"""
run ncldraw plotter
:param plotter_task: a dict config of plotter task.
{
'type': 'ploto.plotter.ncldraw_plotter',
'task_files': [
{
'file_path': 'task.xml',
'file_content': 'task file content'
}
],
'resource_files':[
{
'file_path': 'fill.xml', # resource file path, NOTE: this is a relative path currently.
'file_content': 'content', # resource file content
}
],
'time_level': '2017071400084', # YYYYMMDDHH + HHH, used by ncldraw program.
'image_path': 'image.png', # image file path
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
logger.info('prepare plot...')
# resource file
if 'resource_files' in plotter_task:
for a_resource_file_config in plotter_task['resource_files']:
generate_resource_file(a_resource_file_config, config)
# task file
cur_index = 0
for a_task_file_config in plotter_task['task_files']:
logger.info('task {cur_index}:'.format(cur_index=cur_index))
changed_task_file_path = generate_task_file(a_task_file_config,
work_dir)
param = {
'task_file': changed_task_file_path,
'time_level': plotter_task['time_level']
}
logger.info('running ncldraw...')
command = [
'ncldraw {task_file} {time_level}'.format(
task_file=param['task_file'], time_level=param['time_level'])
]
# print(command)
ncldraw_result = subprocess.run(command, shell=True)
# logger.debug(ncldraw_result.stdout)
# logger.debug(ncldraw_result.stderr)
logger.info('task {cur_index}: done'.format(cur_index=cur_index))
logger.info('running ncldraw...done')
def draw_figures(ncl_script, task, work_dir, config):
"""
:param ncl_script:
:param task:
{
common: {
model_info: {
id: "FGOALS-g3",
atm_id: "GAMIL",
ocn_id: "LICOM",
ice_id: "CICE",
},
case_info: {
id: "piControl-bugfix-licom-80368d",
},
date: {
start: "0030-01-01",
end: "0060-12-31"
}
},
}
:param work_dir:
:param config:
:return:
"""
logger = get_logger()
os.chdir(work_dir)
common_config = task["common"]
esmdiag_env = os.environ.copy()
esmdiag_env["ESMDIAG_ROOT"] = config["esmdiag"]["root"]
# logger.info(esmdiag_env)
logger.info("run ncl script...")
ncl_command = [
# '/bin/bash',
# '-i', '-c',
'ncl -Q '
'model_id=\\"{model_id}\\" '
'model_atm_id=\\"{model_atm_id}\\" '
'model_ocn_id=\\"{model_ocn_id}\\" '
'model_ice_id=\\"{model_ice_id}\\" '
'case_id=\\"{case_id}\\" '
'start_date=\\"{start_date}\\" '
'end_date=\\"{end_date}\\" '
'{ncl_script}'.format(
model_id=common_config["model_info"]["id"],
model_atm_id=common_config["model_info"]["atm_id"],
model_ocn_id=common_config["model_info"]["ocn_id"],
model_ice_id=common_config["model_info"]["ice_id"],
case_id=common_config["case_info"]["id"],
start_date=common_config["date"]["start"],
end_date=common_config["date"]["end"],
ncl_script=ncl_script)
]
logger.info(
"ncl command: {ncl_command}".format(ncl_command=' '.join(ncl_command)))
ncl_result = subprocess.run(
ncl_command,
env=esmdiag_env,
# start_new_session=True,
shell=True,
)
logger.info("run ncl script...done")