def run_miles_block():
steps = []
steps.extend(get_fetcher_steps())
steps.extend(get_processor_steps())
steps.extend(get_plotter_steps())
config = zmnam_config.config
run_ploto({"data": {"steps": steps}}, config)
def test_precip():
print("test begin...")
from ploto_server.common.esmdiag.metrics.climo.figures.precip import generate_figure_task
task = generate_figure_task(
figure_config={
'name': 'precip',
'task_id': str(uuid.uuid4())
},
common_config={
'model_info': {
'id': "FGOALS-g3",
'atm_id': "GAMIL",
'ocn_id': "LICOM",
'ice_id': "CICE",
},
'case_info': {
'id': "gamil_wu_run11",
},
'date': {
'start': "1981-01-01",
'end': "1981-12-01"
}
},
server_config={
'esmdiag': {
'web': {
'url': 'http://192.168.212.201:8088',
'api': {
'task_status': '/api/task/status'
},
'plot_base_dir':
'/home/hujk/clusterfs/wangdp/ploto/plot_base',
},
},
},
)
print(json.dumps(task, indent=2))
from ploto.run import run_ploto
run_ploto(message={'data': task},
config={
'base': {
'run_base_dir':
'/home/hujk/clusterfs/wangdp/ploto/run_base',
'python_exe':
'/home/hujk/.pyenv/versions/ploto-env/bin/python3'
},
'edp_fetcher': {
'edp_module_path': "/home/hujk/pyProject/"
},
'esmdiag': {
'root': '/home/hujk/ploto/ploto/vendor/esmdiag'
}
})
def run_dry_days():
steps = []
steps.extend(get_fetcher_tasks())
steps.extend(get_processor_tasks())
steps.extend(get_combine_metadata_tasks())
steps.append(get_plotter_task())
config = diurnal_config.config
run_ploto({"data": {"steps": steps}}, config)
def run_climwip():
steps = []
steps.extend(get_fetcher_steps())
steps.extend(get_processor_steps())
steps.extend(get_plotter_steps())
config = climwip_config.config
Path(config["base"]["run_base_dir"]).mkdir(parents=True, exist_ok=True)
run_ploto({
"data": {
"steps": steps
}
}, config)
def run_dry_days():
steps = []
exp_datasets = dry_days_recipe.exp_datasets
variables = dry_days_recipe.variables
tasks = [
{
"exp_dataset": d,
"variable": v,
}
for d, v in itertools.product(exp_datasets, variables)
]
steps.extend([get_fetcher(**task) for task in tasks])
datasets = [
{
**d,
"recipe_dataset_index": index,
"alias": d["dataset"]
}
for index, d in enumerate(exp_datasets)
]
for d, v in itertools.product(datasets, variables):
steps.append(get_processor(d, v))
plot_task = generate_default_plot_task()
steps.extend([
{
"step_type": "plotter",
"type": "ploto_esmvaltool.plotter.esmvaltool_diag_plotter",
**plot_task,
"config": dry_days_config.plot_config,
"input_files": [
"{work_dir}" + f"/processor/preproc/{d['dataset']}/{v['variable_group']}/metadata.yml"
for d in datasets
],
}
for v in variables
])
config = dry_days_config.config
run_ploto({
"data": {
"steps": steps
}
}, config)
def run_meteor_draw_task(plot_task: dict):
task = {
'steps': [
{
'step_type': 'plotter',
'type': 'ploto_gidat.plotter.meteor_draw_plotter',
'plot_task': plot_task,
},
],
}
message = {
'app': 'ploto',
'type': 'ploto-gidat',
'timestamp': time.time(),
'data': task
}
config = current_app.config["server_config"]["ploto"]
from ploto.run import run_ploto
run_ploto(message, config)
def run_example():
work_dir = "./dist/cases/case1/run"
esmvaltool_config_file = "./dist/config.yml"
# load recipe.yml
recipe_file_path = "./dist/recipe.yml"
with open(recipe_file_path, "r") as f:
raw_recipe = yaml.safe_load(f)
steps = []
steps.extend(get_processor_tasks(esmvaltool_config_file, raw_recipe))
steps.extend(get_plotter_task(esmvaltool_config_file, raw_recipe))
config = {
"esmvaltool_python_plotter": {},
"esmvaltool": {
"executables": {
"py":
"/home/hujk/.pyenv/versions/anaconda3-2019.10/envs/esmvaltool/bin/python3"
},
"recipes": {
"base":
"/home/hujk/ploto/esmvaltool/study/esmvaltool/ESMValTool/esmvaltool/recipes",
},
"diag_scripts": {
"base":
"/home/hujk/ploto/esmvaltool/study/esmvaltool/ESMValTool/esmvaltool/diag_scripts",
},
},
"base": {
"run_base_dir":
"/home/hujk/ploto/ploto-esmvaltool/dist/cases/case1/run"
}
}
run_ploto({"data": {"steps": steps}}, config)
def test_esmdiag_edp():
"""
test A climo/precip ESMDiag task using data from EDP(earth data platform)
"""
common_config = {
'model_info': {
'id': "FGOALS-g3",
'atm_id': "GAMIL",
'ocn_id': "LICOM",
'ice_id': "CICE",
},
'case_info': {
'id': "gamil_wu_run11",
},
'date': {
'start': "1981-01-01",
'end': "1981-12-01"
}
}
task = {
'steps': [
{
'step_type': 'fetcher',
'common': common_config,
'type': 'ploto_esmdiag.fetcher.edp_fetcher',
'query_param': {
'type': 'nc',
'output_dir': './data',
'file_prefix': 'GAMIL.gamil_wu_run11.step1',
'date_range': ['19810101', '19820101'],
'field_names': [
'PRECT',
'PRECC',
'PRECL',
'PS'
],
'datedif': 'h0'
},
},
{
'step_type': 'fetcher',
'common': common_config,
'type': 'ploto_esmdiag.fetcher.edp_fetcher',
'query_param': {
'type': 'nc',
'output_dir': './data',
'file_prefix': 'GAMIL.gamil_wu_run11.step2',
'date_range': ['19820101', '19820101'],
'field_names': [
'gw'
],
'datedif': 'h0'
}
},
{
'step_type': 'processor',
'type': 'ploto.processor.cdo_processor',
'operator': 'select',
'params': {
'name': 'PS',
'startdate': '1981-01-01',
'enddate': '1981-12-01'
},
'input_files': [
'./data/GAMIL.gamil_wu_run11.step1.*.nc'
],
'output_file': './GAMIL.gamil_wu_run11.PS.monthly.1981-01-01:1981-12-01.nc',
},
{
'step_type': 'processor',
'type': 'ploto.processor.cdo_processor',
'operator': 'select',
'params': {
'name': 'PRECT',
'startdate': '1981-01-01',
'enddate': '1981-12-01'
},
'input_files': [
'./data/GAMIL.gamil_wu_run11.step1.*.nc'
],
'output_file': './GAMIL.gamil_wu_run11.PRECT.monthly.1981-01-01:1981-12-01.nc',
},
{
'step_type': 'processor',
'type': 'ploto.processor.cdo_processor',
'operator': 'select',
'params': {
'name': 'PRECC',
'startdate': '1981-01-01',
'enddate': '1981-12-01'
},
'input_files': [
'./data/GAMIL.gamil_wu_run11.step1.*.nc'
],
'output_file': './GAMIL.gamil_wu_run11.PRECC.monthly.1981-01-01:1981-12-01.nc',
},
{
'step_type': 'processor',
'type': 'ploto.processor.cdo_processor',
'operator': 'select',
'params': {
'name': 'PRECL',
'startdate': '1981-01-01',
'enddate': '1981-12-01'
},
'input_files': [
'./data/GAMIL.gamil_wu_run11.step1.*.nc'
],
'output_file': './GAMIL.gamil_wu_run11.PRECL.monthly.1981-01-01:1981-12-01.nc',
},
{
'step_type': 'processor',
'type': 'ploto.processor.cdo_processor',
'operator': 'select',
'params': {
'name': 'gw',
},
'input_files': [
'./data/GAMIL.gamil_wu_run11.step2.*.nc'
],
'output_file': './FGOALS-g3.gamil_wu_run11.gw.nc',
},
{
'step_type': 'plotter',
'type': 'ploto_esmdiag.plotter.esmdiag_plotter',
'metric': 'climo',
'figure': 'precip',
'common': common_config,
},
],
}
message = {
'app': 'ploto',
'type': 'ploto',
'timestamp': time.time(),
'data': task
}
config = {
'base': {
'run_base_dir': '/home/hujk/clusterfs/wangdp/ploto/run_base',
'python_exe': '/home/hujk/.pyenv/versions/ploto-env/bin/python3'
},
'edp_fetcher': {
'edp_module_path': "/home/hujk/pyProject/"
},
'esmdiag': {
'root': '/home/hujk/ploto/ploto/vendor/esmdiag'
}
}
from ploto.run import run_ploto
run_ploto(message, config)
def receive_giset_plot_example():
"""
Run plot task in server.
POST DATA:
{
"data_source": {
"system_name": "grapes_gfs_gmf",
"username": "******",
"user_id": "u0184",
"data_type": "storage",
"routing_key": "GFSNEW.niuxingy",
"test_ID": "TG2000617"
},
"start_time": "2020070100000",
"step": "6",
"hh_list": [
"00",
"12"
],
"end_time": "2020070200000",
"fcstlen": "96",
"plot_task": {
// ...skip...
}
}
RETURN DATA：
{
"status": "ok",
}
"""
request_data = request.json
data_source = request_data["data_source"]
start_valid_time = request_data["start_time"]
end_valid_time = request_data["end_time"]
forecast_length = int(request_data["fcstlen"])
forecast_step = int(request_data["step"])
start_hours = [int(f) for f in request_data["hh_list"]]
plot_task_template = request_data['plot_task']
start_time = pd.to_datetime(start_valid_time[:10], format="%Y%m%d%H")
end_time = pd.to_datetime(end_valid_time[:10], format="%Y%m%d%H")
tasks = generate_meteor_draw_tasks(
data_source,
start_time,
end_time,
start_hours,
forecast_length,
forecast_step,
plot_task_template,
)
config = current_app.config["server_config"]["ploto"]
for task in tasks:
message = {
'app': 'ploto',
'type': 'ploto-gidat',
'timestamp': time.time(),
'data': task
}
run_ploto(message, config)
return jsonify({
'status': 'ok',
})
def test_anomaly():
print("test begin...")
common_config = {
'model_info': {
'id': "FGOALS-g3",
'atm_id': "GAMIL",
'ocn_id': "LICOM",
'ice_id': "CICE",
},
'case_info': {
'id': "gamil_wu_run11",
},
'date': {
'start': "1982-01-01",
'end': "1991-12-31"
},
}
start_date = datetime.datetime.strptime(common_config['date']['start'],
"%Y-%m-%d")
end_date = datetime.datetime.strptime(common_config['date']['end'],
"%Y-%m-%d")
date_range = [
(start_date - datetime.timedelta(days=30)).strftime("%Y%m%d"),
(end_date + datetime.timedelta(days=30)).strftime("%Y%m%d")
]
steps = []
steps.append({
'step_type': 'fetcher',
'common': common_config,
'type': 'ploto_esmdiag.fetcher.edp_fetcher',
'query_param': {
'type': 'nc',
'output_dir': './data',
'file_prefix': 'step1.',
'date_range': date_range,
'field_names': ['U'],
'datedif': 'h1'
},
})
time_range_string = "{start_date}:{end_date}".format(
start_date=common_config['date']['start'],
end_date=common_config['date']['end'],
)
output_file_pattern = "{file_prefix}.{name}.daily.{time_range}.nc"
steps.append({
'step_type':
'processor',
'type':
'ploto.processor.cdo_processor',
'operator':
'select',
'params': {
'name': 'U',
'startdate': common_config['date']['start'],
'enddate': common_config['date']['end']
},
'input_files':
['./data/{step1_file_prefix}.*.nc'.format(step1_file_prefix='step1.')],
'output_file':
output_file_pattern.format(
file_prefix='step2.',
time_range=time_range_string,
name='U',
),
})
steps.append({
'step_type':
'processor',
'type':
'ploto_esmdiag.processor.esmdiag_data_processor',
'action':
'anomaly',
'input_file':
output_file_pattern.format(file_prefix='step2.',
time_range=time_range_string,
name='U'),
'output_file':
"{file_prefix}.{name}.daily.anomaly.{time_range}.nc".format(
file_prefix='step3.', time_range=time_range_string, name='U'),
})
from ploto.run import run_ploto
run_ploto(message={'data': {
'steps': steps
}},
config={
'base': {
'run_base_dir':
'/home/hujk/clusterfs/wangdp/ploto/run_base',
'python_exe':
'/home/hujk/.pyenv/versions/ploto-env/bin/python3'
},
'edp_fetcher': {
'edp_module_path':
"/home/hujk/clusterfs/wangdp/pyProject"
},
'esmdiag': {
'root': '/home/hujk/ploto/ploto/vendor/esmdiag'
}
})
