def create_firstcyc_task(cdump='gdas'):
'''
This task is needed to run to finalize the first half cycle
'''
task = 'firstcyc'
taskstr = '%s' % task
deps = []
data = '&EXPDIR;/logs/@Y@m@[email protected]'
dep_dict = {'type':'data', 'data':data, 'offset':'24:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type':'cycleexist', 'condition':'not', 'offset':'-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task_dict = {'taskname': '%s' % taskstr, \
'cycledef': 'first', \
'maxtries': '&MAXTRIES;', \
'final' : True, \
'command': 'sleep 1', \
'jobname': '&PSLOT;_%s_@H' % taskstr, \
'account': '&ACCOUNT;', \
'queue': '&QUEUE_ARCH;', \
'walltime': '&WALLTIME_ARCH_%s;' % cdump.upper(), \
'native': '&NATIVE_ARCH_%s;' % cdump.upper(), \
'resources': '&RESOURCES_ARCH_%s;' % cdump.upper(), \
'log': '&ROTDIR;/logs/@Y@m@d@H/%s.log' % taskstr, \
'dependency': dependencies}
task = rocoto.create_task(task_dict)
return ''.join(task)
def create_firstcyc_task(cdump='gdas'):
'''
This task is needed to run to finalize the first half cycle
'''
task = 'firstcyc'
taskstr = f'{task}'
deps = []
data = '&EXPDIR;/logs/@Y@m@[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': '24:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task_dict = {'taskname': f'{taskstr}' , \
'cycledef': 'first', \
'maxtries': '&MAXTRIES;', \
'final' : True, \
'command': 'sleep 1', \
'jobname': f'&PSLOT;_{taskstr}_@H', \
'account': '&ACCOUNT;', \
'queue': '&QUEUE_SERVICE;', \
'walltime': f'&WALLTIME_ARCH_{cdump.upper()};', \
'native': f'&NATIVE_ARCH_{cdump.upper()};', \
'resources': f'&RESOURCES_ARCH_{cdump.upper()};', \
'log': f'&ROTDIR;/logs/@Y@m@d@H/{taskstr}.log', \
'dependency': dependencies}
if get_scheduler(detectMachine()) in ['slurm']:
task_dict['queue'] = '&QUEUE;'
task_dict['partition'] = '&PARTITION_SERVICE;'
task = rocoto.create_task(task_dict)
return ''.join(task)
def get_workflow(dict_configs, cdump='gdas'):
'''
Create tasks for forecast only workflow
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;'))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
base = dict_configs['base']
do_wave = base.get('DO_WAVE', 'NO').upper()
do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
do_wafs = base.get('WAFSF', 'NO').upper()
do_metp = base.get('DO_METP', 'NO').upper()
tasks = []
# getics
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;[email protected]'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;[email protected]'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
deps = rocoto.create_dependency(dep_condition='and', dep=deps)
dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps)
deps = []
deps.append(dependencies)
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('getic',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# chgres fv3ic
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;[email protected]'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;[email protected]'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
deps = rocoto.create_dependency(dep_condition='and', dep=deps)
dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps)
deps = []
deps.append(dependencies)
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('fv3ic',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveinit
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars)
tasks.append(task)
tasks.append('\n')
# waveprep
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('waveprep',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# fcst
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@[email protected]_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
dep_dict = {'type': 'task', 'name': '%swaveprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps)
deps = []
deps.append(dependencies)
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('fcst',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]#dep#.txt' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post',
cdump=cdump,
envar=postenvars,
dependency=dependencies,
metatask='post',
varname=varname1,
varval=varval1,
vardict=vardict)
tasks.append(task)
tasks.append('\n')
# wavepostsbs
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gnh_10m.@Y@m@d.@H0000' % (
cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.aoc_9km.@Y@m@d.@H0000' % (
cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gsh_15m.@Y@m@d.@H0000' % (
cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wavepostsbs',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostbndpnt
if do_wave in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavepostbndpnt',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostbndpntbll
if do_wave in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavepostbndpntbll',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostpnt
if do_wave in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%swavepostbndpntbll' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wavepostpnt',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavestat
#if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
# deps = []
# dep_dict = {'type':'task', 'name':'%swavepost' % cdump}
# deps.append(rocoto.add_dependency(dep_dict))
# dependencies = rocoto.create_dependency(dep=deps)
# task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies)
# tasks.append(task)
# tasks.append('\n')
# wavegempak
if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavegempak',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveawipsbulls
if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('waveawipsbulls',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveawipsgridded
if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('waveawipsgridded',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafs
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafs',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgcip
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgcip',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgrib2
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgrib2',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgrib20p25
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgrib20p25',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsblending
if do_wafs in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swafsgrib2' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wafsblending',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsblending0p25
if do_wafs in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swafsgrib20p25' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wafsblending0p25',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# metp
if do_metp in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'task',
'name': '%sarch' % cdump,
'offset': '-&INTERVAL;'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;')
metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#')
metpenvars = envars + [sdate_gfs] + [metpcase]
varname1 = 'metpcase'
varval1 = 'g2g1 g2o1 pcp1'
task = wfu.create_wf_task('metp',
cdump=cdump,
envar=metpenvars,
dependency=dependencies,
metatask='metp',
varname=varname1,
varval=varval1)
tasks.append(task)
tasks.append('\n')
# arch
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch',
cdump=cdump,
envar=envars,
dependency=dependencies,
final=True)
tasks.append(task)
tasks.append('\n')
return ''.join(tasks)
def get_workflow(dict_configs, cdump='gdas'):
'''
Create tasks for forecast only workflow
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;'))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
tasks = []
# getics
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/pgbanl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/siganl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/sfcanl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
deps = rocoto.create_dependency(dep_condition='and', dep=deps)
dependencies = rocoto.create_dependency(dep_condition='not', dep=deps)
task = wfu.create_wf_task('getic',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# chgres
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/siganl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/sfcanl.&CDUMP;.@Y@m@d@H'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
deps = rocoto.create_dependency(dep_condition='and', dep=deps)
dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps)
deps = []
deps.append(dependencies)
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('fv3ic',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# fcst
deps = []
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('fcst',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post',
cdump=cdump,
envar=postenvars,
dependency=dependencies,
metatask='post',
varname=varname1,
varval=varval1,
vardict=vardict)
tasks.append(task)
tasks.append('\n')
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# arch
deps = []
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch',
cdump=cdump,
envar=envars,
dependency=dependencies,
final=True)
tasks.append(task)
tasks.append('\n')
return ''.join(tasks)
def get_hyb_tasks(dict_configs, cycledef='enkf'):
'''
Create Hybrid tasks
'''
# Determine groups based on ensemble size and grouping
base = dict_configs['base']
nens = base['NMEM_ENKF']
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
eobs = dict_configs['eobs']
nens_eomg = eobs['NMEM_EOMGGRP']
neomg_grps = nens / nens_eomg
EOMGGROUPS = ' '.join(['%02d' % x for x in range(1, neomg_grps + 1)])
efcs = dict_configs['efcs']
nens_efcs = efcs['NMEM_EFCSGRP']
nefcs_grps = nens / nens_efcs
EFCSGROUPS = ' '.join(['%02d' % x for x in range(1, nefcs_grps + 1)])
earc = dict_configs['earc']
nens_earc = earc['NMEM_EARCGRP']
nearc_grps = nens / nens_earc
EARCGROUPS = ' '.join(['%02d' % x for x in range(0, nearc_grps + 1)])
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>'))
#envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#')
dict_tasks = OrderedDict()
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
# eobs
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef)
dict_tasks['%seobs' % cdump] = task
# eomn, eomg
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
deps = []
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
eomgenvars= envars1 + [ensgrp]
task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies,
metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef)
dict_tasks['%seomn' % cdump] = task
# eupd
deps = []
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump}
else:
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef)
dict_tasks['%seupd' % cdump] = task
# All hybrid tasks beyond this point are always executed in the GDAS cycle
cdump = 'gdas'
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas'
# ecen
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%seupd' % cdump_eupd}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('ecen', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef)
dict_tasks['%secen' % cdump] = task
# efmn, efcs
deps = []
dep_dict = {'type': 'task', 'name': '%secen' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
efcsenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies,
metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef)
dict_tasks['%sefmn' % cdump] = task
# epmn, epos
deps = []
dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
eposenvars = envars1 + [fhrgrp] + [fhrlst]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies,
metatask='epmn', varname=varname1, varval=varval1, vardict=vardict)
dict_tasks['%sepmn' % cdump] = task
# eamn, earc
deps = []
dep_dict = {'type': 'metatask', 'name': '%sepmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
earcenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies,
metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef)
dict_tasks['%seamn' % cdump] = task
return dict_tasks
def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS tasks
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
base = dict_configs['base']
gfs_cyc = base.get('gfs_cyc', 0)
dohybvar = base.get('DOHYBVAR', 'NO').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
dumpsuffix = base.get('DUMP_SUFFIX', '')
dict_tasks = OrderedDict()
# prep
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % 'gdas', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/gdas.@Y@m@d/@H/[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
data = '&DMPDIR;/@Y@m@d@H/%s%s/%[email protected]_d' % (cdump, dumpsuffix, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False
if gfs_enkf and cdump in ['gfs']:
if gfs_cyc == 4:
task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies)
else:
task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas')
else:
task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%sprep' % cdump] = task
# anal
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if dohybvar in ['y', 'Y', 'yes', 'YES']:
dep_dict = {'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
else:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%sanal' % cdump] = task
# fcst
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if cdump in ['gdas']:
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
elif cdump in ['gfs']:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%sfcst' % cdump] = task
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies,
metatask='post', varname=varname1, varval=varval1, vardict=vardict)
dict_tasks['%spost' % cdump] = task
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%svrfy' % cdump] = task
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
#postsnd
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%spostsnd' % cdump] = task
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
# awips
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.grib2.idx' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies,
metatask='awips', varname=varname1, varval=varval1, vardict=vardict)
dict_tasks['%sawips' % cdump] = task
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
# gempak
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%sgempak' % cdump] = task
# arch
deps = []
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies)
dict_tasks['%sarch' % cdump] = task
return dict_tasks
def get_hyb_tasks(dict_configs,
EOMGGROUPS,
EFCSGROUPS,
EARCGROUPS,
cdump='gdas'):
'''
Create Hybrid tasks
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#')
base = dict_configs['base']
machine = base['machine']
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
tasks = []
# eobs
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'task',
'name': '%sepos' % cdump,
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('eobs',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# eomn, eomg
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
deps = []
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
eomgenvars = envars + [ensgrp]
task = wfu.create_wf_task('eomg',
cdump=cdump,
envar=eomgenvars,
dependency=dependencies,
metatask='eomn',
varname='grp',
varval=EOMGGROUPS)
tasks.append(task)
tasks.append('\n')
# eupd
deps = []
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump}
else:
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('eupd',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# ecen
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%seupd' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('ecen',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# efmn, efcs
deps = []
dep_dict = {'type': 'task', 'name': '%secen' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
efcsenvars = envars + [ensgrp]
task = wfu.create_wf_task('efcs',
cdump=cdump,
envar=efcsenvars,
dependency=dependencies,
metatask='efmn',
varname='grp',
varval=EFCSGROUPS)
tasks.append(task)
tasks.append('\n')
# epos
deps = []
dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('epos',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# eamn, earc
deps = []
dep_dict = {'type': 'task', 'name': '%sepos' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
earcenvars = envars + [ensgrp]
task = wfu.create_wf_task('earc',
cdump=cdump,
envar=earcenvars,
dependency=dependencies,
metatask='eamn',
varname='grp',
varval=EARCGROUPS)
tasks.append(task)
tasks.append('\n')
return ''.join(tasks)
def get_gdasgfs_tasks(dict_configs, cdump='gdas', dohybvar='NO'):
'''
Create GDAS or GFS tasks
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
tasks = []
# prep
deps = []
dep_dict = {
'type': 'metatask',
'name': '%spost' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/gdas.@Y@m@d/@H/[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
data = '&DMPDIR;/@Y@m@d@H/%s/%[email protected]_d' % (cdump,
cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# anal
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if dohybvar in ['y', 'Y', 'yes', 'YES']:
dep_dict = {
'type': 'task',
'name': '%sepos' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
else:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('anal',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# fcst
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if cdump in ['gdas']:
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
elif cdump in ['gfs']:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('fcst',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
postenvars = envars + [fhrgrp] + [fhrlst]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post',
cdump=cdump,
envar=postenvars,
dependency=dependencies,
metatask='post',
varname=varname1,
varval=varval1,
vardict=vardict)
tasks.append(task)
tasks.append('\n')
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# arch
deps = []
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch',
cdump=cdump,
envar=envars,
dependency=dependencies)
tasks.append(task)
tasks.append('\n')
return ''.join(tasks)
