def get_definitions(base):
'''
Create entities related to the experiment
'''
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
strings = []
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as name, cycle, resolution -->\n')
strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT'])
strings.append('\t<!ENTITY CDUMP "%s">\n' % base['CDUMP'])
strings.append('\t<!ENTITY CASE "%s">\n' % base['CASE'])
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as starting, ending dates -->\n')
strings.append('\t<!ENTITY SDATE "%s">\n' %
base['SDATE'].strftime('%Y%m%d%H%M'))
strings.append('\t<!ENTITY EDATE "%s">\n' %
base['EDATE'].strftime('%Y%m%d%H%M'))
if base['INTERVAL'] is None:
print 'cycle INTERVAL cannot be None'
sys.exit(1)
strings.append('\t<!ENTITY INTERVAL "%s">\n' % base['INTERVAL'])
strings.append('\n')
strings.append('\t<!-- Run Envrionment -->\n')
strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR'])
strings.append('\n')
strings.append('\t<!-- Experiment related directories -->\n')
strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR'])
strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR'])
strings.append('\t<!ENTITY ICSDIR "%s">\n' % base['ICSDIR'])
strings.append('\n')
strings.append('\t<!-- Directories for driving the workflow -->\n')
strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs'])
strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB'])
strings.append('\n')
strings.append('\t<!-- Machine related entities -->\n')
strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT'])
strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE'])
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
if scheduler in ['slurm']:
strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' %
base['QUEUE_ARCH'])
strings.append('\t<!ENTITY SCHEDULER "%s">\n' % scheduler)
strings.append('\n')
strings.append('\t<!-- Toggle HPSS archiving -->\n')
strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
strings.append('\n')
strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
strings.append('\t<!ENTITY CYCLETHROTTLE "2">\n')
strings.append('\t<!ENTITY TASKTHROTTLE "25">\n')
strings.append('\t<!ENTITY MAXTRIES "2">\n')
strings.append('\n')
return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS resource entities
'''
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
compute_partition = base.get('COMPUTE_PARTITION_LINE', None)
service_partition = base.get('SERVICE_PARTITION_LINE', None)
tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch']
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
tasks += ['postsnd']
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
tasks += ['gempak']
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
tasks += ['awips']
dict_resources = OrderedDict()
for task in tasks:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
if task in ['arch', 'post']:
partition = service_partition
else:
partition = compute_partition
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
reservation = base.get('RESERVATION', 'NONE').upper()
scheduler = wfu.get_scheduler(machine)
do_wave = base.get('DO_WAVE', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
do_metp = base.get('DO_METP', 'NO').upper()
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, reservation, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and machine in ['ORION'] and task not in [
'getic', 'arch'
]:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_BATCH;">\n' %
taskstr)
if scheduler in ['slurm'] and task in ['getic', 'arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_SERVICE;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['getic', 'arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
parser.add_argument('--resens', help='resolution of the ensemble model forecast', type=int, required=False, default=192)
parser.add_argument('--comrot', help='full path to COMROT', type=str, required=False, default=None)
parser.add_argument('--expdir', help='full path to EXPDIR', type=str, required=False, default=None)
parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', type=str, required=True)
parser.add_argument('--edate', help='end date experiment', type=str, required=True)
parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False)
parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None)
parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20)
parser.add_argument('--cdump', help='CDUMP to start the experiment', type=str, required=False, default='gdas')
parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False)
parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None)
parser.add_argument('--start', help='restart mode: warm or cold', type=str, choices=['warm', 'cold'], required=False, default='cold')
args = parser.parse_args()
machine = wfu.detectMachine()
configdir = args.configdir
if not configdir:
configdir = os.path.abspath(os.path.dirname(__file__) + '/../parm/config')
pslot = args.pslot
idate = datetime.strptime(args.idate, '%Y%m%d%H')
edate = datetime.strptime(args.edate, '%Y%m%d%H')
icsdir = args.icsdir
resdet = args.resdet
resens = args.resens
comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot)
expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot)
nens = args.nens
cdump = args.cdump
def get_workflow(dict_configs, cdump='gdas'):
'''
Create tasks for forecast only workflow
'''
envars = []
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;'))
envars.append(rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
hpssarch = base.get('HPSSARCH', 'NO').upper()
do_wave = base.get('DO_WAVE', 'NO').upper()
do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
do_wafs = base.get('WAFSF', 'NO').upper()
do_metp = base.get('DO_METP', 'NO').upper()
tasks = []
# getic
if hpssarch in ['YES']:
deps = []
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# init
deps = []
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/[email protected]'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
if hpssarch in ['YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%sgetic' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies2 = rocoto.create_dependency(dep=deps)
deps = []
deps.append(dependencies)
if hpssarch in ['YES']:
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveinit
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars)
tasks.append(task)
tasks.append('\n')
# waveprep
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%sinit' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# fcst
deps = []
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc'
dep_dict = {'type':'data', 'data':data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
dep_dict = {'type': 'task', 'name': '%swaveprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps)
deps = []
deps.append(dependencies)
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps.append(dependencies2)
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]#dep#.txt' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies,
metatask='post', varname=varname1, varval=varval1, vardict=vardict)
tasks.append(task)
tasks.append('\n')
# wavepostsbs
if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gnh_10m.@Y@m@d.@H0000' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.aoc_9km.@Y@m@d.@H0000' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gsh_15m.@Y@m@d.@H0000' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostbndpnt
if do_wave in ['Y', 'YES']:
deps = []
dep_dict = {'type':'task', 'name':'%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostbndpntbll
if do_wave in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavepostpnt
if do_wave in ['Y', 'YES']:
deps = []
dep_dict = {'type':'task', 'name':'%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type':'task', 'name':'%swavepostbndpntbll' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wavegempak
if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']:
deps = []
dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveawipsbulls
if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']:
deps = []
dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type':'task', 'name':'%swavepostpnt' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# waveawipsgridded
if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']:
deps = []
dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafs
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgcip
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgrib2
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsgrib20p25
if do_wafs in ['Y', 'YES']:
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsblending
if do_wafs in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swafsgrib2' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# wafsblending0p25
if do_wafs in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%swafsgrib20p25' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
#postsnd
if do_bufrsnd in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# awips
if do_awips in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies,
metatask='awips', varname=varname1, varval=varval1, vardict=vardict)
tasks.append(task)
tasks.append('\n')
# gempak
if do_gempak in ['Y', 'YES']:
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# vrfy
deps = []
dep_dict = {'type':'metatask', 'name':'%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies)
tasks.append(task)
tasks.append('\n')
# metp
if do_metp in ['Y', 'YES']:
deps = []
dep_dict = {'type':'metatask', 'name':'%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type':'task', 'name':'%sarch' % cdump, 'offset':'-&INTERVAL;'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;')
metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#')
metpenvars = envars + [sdate_gfs] + [metpcase]
varname1 = 'metpcase'
varval1 = 'g2g1 g2o1 pcp1'
task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies,
metatask='metp', varname=varname1, varval=varval1)
tasks.append(task)
tasks.append('\n')
# arch
deps = []
dep_dict = {'type':'metatask', 'name':'%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type':'task', 'name':'%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if do_wave in ['Y', 'YES']:
dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True)
tasks.append(task)
tasks.append('\n')
return ''.join(tasks)
def get_hyb_tasks(dict_configs, cycledef='enkf'):
'''
Create Hybrid tasks
'''
# Determine groups based on ensemble size and grouping
base = dict_configs['base']
nens = base['NMEM_ENKF']
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
eobs = dict_configs['eobs']
nens_eomg = eobs['NMEM_EOMGGRP']
neomg_grps = nens / nens_eomg
EOMGGROUPS = ' '.join(['%02d' % x for x in range(1, neomg_grps + 1)])
efcs = dict_configs['efcs']
nens_efcs = efcs['NMEM_EFCSGRP']
nefcs_grps = nens / nens_efcs
EFCSGROUPS = ' '.join(['%02d' % x for x in range(1, nefcs_grps + 1)])
earc = dict_configs['earc']
nens_earc = earc['NMEM_EARCGRP']
nearc_grps = nens / nens_earc
EARCGROUPS = ' '.join(['%02d' % x for x in range(0, nearc_grps + 1)])
envars = []
if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']:
envars.append(rocoto.create_envar(name='SLURM_SET', value='YES'))
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
#envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#')
dict_tasks = OrderedDict()
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
# eobs
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'metatask',
'name': '%sepmn' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('eobs',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%seobs' % cdump] = task
# eomn, eomg
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
deps = []
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
eomgenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('eomg',
cdump=cdump,
envar=eomgenvars,
dependency=dependencies,
metatask='eomn',
varname='grp',
varval=EOMGGROUPS,
cycledef=cycledef)
dict_tasks['%seomn' % cdump] = task
# eupd
deps = []
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump}
else:
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('eupd',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%seupd' % cdump] = task
# All hybrid tasks beyond this point are always executed in the GDAS cycle
cdump = 'gdas'
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas'
# ecen
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%seupd' % cdump_eupd}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('ecen',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%secen' % cdump] = task
# efmn, efcs
deps = []
dep_dict = {'type': 'task', 'name': '%secen' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
efcsenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('efcs',
cdump=cdump,
envar=efcsenvars,
dependency=dependencies,
metatask='efmn',
varname='grp',
varval=EFCSGROUPS,
cycledef=cycledef)
dict_tasks['%sefmn' % cdump] = task
# epmn, epos
deps = []
dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
eposenvars = envars1 + [fhrgrp] + [fhrlst]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('epos',
cdump=cdump,
envar=eposenvars,
dependency=dependencies,
metatask='epmn',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%sepmn' % cdump] = task
# eamn, earc
deps = []
dep_dict = {'type': 'metatask', 'name': '%sepmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
earcenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('earc',
cdump=cdump,
envar=earcenvars,
dependency=dependencies,
metatask='eamn',
varname='grp',
varval=EARCGROUPS,
cycledef=cycledef)
dict_tasks['%seamn' % cdump] = task
return dict_tasks
def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS tasks
'''
envars = []
if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']:
envars.append(rocoto.create_envar(name='SLURM_SET', value='YES'))
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
base = dict_configs['base']
gfs_cyc = base.get('gfs_cyc', 0)
dohybvar = base.get('DOHYBVAR', 'NO').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
dumpsuffix = base.get('DUMP_SUFFIX', '')
dict_tasks = OrderedDict()
# prep
deps = []
dep_dict = {
'type': 'metatask',
'name': '%spost' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/gdas.@Y@m@d/@H/[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
data = '&DMPDIR;/%s%s.@Y@m@d/@H/%[email protected]_d' % (
cdump, dumpsuffix, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'
] and dohybvar in ['Y', 'YES'] else False
if gfs_enkf and cdump in ['gfs']:
if gfs_cyc == 4:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies)
else:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies,
cycledef='gdas')
else:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sprep' % cdump] = task
# anal
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if dohybvar in ['y', 'Y', 'yes', 'YES']:
dep_dict = {
'type': 'metatask',
'name': '%sepmn' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
else:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('anal',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sanal' % cdump] = task
# fcst
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if cdump in ['gdas']:
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
elif cdump in ['gfs']:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('fcst',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sfcst' % cdump] = task
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post',
cdump=cdump,
envar=postenvars,
dependency=dependencies,
metatask='post',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%spost' % cdump] = task
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%svrfy' % cdump] = task
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
#postsnd
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('postsnd',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%spostsnd' % cdump] = task
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
# awips
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.grib2.idx' % (
cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('awips',
cdump=cdump,
envar=awipsenvars,
dependency=dependencies,
metatask='awips',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%sawips' % cdump] = task
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
# gempak
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('gempak',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sgempak' % cdump] = task
# arch
deps = []
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sarch' % cdump] = task
return dict_tasks
def get_hyb_resources(dict_configs):
'''
Create hybrid resource entities
'''
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
compute_partition = base.get('COMPUTE_PARTITION_LINE', None)
dict_resources = OrderedDict()
# These tasks can be run in either or both cycles
tasks1 = ['eobs', 'eomg', 'eupd']
if lobsdiag_forenkf in ['.T.', '.TRUE.']:
tasks.remove('eomg')
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
for task in tasks1:
cfg = dict_configs['eobs'] if task in ['eomg'
] else dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' %
(taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' %
(taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' %
(taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' %
(taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
# These tasks are always run as part of the GDAS cycle
cdump = 'gdas'
tasks2 = ['ecen', 'efcs', 'epos', 'earc']
for task in tasks2:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['earc']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
def get_definitions(base):
'''
Create entities related to the experiment
'''
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
strings = []
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as name, starting, ending dates -->\n'
)
strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT'])
strings.append('\t<!ENTITY SDATE "%s">\n' %
base['SDATE'].strftime('%Y%m%d%H%M'))
strings.append('\t<!ENTITY EDATE "%s">\n' %
base['EDATE'].strftime('%Y%m%d%H%M'))
if base['gfs_cyc'] != 0:
strings.append(get_gfs_dates(base))
strings.append('\n')
strings.append('\t<!-- Run Envrionment -->\n')
strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR'])
strings.append('\n')
strings.append('\t<!-- Experiment and Rotation directory -->\n')
strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR'])
strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR'])
strings.append('\n')
strings.append('\t<!-- Directories for driving the workflow -->\n')
strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs'])
strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB'])
strings.append('\t<!ENTITY DMPDIR "%s">\n' % base['DMPDIR'])
strings.append('\n')
strings.append('\t<!-- Machine related entities -->\n')
strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT'])
strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE'])
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
if scheduler in ['slurm']:
strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' %
base['QUEUE_ARCH'])
else:
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
strings.append('\t<!ENTITY SCHEDULER "%s">\n' %
wfu.get_scheduler(base['machine']))
if 'COMPUTE_PARTITION' in base:
strings.append('\t<!ENTITY COMPUTE_PARTITION "%s">\n' %
base.get('COMPUTE_PARTITION', None))
if 'SERVICE_PARTITION' in base:
strings.append('\t<!ENTITY SERVICE_PARTITION "%s">\n' %
base.get('SERVICE_PARTITION', None))
strings.append('\n')
strings.append('\t<!-- Toggle HPSS archiving -->\n')
strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
strings.append('\n')
strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
strings.append('\t<!ENTITY CYCLETHROTTLE "3">\n')
strings.append('\t<!ENTITY TASKTHROTTLE "25">\n')
strings.append('\t<!ENTITY MAXTRIES "2">\n')
strings.append('\n')
return ''.join(strings)
