def get_definitions(base):
'''
Create entities related to the experiment
'''
strings = []
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as name, cycle, resolution -->\n')
strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT'])
strings.append('\t<!ENTITY CDUMP "%s">\n' % base['CDUMP'])
strings.append('\t<!ENTITY CASE "%s">\n' % base['CASE'])
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as starting, ending dates -->\n')
strings.append('\t<!ENTITY SDATE "%s">\n' %
base['SDATE'].strftime('%Y%m%d%H%M'))
strings.append('\t<!ENTITY EDATE "%s">\n' %
base['EDATE'].strftime('%Y%m%d%H%M'))
if base['INTERVAL'] is None:
print 'cycle INTERVAL cannot be None'
sys.exit(1)
strings.append('\t<!ENTITY INTERVAL "%s">\n' % base['INTERVAL'])
strings.append('\n')
strings.append('\t<!-- Run Envrionment -->\n')
strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR'])
strings.append('\n')
strings.append('\t<!-- Experiment related directories -->\n')
strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR'])
strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR'])
strings.append('\t<!ENTITY ICSDIR "%s">\n' % base['ICSDIR'])
strings.append('\n')
strings.append('\t<!-- Directories for driving the workflow -->\n')
strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs'])
strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB'])
strings.append('\n')
strings.append('\t<!-- Machine related entities -->\n')
strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT'])
strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE'])
if base['machine'] == 'THEIA' and wfu.check_slurm():
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE'])
strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' %
base['QUEUE_ARCH'])
else:
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
strings.append('\t<!ENTITY SCHEDULER "%s">\n' %
wfu.get_scheduler(base['machine']))
strings.append('\n')
strings.append('\t<!-- Toggle HPSS archiving -->\n')
strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
strings.append('\n')
strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
strings.append('\t<!ENTITY CYCLETHROTTLE "2">\n')
strings.append('\t<!ENTITY TASKTHROTTLE "20">\n')
strings.append('\t<!ENTITY MAXTRIES "2">\n')
strings.append('\n')
return ''.join(strings)
def get_definitions(base):
'''
Create entities related to the experiment
'''
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
hpssarch = base.get('HPSSARCH', 'NO').upper()
strings = []
strings.append('\n')
strings.append('\t<!-- Experiment parameters such as name, cycle, resolution -->\n')
strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT'])
strings.append('\t<!ENTITY CDUMP "%s">\n' % base['CDUMP'])
strings.append('\t<!ENTITY CASE "%s">\n' % base['CASE'])
strings.append('\n')
strings.append('\t<!-- Experiment parameters such as starting, ending dates -->\n')
strings.append('\t<!ENTITY SDATE "%s">\n' % base['SDATE'].strftime('%Y%m%d%H%M'))
strings.append('\t<!ENTITY EDATE "%s">\n' % base['EDATE'].strftime('%Y%m%d%H%M'))
if base['INTERVAL'] is None:
print 'cycle INTERVAL cannot be None'
sys.exit(1)
strings.append('\t<!ENTITY INTERVAL "%s">\n' % base['INTERVAL'])
strings.append('\n')
strings.append('\t<!-- Run Envrionment -->\n')
strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR'])
strings.append('\n')
strings.append('\t<!-- Experiment related directories -->\n')
strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR'])
strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR'])
strings.append('\t<!ENTITY ICSDIR "%s">\n' % base['ICSDIR'])
strings.append('\n')
strings.append('\t<!-- Directories for driving the workflow -->\n')
strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs'])
strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB'])
strings.append('\n')
strings.append('\t<!-- Machine related entities -->\n')
strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT'])
strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE'])
strings.append('\t<!ENTITY QUEUE_SERVICE "%s">\n' % base['QUEUE_SERVICE'])
if scheduler in ['slurm'] and machine in ['ORION']:
strings.append('\t<!ENTITY PARTITION_BATCH "%s">\n' % base['PARTITION_BATCH'])
if scheduler in ['slurm']:
strings.append('\t<!ENTITY PARTITION_SERVICE "%s">\n' % base['QUEUE_SERVICE'])
strings.append('\t<!ENTITY SCHEDULER "%s">\n' % scheduler)
strings.append('\n')
strings.append('\t<!-- Toggle HPSS archiving -->\n')
strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "%s">\n' % base['HPSSARCH'])
strings.append('\n')
strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
strings.append('\t<!ENTITY CYCLETHROTTLE "2">\n')
strings.append('\t<!ENTITY TASKTHROTTLE "25">\n')
strings.append('\t<!ENTITY MAXTRIES "2">\n')
strings.append('\n')
return ''.join(strings)
def get_definitions(base):
'''
Create entities related to the experiment
'''
strings = []
strings.append('\n')
strings.append(
'\t<!-- Experiment parameters such as name, starting, ending dates -->\n'
)
strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT'])
strings.append('\t<!ENTITY SDATE "%s">\n' %
base['SDATE'].strftime('%Y%m%d%H%M'))
strings.append('\t<!ENTITY EDATE "%s">\n' %
base['EDATE'].strftime('%Y%m%d%H%M'))
if base['gfs_cyc'] != 0:
strings.append(get_gfs_dates(base))
strings.append('\n')
strings.append('\t<!-- Run Envrionment -->\n')
strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR'])
strings.append('\n')
strings.append('\t<!-- Experiment and Rotation directory -->\n')
strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR'])
strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR'])
strings.append('\n')
strings.append('\t<!-- Directories for driving the workflow -->\n')
strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs'])
strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB'])
strings.append('\t<!ENTITY DMPDIR "%s">\n' % base['DMPDIR'])
strings.append('\n')
strings.append('\t<!-- Machine related entities -->\n')
strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT'])
strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE'])
if base['machine'] == 'THEIA' and wfu.check_slurm():
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' %
base['QUEUE_ARCH'])
else:
strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH'])
strings.append('\t<!ENTITY SCHEDULER "%s">\n' %
wfu.get_scheduler(base['machine']))
strings.append('\n')
strings.append('\t<!-- Toggle HPSS archiving -->\n')
strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
strings.append('\n')
strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
strings.append('\t<!ENTITY CYCLETHROTTLE "3">\n')
strings.append('\t<!ENTITY TASKTHROTTLE "20">\n')
strings.append('\t<!ENTITY MAXTRIES "2">\n')
strings.append('\n')
return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS resource entities
'''
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
compute_partition = base.get('COMPUTE_PARTITION_LINE', None)
service_partition = base.get('SERVICE_PARTITION_LINE', None)
tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch']
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
tasks += ['postsnd']
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
tasks += ['gempak']
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
tasks += ['awips']
dict_resources = OrderedDict()
for task in tasks:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
if task in ['arch', 'post']:
partition = service_partition
else:
partition = compute_partition
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
reservation = base.get('RESERVATION', 'NONE').upper()
scheduler = wfu.get_scheduler(machine)
do_wave = base.get('DO_WAVE', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
do_metp = base.get('DO_METP', 'NO').upper()
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, reservation, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and machine in ['ORION'] and task not in [
'getic', 'arch'
]:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_BATCH;">\n' %
taskstr)
if scheduler in ['slurm'] and task in ['getic', 'arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_SERVICE;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['getic', 'arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_hyb_tasks(dict_configs, cycledef='enkf'):
'''
Create Hybrid tasks
'''
# Determine groups based on ensemble size and grouping
base = dict_configs['base']
nens = base['NMEM_ENKF']
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
eobs = dict_configs['eobs']
nens_eomg = eobs['NMEM_EOMGGRP']
neomg_grps = nens / nens_eomg
EOMGGROUPS = ' '.join(['%02d' % x for x in range(1, neomg_grps + 1)])
efcs = dict_configs['efcs']
nens_efcs = efcs['NMEM_EFCSGRP']
nefcs_grps = nens / nens_efcs
EFCSGROUPS = ' '.join(['%02d' % x for x in range(1, nefcs_grps + 1)])
earc = dict_configs['earc']
nens_earc = earc['NMEM_EARCGRP']
nearc_grps = nens / nens_earc
EARCGROUPS = ' '.join(['%02d' % x for x in range(0, nearc_grps + 1)])
envars = []
if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']:
envars.append(rocoto.create_envar(name='SLURM_SET', value='YES'))
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
#envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#')
dict_tasks = OrderedDict()
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
# eobs
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'metatask',
'name': '%sepmn' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('eobs',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%seobs' % cdump] = task
# eomn, eomg
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
deps = []
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
eomgenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('eomg',
cdump=cdump,
envar=eomgenvars,
dependency=dependencies,
metatask='eomn',
varname='grp',
varval=EOMGGROUPS,
cycledef=cycledef)
dict_tasks['%seomn' % cdump] = task
# eupd
deps = []
if lobsdiag_forenkf in ['.F.', '.FALSE.']:
dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump}
else:
dep_dict = {'type': 'task', 'name': '%seobs' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('eupd',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%seupd' % cdump] = task
# All hybrid tasks beyond this point are always executed in the GDAS cycle
cdump = 'gdas'
envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump)
envars1 = envars + [envar_cdump]
cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas'
# ecen
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%seupd' % cdump_eupd}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('ecen',
cdump=cdump,
envar=envars1,
dependency=dependencies,
cycledef=cycledef)
dict_tasks['%secen' % cdump] = task
# efmn, efcs
deps = []
dep_dict = {'type': 'task', 'name': '%secen' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
efcsenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('efcs',
cdump=cdump,
envar=efcsenvars,
dependency=dependencies,
metatask='efmn',
varname='grp',
varval=EFCSGROUPS,
cycledef=cycledef)
dict_tasks['%sefmn' % cdump] = task
# epmn, epos
deps = []
dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
eposenvars = envars1 + [fhrgrp] + [fhrlst]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('epos',
cdump=cdump,
envar=eposenvars,
dependency=dependencies,
metatask='epmn',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%sepmn' % cdump] = task
# eamn, earc
deps = []
dep_dict = {'type': 'metatask', 'name': '%sepmn' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
earcenvars = envars1 + [ensgrp]
task = wfu.create_wf_task('earc',
cdump=cdump,
envar=earcenvars,
dependency=dependencies,
metatask='eamn',
varname='grp',
varval=EARCGROUPS,
cycledef=cycledef)
dict_tasks['%seamn' % cdump] = task
return dict_tasks
def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS tasks
'''
envars = []
if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']:
envars.append(rocoto.create_envar(name='SLURM_SET', value='YES'))
envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;'))
envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;'))
envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;'))
envars.append(
rocoto.create_envar(name='CDATE',
value='<cyclestr>@Y@m@d@H</cyclestr>'))
envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump))
envars.append(
rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>'))
envars.append(
rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
base = dict_configs['base']
gfs_cyc = base.get('gfs_cyc', 0)
dohybvar = base.get('DOHYBVAR', 'NO').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
dumpsuffix = base.get('DUMP_SUFFIX', '')
dict_tasks = OrderedDict()
# prep
deps = []
dep_dict = {
'type': 'metatask',
'name': '%spost' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
data = '&ROTDIR;/gdas.@Y@m@d/@H/[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
data = '&DMPDIR;/%s%s.@Y@m@d/@H/%[email protected]_d' % (
cdump, dumpsuffix, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'
] and dohybvar in ['Y', 'YES'] else False
if gfs_enkf and cdump in ['gfs']:
if gfs_cyc == 4:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies)
else:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies,
cycledef='gdas')
else:
task = wfu.create_wf_task('prep',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sprep' % cdump] = task
# anal
deps = []
dep_dict = {'type': 'task', 'name': '%sprep' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if dohybvar in ['y', 'Y', 'yes', 'YES']:
dep_dict = {
'type': 'metatask',
'name': '%sepmn' % 'gdas',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
else:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('anal',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sanal' % cdump] = task
# fcst
deps = []
dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
if cdump in ['gdas']:
dep_dict = {
'type': 'cycleexist',
'condition': 'not',
'offset': '-06:00:00'
}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
elif cdump in ['gfs']:
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('fcst',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sfcst' % cdump] = task
# post
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_postgroups(dict_configs['post'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('post',
cdump=cdump,
envar=postenvars,
dependency=dependencies,
metatask='post',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%spost' % cdump] = task
# vrfy
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('vrfy',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%svrfy' % cdump] = task
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
#postsnd
deps = []
dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('postsnd',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%spostsnd' % cdump] = task
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
# awips
deps = []
data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.grib2.idx' % (
cdump, cdump)
dep_dict = {'type': 'data', 'data': data}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
varname1, varname2, varname3 = 'grp', 'dep', 'lst'
varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'],
cdump=cdump)
vardict = {varname2: varval2, varname3: varval3}
task = wfu.create_wf_task('awips',
cdump=cdump,
envar=awipsenvars,
dependency=dependencies,
metatask='awips',
varname=varname1,
varval=varval1,
vardict=vardict)
dict_tasks['%sawips' % cdump] = task
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
# gempak
deps = []
dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)
task = wfu.create_wf_task('gempak',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sgempak' % cdump] = task
# arch
deps = []
dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
task = wfu.create_wf_task('arch',
cdump=cdump,
envar=envars,
dependency=dependencies)
dict_tasks['%sarch' % cdump] = task
return dict_tasks
def get_hyb_resources(dict_configs):
'''
Create hybrid resource entities
'''
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc = base.get('EUPD_CYC', 'gdas').upper()
compute_partition = base.get('COMPUTE_PARTITION_LINE', None)
dict_resources = OrderedDict()
# These tasks can be run in either or both cycles
tasks1 = ['eobs', 'eomg', 'eupd']
if lobsdiag_forenkf in ['.T.', '.TRUE.']:
tasks.remove('eomg')
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
for task in tasks1:
cfg = dict_configs['eobs'] if task in ['eomg'
] else dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' %
(taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' %
(taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' %
(taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' %
(taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
# These tasks are always run as part of the GDAS cycle
cdump = 'gdas'
tasks2 = ['ecen', 'efcs', 'epos', 'earc']
for task in tasks2:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['earc']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
