def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
machine = dict_configs['base']['machine']
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS resource entities
'''
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
scheduler = wfu.get_scheduler(machine)
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
compute_partition = base.get('COMPUTE_PARTITION_LINE', None)
service_partition = base.get('SERVICE_PARTITION_LINE', None)
tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch']
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
tasks += ['postsnd']
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
tasks += ['gempak']
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
tasks += ['awips']
dict_resources = OrderedDict()
for task in tasks:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
if task in ['arch', 'post']:
partition = service_partition
else:
partition = compute_partition
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and task in ['arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', wfu.detectMachine())
reservation = base.get('RESERVATION', 'NONE').upper()
scheduler = wfu.get_scheduler(machine)
do_wave = base.get('DO_WAVE', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
do_metp = base.get('DO_METP', 'NO').upper()
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, reservation, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if scheduler in ['slurm'] and machine in ['ORION'] and task not in [
'getic', 'arch'
]:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_BATCH;">\n' %
taskstr)
if scheduler in ['slurm'] and task in ['getic', 'arch']:
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_SERVICE;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS resource entities
'''
base = dict_configs['base']
machine = base.get('machine', 'WCOSS_C')
do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
do_gempak = base.get('DO_GEMPAK', 'NO').upper()
do_awips = base.get('DO_AWIPS', 'NO').upper()
tasks = ['prep', 'anal', 'gldas', 'fcst', 'post', 'vrfy', 'arch']
if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']:
tasks += ['postsnd']
if cdump in ['gfs'] and do_gempak in ['Y', 'YES']:
tasks += ['gempak']
if cdump in ['gfs'] and do_awips in ['Y', 'YES']:
tasks += ['awips']
dict_resources = OrderedDict()
for task in tasks:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if base['machine'] == 'THEIA' and wfu.check_slurm() and task == 'arch':
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources
def get_resources(dict_configs, cdump='gdas'):
'''
Create resource entities
'''
strings = []
strings.append(
'\t<!-- BEGIN: Resource requirements for the workflow -->\n')
strings.append('\n')
base = dict_configs['base']
machine = base.get('machine', 'WCOSS_C')
for task in taskplan:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(
machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
if base['machine'] == 'THEIA' and wfu.check_slurm() and task == 'arch':
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
elif base['machine'] == 'THEIA' and wfu.check_slurm(
) and task == 'getic':
strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' %
taskstr)
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
if len(memstr) != 0:
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' %
(taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
strings.append('\n')
strings.append('\t<!-- END: Resource requirements for the workflow -->\n')
return ''.join(strings)
def get_hyb_resources(dict_configs, cdump='gdas'):
'''
Create hybrid resource entities
'''
strings = []
strings.append('\n')
strings.append(
'\t<!-- BEGIN: Resource requirements for hybrid part of the workflow -->\n'
)
strings.append('\n')
base = dict_configs['base']
machine = base['machine']
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
tasks = ['eobs', 'eomg', 'eupd', 'ecen', 'efcs', 'epos', 'earc']
if lobsdiag_forenkf in ['.T.', '.TRUE.']: tasks.remove('eomg')
for task in tasks:
cfg = dict_configs['eobs'] if task in ['eomg'] else dict_configs[task]
wtimestr, resstr, queuestr, memstr = wfu.get_resources(machine,
cfg,
task,
cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "">\n' % (taskstr))
strings.append('\n')
strings.append(
'\t<!-- END: Resource requirements for hybrid part of the workflow-->\n'
)
return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'):
'''
Create GDAS or GFS resource entities
'''
strings = []
strings.append('\n')
strings.append(
'\t<!-- BEGIN: Resource requirements for %s part of the workflow -->\n'
% cdump.upper())
strings.append('\n')
machine = dict_configs['base']['machine']
tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch']
for task in tasks:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr = wfu.get_resources(machine,
cfg,
task,
cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "">\n' % (taskstr))
strings.append('\n')
strings.append(
'\t<!-- END: Resource requirements for %s part of the workflow -->\n' %
cdump.upper())
return ''.join(strings)
def get_hyb_resources(dict_configs):
'''
Create hybrid resource entities
'''
base = dict_configs['base']
machine = base.get('machine', 'WCOSS_C')
lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper()
eupd_cyc= base.get('EUPD_CYC', 'gdas').upper()
dict_resources = OrderedDict()
# These tasks can be run in either or both cycles
tasks1 = ['eobs', 'eomg', 'eupd']
if lobsdiag_forenkf in ['.T.', '.TRUE.']:
tasks.remove('eomg')
if eupd_cyc in ['BOTH']:
cdumps = ['gfs', 'gdas']
elif eupd_cyc in ['GFS']:
cdumps = ['gfs']
elif eupd_cyc in ['GDAS']:
cdumps = ['gdas']
for cdump in cdumps:
for task in tasks1:
cfg = dict_configs['eobs'] if task in ['eomg'] else dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
# These tasks are always run as part of the GDAS cycle
cdump = 'gdas'
tasks2 = ['ecen', 'efcs', 'epos', 'earc']
for task in tasks2:
cfg = dict_configs[task]
wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump)
taskstr = '%s_%s' % (task.upper(), cdump.upper())
strings = []
strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr))
strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr))
strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr))
strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr))
strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr))
dict_resources['%s%s' % (cdump, task)] = ''.join(strings)
return dict_resources