def get_definitions(base): ''' Create entities related to the experiment ''' machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) strings = [] strings.append('\n') strings.append( '\t<!-- Experiment parameters such as name, cycle, resolution -->\n') strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT']) strings.append('\t<!ENTITY CDUMP "%s">\n' % base['CDUMP']) strings.append('\t<!ENTITY CASE "%s">\n' % base['CASE']) strings.append('\n') strings.append( '\t<!-- Experiment parameters such as starting, ending dates -->\n') strings.append('\t<!ENTITY SDATE "%s">\n' % base['SDATE'].strftime('%Y%m%d%H%M')) strings.append('\t<!ENTITY EDATE "%s">\n' % base['EDATE'].strftime('%Y%m%d%H%M')) if base['INTERVAL'] is None: print 'cycle INTERVAL cannot be None' sys.exit(1) strings.append('\t<!ENTITY INTERVAL "%s">\n' % base['INTERVAL']) strings.append('\n') strings.append('\t<!-- Run Envrionment -->\n') strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR']) strings.append('\n') strings.append('\t<!-- Experiment related directories -->\n') strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR']) strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR']) strings.append('\t<!ENTITY ICSDIR "%s">\n' % base['ICSDIR']) strings.append('\n') strings.append('\t<!-- Directories for driving the workflow -->\n') strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs']) strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB']) strings.append('\n') strings.append('\t<!-- Machine related entities -->\n') strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT']) strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE']) strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH']) if scheduler in ['slurm']: strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' % base['QUEUE_ARCH']) strings.append('\t<!ENTITY SCHEDULER "%s">\n' % scheduler) strings.append('\n') strings.append('\t<!-- Toggle HPSS archiving -->\n') strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n') strings.append('\n') strings.append('\t<!-- ROCOTO parameters that control workflow -->\n') strings.append('\t<!ENTITY CYCLETHROTTLE "2">\n') strings.append('\t<!ENTITY TASKTHROTTLE "25">\n') strings.append('\t<!ENTITY MAXTRIES "2">\n') strings.append('\n') return ''.join(strings)
def get_gdasgfs_resources(dict_configs, cdump='gdas'): ''' Create GDAS or GFS resource entities ''' base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() compute_partition = base.get('COMPUTE_PARTITION_LINE', None) service_partition = base.get('SERVICE_PARTITION_LINE', None) tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: tasks += ['postsnd'] if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: tasks += ['gempak'] if cdump in ['gfs'] and do_awips in ['Y', 'YES']: tasks += ['awips'] dict_resources = OrderedDict() for task in tasks: cfg = dict_configs[task] wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources( machine, cfg, task, cdump=cdump) taskstr = '%s_%s' % (task.upper(), cdump.upper()) if task in ['arch', 'post']: partition = service_partition else: partition = compute_partition strings = [] strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr)) if scheduler in ['slurm'] and task in ['arch']: strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' % taskstr) strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr)) strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr)) if len(memstr) != 0: strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr)) strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr)) dict_resources['%s%s' % (cdump, task)] = ''.join(strings) return dict_resources
def get_resources(dict_configs, cdump='gdas'): ''' Create resource entities ''' strings = [] strings.append( '\t<!-- BEGIN: Resource requirements for the workflow -->\n') strings.append('\n') base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) reservation = base.get('RESERVATION', 'NONE').upper() scheduler = wfu.get_scheduler(machine) do_wave = base.get('DO_WAVE', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() for task in taskplan: cfg = dict_configs[task] wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources( machine, cfg, task, reservation, cdump=cdump) taskstr = '%s_%s' % (task.upper(), cdump.upper()) strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr)) if scheduler in ['slurm'] and machine in ['ORION'] and task not in [ 'getic', 'arch' ]: strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_BATCH;">\n' % taskstr) if scheduler in ['slurm'] and task in ['getic', 'arch']: strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_SERVICE;">\n' % taskstr) strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr)) strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr)) if len(memstr) != 0: strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr)) strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr)) strings.append('\n') strings.append('\t<!-- END: Resource requirements for the workflow -->\n') return ''.join(strings)
def get_resources(dict_configs, cdump='gdas'): ''' Create resource entities ''' strings = [] strings.append( '\t<!-- BEGIN: Resource requirements for the workflow -->\n') strings.append('\n') base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) for task in taskplan: cfg = dict_configs[task] wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources( machine, cfg, task, cdump=cdump) taskstr = '%s_%s' % (task.upper(), cdump.upper()) strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr)) if scheduler in ['slurm'] and task in ['getic', 'arch']: strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' % taskstr) strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr)) strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr)) if len(memstr) != 0: strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr)) strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr)) strings.append('\n') strings.append('\t<!-- END: Resource requirements for the workflow -->\n') return ''.join(strings)
parser.add_argument('--resens', help='resolution of the ensemble model forecast', type=int, required=False, default=192) parser.add_argument('--comrot', help='full path to COMROT', type=str, required=False, default=None) parser.add_argument('--expdir', help='full path to EXPDIR', type=str, required=False, default=None) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', type=str, required=True) parser.add_argument('--edate', help='end date experiment', type=str, required=True) parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False) parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None) parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20) parser.add_argument('--cdump', help='CDUMP to start the experiment', type=str, required=False, default='gdas') parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False) parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None) parser.add_argument('--start', help='restart mode: warm or cold', type=str, choices=['warm', 'cold'], required=False, default='cold') args = parser.parse_args() machine = wfu.detectMachine() configdir = args.configdir if not configdir: configdir = os.path.abspath(os.path.dirname(__file__) + '/../parm/config') pslot = args.pslot idate = datetime.strptime(args.idate, '%Y%m%d%H') edate = datetime.strptime(args.edate, '%Y%m%d%H') icsdir = args.icsdir resdet = args.resdet resens = args.resens comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot) expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot) nens = args.nens cdump = args.cdump
def get_workflow(dict_configs, cdump='gdas'): ''' Create tasks for forecast only workflow ''' envars = [] envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) envars.append(rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>')) envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;')) envars.append(rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>')) envars.append(rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>')) base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) hpssarch = base.get('HPSSARCH', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_wafs = base.get('WAFSF', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() tasks = [] # getic if hpssarch in ['YES']: deps = [] data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # init deps = [] data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/[email protected]' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/[email protected]' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) if hpssarch in ['YES']: deps = [] dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies2 = rocoto.create_dependency(dep=deps) deps = [] deps.append(dependencies) if hpssarch in ['YES']: deps.append(dependencies2) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # waveinit if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars) tasks.append(task) tasks.append('\n') # waveprep if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: deps = [] dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%sinit' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # fcst deps = [] data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@[email protected]_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: deps = [] dep_dict = {'type': 'task', 'name': '%swaveprep' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps) deps = [] deps.append(dependencies) if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: deps.append(dependencies2) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # post deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]#dep#.txt' % (cdump, cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, metatask='post', varname=varname1, varval=varval1, vardict=vardict) tasks.append(task) tasks.append('\n') # wavepostsbs if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gnh_10m.@Y@m@d.@H0000' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.aoc_9km.@Y@m@d.@H0000' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/wave/rundata/%swave.out_grd.gsh_15m.@Y@m@d.@H0000' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wavepostbndpnt if do_wave in ['Y', 'YES']: deps = [] dep_dict = {'type':'task', 'name':'%sfcst' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wavepostbndpntbll if do_wave in ['Y', 'YES']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wavepostpnt if do_wave in ['Y', 'YES']: deps = [] dep_dict = {'type':'task', 'name':'%sfcst' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%swavepostbndpntbll' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wavegempak if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: deps = [] dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # waveawipsbulls if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: deps = [] dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%swavepostpnt' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # waveawipsgridded if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: deps = [] dep_dict = {'type':'task', 'name':'%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafs if do_wafs in ['Y', 'YES']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafsgcip if do_wafs in ['Y', 'YES']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafsgrib2 if do_wafs in ['Y', 'YES']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafsgrib20p25 if do_wafs in ['Y', 'YES']: deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/%s.@Y@m@d/@H/atmos/%[email protected]' % (cdump,cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafsblending if do_wafs in ['Y', 'YES']: deps = [] dep_dict = {'type': 'task', 'name': '%swafsgrib2' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # wafsblending0p25 if do_wafs in ['Y', 'YES']: deps = [] dep_dict = {'type': 'task', 'name': '%swafsgrib20p25' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') #postsnd if do_bufrsnd in ['Y', 'YES']: deps = [] dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # awips if do_awips in ['Y', 'YES']: deps = [] dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, metatask='awips', varname=varname1, varval=varval1, vardict=vardict) tasks.append(task) tasks.append('\n') # gempak if do_gempak in ['Y', 'YES']: deps = [] dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # vrfy deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # metp if do_metp in ['Y', 'YES']: deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%sarch' % cdump, 'offset':'-&INTERVAL;'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;') metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') metpenvars = envars + [sdate_gfs] + [metpcase] varname1 = 'metpcase' varval1 = 'g2g1 g2o1 pcp1' task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, metatask='metp', varname=varname1, varval=varval1) tasks.append(task) tasks.append('\n') # arch deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) if do_wave in ['Y', 'YES']: dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) tasks.append(task) tasks.append('\n') return ''.join(tasks)
def get_hyb_tasks(dict_configs, cycledef='enkf'): ''' Create Hybrid tasks ''' # Determine groups based on ensemble size and grouping base = dict_configs['base'] nens = base['NMEM_ENKF'] lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() eobs = dict_configs['eobs'] nens_eomg = eobs['NMEM_EOMGGRP'] neomg_grps = nens / nens_eomg EOMGGROUPS = ' '.join(['%02d' % x for x in range(1, neomg_grps + 1)]) efcs = dict_configs['efcs'] nens_efcs = efcs['NMEM_EFCSGRP'] nefcs_grps = nens / nens_efcs EFCSGROUPS = ' '.join(['%02d' % x for x in range(1, nefcs_grps + 1)]) earc = dict_configs['earc'] nens_earc = earc['NMEM_EARCGRP'] nearc_grps = nens / nens_earc EARCGROUPS = ' '.join(['%02d' % x for x in range(0, nearc_grps + 1)]) envars = [] if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) envars.append( rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>')) #envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump)) envars.append( rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>')) envars.append( rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>')) ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#') dict_tasks = OrderedDict() if eupd_cyc in ['BOTH']: cdumps = ['gfs', 'gdas'] elif eupd_cyc in ['GFS']: cdumps = ['gfs'] elif eupd_cyc in ['GDAS']: cdumps = ['gdas'] for cdump in cdumps: envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump) envars1 = envars + [envar_cdump] # eobs deps = [] dep_dict = {'type': 'task', 'name': '%sprep' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { 'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00' } deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) dict_tasks['%seobs' % cdump] = task # eomn, eomg if lobsdiag_forenkf in ['.F.', '.FALSE.']: deps = [] dep_dict = {'type': 'task', 'name': '%seobs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) eomgenvars = envars1 + [ensgrp] task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) dict_tasks['%seomn' % cdump] = task # eupd deps = [] if lobsdiag_forenkf in ['.F.', '.FALSE.']: dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump} else: dep_dict = {'type': 'task', 'name': '%seobs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) dict_tasks['%seupd' % cdump] = task # All hybrid tasks beyond this point are always executed in the GDAS cycle cdump = 'gdas' envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump) envars1 = envars + [envar_cdump] cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas' # ecen deps = [] dep_dict = {'type': 'task', 'name': '%sanal' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%seupd' % cdump_eupd} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('ecen', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) dict_tasks['%secen' % cdump] = task # efmn, efcs deps = [] dep_dict = {'type': 'task', 'name': '%secen' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { 'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00' } deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) efcsenvars = envars1 + [ensgrp] task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies, metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef) dict_tasks['%sefmn' % cdump] = task # epmn, epos deps = [] dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') eposenvars = envars1 + [fhrgrp] + [fhrlst] varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies, metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) dict_tasks['%sepmn' % cdump] = task # eamn, earc deps = [] dep_dict = {'type': 'metatask', 'name': '%sepmn' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) earcenvars = envars1 + [ensgrp] task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies, metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef) dict_tasks['%seamn' % cdump] = task return dict_tasks
def get_gdasgfs_tasks(dict_configs, cdump='gdas'): ''' Create GDAS or GFS tasks ''' envars = [] if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) envars.append( rocoto.create_envar(name='CDATE', value='<cyclestr>@Y@m@d@H</cyclestr>')) envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump)) envars.append( rocoto.create_envar(name='PDY', value='<cyclestr>@Y@m@d</cyclestr>')) envars.append( rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>')) base = dict_configs['base'] gfs_cyc = base.get('gfs_cyc', 0) dohybvar = base.get('DOHYBVAR', 'NO').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() dumpsuffix = base.get('DUMP_SUFFIX', '') dict_tasks = OrderedDict() # prep deps = [] dep_dict = { 'type': 'metatask', 'name': '%spost' % 'gdas', 'offset': '-06:00:00' } deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/gdas.@Y@m@d/@H/[email protected]' dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) data = '&DMPDIR;/%s%s.@Y@m@d/@H/%[email protected]_d' % ( cdump, dumpsuffix, cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS' ] and dohybvar in ['Y', 'YES'] else False if gfs_enkf and cdump in ['gfs']: if gfs_cyc == 4: task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) else: task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') else: task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sprep' % cdump] = task # anal deps = [] dep_dict = {'type': 'task', 'name': '%sprep' % cdump} deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES']: dep_dict = { 'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00' } deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sanal' % cdump] = task # fcst deps = [] dep_dict = {'type': 'task', 'name': '%sanal' % cdump} deps.append(rocoto.add_dependency(dep_dict)) if cdump in ['gdas']: dep_dict = { 'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00' } deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) elif cdump in ['gfs']: dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sfcst' % cdump] = task # post deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.nemsio' % (cdump, cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, metatask='post', varname=varname1, varval=varval1, vardict=vardict) dict_tasks['%spost' % cdump] = task # vrfy deps = [] dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%svrfy' % cdump] = task if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: #postsnd deps = [] dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%spostsnd' % cdump] = task if cdump in ['gfs'] and do_awips in ['Y', 'YES']: # awips deps = [] data = '&ROTDIR;/%s.@Y@m@d/@H/%[email protected]#dep#.grib2.idx' % ( cdump, cdump) dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, metatask='awips', varname=varname1, varval=varval1, vardict=vardict) dict_tasks['%sawips' % cdump] = task if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: # gempak deps = [] dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sgempak' % cdump] = task # arch deps = [] dep_dict = {'type': 'task', 'name': '%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sarch' % cdump] = task return dict_tasks
def get_hyb_resources(dict_configs): ''' Create hybrid resource entities ''' base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() compute_partition = base.get('COMPUTE_PARTITION_LINE', None) dict_resources = OrderedDict() # These tasks can be run in either or both cycles tasks1 = ['eobs', 'eomg', 'eupd'] if lobsdiag_forenkf in ['.T.', '.TRUE.']: tasks.remove('eomg') if eupd_cyc in ['BOTH']: cdumps = ['gfs', 'gdas'] elif eupd_cyc in ['GFS']: cdumps = ['gfs'] elif eupd_cyc in ['GDAS']: cdumps = ['gdas'] for cdump in cdumps: for task in tasks1: cfg = dict_configs['eobs'] if task in ['eomg' ] else dict_configs[task] wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources( machine, cfg, task, cdump=cdump) taskstr = '%s_%s' % (task.upper(), cdump.upper()) strings = [] strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr)) strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr)) strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr)) if len(memstr) != 0: strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr)) strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr)) dict_resources['%s%s' % (cdump, task)] = ''.join(strings) # These tasks are always run as part of the GDAS cycle cdump = 'gdas' tasks2 = ['ecen', 'efcs', 'epos', 'earc'] for task in tasks2: cfg = dict_configs[task] wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources( machine, cfg, task, cdump=cdump) taskstr = '%s_%s' % (task.upper(), cdump.upper()) strings = [] strings.append('\t<!ENTITY QUEUE_%s "%s">\n' % (taskstr, queuestr)) if scheduler in ['slurm'] and task in ['earc']: strings.append('\t<!ENTITY PARTITION_%s "&PARTITION_ARCH;">\n' % taskstr) strings.append('\t<!ENTITY WALLTIME_%s "%s">\n' % (taskstr, wtimestr)) strings.append('\t<!ENTITY RESOURCES_%s "%s">\n' % (taskstr, resstr)) if len(memstr) != 0: strings.append('\t<!ENTITY MEMORY_%s "%s">\n' % (taskstr, memstr)) strings.append('\t<!ENTITY NATIVE_%s "%s">\n' % (taskstr, natstr)) dict_resources['%s%s' % (cdump, task)] = ''.join(strings) return dict_resources
def get_definitions(base): ''' Create entities related to the experiment ''' machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) strings = [] strings.append('\n') strings.append( '\t<!-- Experiment parameters such as name, starting, ending dates -->\n' ) strings.append('\t<!ENTITY PSLOT "%s">\n' % base['PSLOT']) strings.append('\t<!ENTITY SDATE "%s">\n' % base['SDATE'].strftime('%Y%m%d%H%M')) strings.append('\t<!ENTITY EDATE "%s">\n' % base['EDATE'].strftime('%Y%m%d%H%M')) if base['gfs_cyc'] != 0: strings.append(get_gfs_dates(base)) strings.append('\n') strings.append('\t<!-- Run Envrionment -->\n') strings.append('\t<!ENTITY RUN_ENVIR "%s">\n' % base['RUN_ENVIR']) strings.append('\n') strings.append('\t<!-- Experiment and Rotation directory -->\n') strings.append('\t<!ENTITY EXPDIR "%s">\n' % base['EXPDIR']) strings.append('\t<!ENTITY ROTDIR "%s">\n' % base['ROTDIR']) strings.append('\n') strings.append('\t<!-- Directories for driving the workflow -->\n') strings.append('\t<!ENTITY HOMEgfs "%s">\n' % base['HOMEgfs']) strings.append('\t<!ENTITY JOBS_DIR "%s">\n' % base['BASE_JOB']) strings.append('\t<!ENTITY DMPDIR "%s">\n' % base['DMPDIR']) strings.append('\n') strings.append('\t<!-- Machine related entities -->\n') strings.append('\t<!ENTITY ACCOUNT "%s">\n' % base['ACCOUNT']) strings.append('\t<!ENTITY QUEUE "%s">\n' % base['QUEUE']) strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH']) if scheduler in ['slurm']: strings.append('\t<!ENTITY PARTITION_ARCH "%s">\n' % base['QUEUE_ARCH']) else: strings.append('\t<!ENTITY QUEUE_ARCH "%s">\n' % base['QUEUE_ARCH']) strings.append('\t<!ENTITY SCHEDULER "%s">\n' % wfu.get_scheduler(base['machine'])) if 'COMPUTE_PARTITION' in base: strings.append('\t<!ENTITY COMPUTE_PARTITION "%s">\n' % base.get('COMPUTE_PARTITION', None)) if 'SERVICE_PARTITION' in base: strings.append('\t<!ENTITY SERVICE_PARTITION "%s">\n' % base.get('SERVICE_PARTITION', None)) strings.append('\n') strings.append('\t<!-- Toggle HPSS archiving -->\n') strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n') strings.append('\n') strings.append('\t<!-- ROCOTO parameters that control workflow -->\n') strings.append('\t<!ENTITY CYCLETHROTTLE "3">\n') strings.append('\t<!ENTITY TASKTHROTTLE "25">\n') strings.append('\t<!ENTITY MAXTRIES "2">\n') strings.append('\n') return ''.join(strings)