def main_tape(): postmsg('hwrf_archive tape step starting') conf = hwrf_expt.conf logger = conf.log('archive.disk') archive = conf.getloc('archive', 'NONE') if conf.has_section('archive'): makethedir = conf.getbool('archive', 'mkdir', False) else: makethedir = False if archive == 'NONE': logger.info('No archive location specified. Exiting.') postmsg('hwrf_archive disk step has nothing to do when archiving is ' 'disabled.') return with NamedDir(conf.getdir('com')): flist = [filename + '\n' for filename in glob.glob('*')] flist.sort() files = ''.join(flist) assert (len(files) > 0) if archive.lower() == 'none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5] == 'disk:': logger.info('Disk archiving enabled.') logger.info('Nothing to do in the HPSS archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hwrf_archive tape step does nothing when using disk ' 'archives.') return elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:': jlogger.error('Ignoring invalid archive method %s in %s' % (archive[0:4], archive)) return if makethedir: adir = os.path.dirname(archive[5:]) logger.info('%s: make this HPSS directory, even if it exists' % (adir, )) mcmd = exe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir] run(mcmd, logger=logger) if archive[0:5] == 'hpss:': path = archive[5:] flags = '-cvp' cmd = exe(conf.getexe('htar'))[flags + 'f', path, '-L', '-'] << files elif archive[0:5] == 'hpsz:': topath = archive[5:] frompath = conf.strinterp('config', '{WORKhwrf}/stage-archive.tar.gz') cmd = exe(conf.getexe('hsi'))['put', frompath, ':', topath] checkrun(cmd, logger=logger) postmsg('hwrf_archive tape step completed')
def products(): jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: starting nhc_products job for {out_prefix}')) with NamedDir(hwrf_expt.WORKhwrf, logger=logging.getLogger()) as t: hwrf_expt.nhcp.run() jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: completed nhc_products job for {out_prefix}'))
def post(): produtil.setup.setup() jlogger.info('starting post') import hwrf_expt hwrf_expt.init_module() run_copier = hwrf_expt.conf.getbool('config', 'post_runs_wrfcopier', False) run_satpost = hwrf_expt.conf.getbool('config', 'run_satpost', True) # Make sure we check all tasks to see if they're posted: hwrf_expt.nonsatpost.state = UNSTARTED hwrf_expt.satpost.state = UNSTARTED if run_copier: hwrf_expt.wrfcopier.state = UNSTARTED logger = logging.getLogger('exhwrf_post') # Change to a temp directory to run the post: with NamedDir(hwrf_expt.WORKhwrf, logger=logger) as t: #hwrf_expt.ds.dump() # dump entire database state to stdout alldone = False while not alldone: before = int(time.time()) if run_copier: if not done(hwrf_expt.wrfcopier): hwrf_expt.wrfcopier.runpart() if not done(hwrf_expt.nonsatpost): hwrf_expt.nonsatpost.runpart() if not done(hwrf_expt.nonsatpost): hwrf_expt.nonsatpost.runpart() if run_satpost: if not done(hwrf_expt.satpost): hwrf_expt.satpost.runpart() if not done(hwrf_expt.nonsatpost): hwrf_expt.nonsatpost.runpart() alldone = ( done(hwrf_expt.satpost) or not run_satpost ) \ and done(hwrf_expt.nonsatpost) \ and ( not run_copier or done(hwrf_expt.wrfcopier) ) after = int(time.time()) took = after - before threshold = 5 sleeptime = 20 if took < threshold: logger.info( 'Post loop iteration took only %d seconds, which is ' 'less than the threshold of %d seconds. Will sleep ' '%d seconds.' % (took, threshold, sleeptime)) time.sleep(sleeptime) else: logger.info('Post loop iteration took %d seconds, ' 'which is above the threshold of %d. ' 'Sleeping only one second.' % (took, threshold)) time.sleep(1) # avoid thrash loop in case of logic error logger.info('Done sleeping.') jlogger.info('completed post')
def gribber(): jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: starting regribbing job for {out_prefix}')) with NamedDir(hwrf_expt.WORKhwrf, logger=logging.getLogger()) as t: hwrf_expt.gribber.uncomplete() #hwrf_expt.gribber.unrun() hwrf_expt.gribber.run() jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: completed regribbing job for {out_prefix}'))
def main_disk(): postmsg('hwrf_archive disk step starting') conf = hwrf_expt.conf logger = conf.log('archive.disk') if conf.has_section('archive'): makethedir = conf.getbool('archive', 'mkdir', False) else: makethedir = False archive = conf.getloc('archive', 'NONE') if archive == 'NONE': logger.info('No archive location specified. Exiting.') postmsg('hwrf_archive disk step has nothing to do when archiving is ' 'disabled.') return with NamedDir(conf.getdir('com')): flist = [filename + '\n' for filename in glob.glob('*')] flist.sort() files = ''.join(flist) assert (len(files) > 0) if archive.lower() == 'none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5] == 'disk:': path = archive[5:] if makethedir: adir = os.path.dirname(path) if not os.path.exists(adir): produtil.fileop.makedirs(adir, logger=logger) flags = '-cvp' if path[-3:] == '.gz' or path[-4:] == '.tgz': flags += 'z' cmd = exe(conf.getexe('tar'))[flags + 'f', path, '-T', '-'] << files elif archive[0:5] == 'hpss:': logger.info('HPSS archiving enabled.') logger.info('Nothing to do in the disk archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hwrf_archive disk step does nothing when using htar ' 'archives.') return elif archive[0:5] == 'hpsz:': path = conf.strinterp('config', '{WORKhwrf}/stage-archive.tar.gz') cmd = exe(conf.getexe('tar'))['-cvpf', path, '-T', '-'] << files else: jlogger.error('Ignoring invalid archive method %s in %s' % (archive[0:4], archive)) return checkrun(cmd, logger=logger) postmsg('hwrf_archive disk step completed')
def copier(): post_runs_copier = hwrf_expt.conf.getbool('config', 'post_runs_wrfcopier', False) if not post_runs_copier: jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: starting wrfcopier job for {out_prefix}')) with NamedDir(hwrf_expt.WORKhwrf, logger=logging.getLogger()) as t: hwrf_expt.wrfcopier.run(check_all=True) jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: completed wrfcopier job for {out_prefix}')) else: jlogger.info('Products job will not run wrfcopier, post will do it.') gribber()
def tracker(n): jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: starting domain {dom} tracker job for {out_prefix}', dom=n)) with NamedDir(hwrf_expt.WORKhwrf, logger=logging.getLogger()) as t: if n == 3: hwrf_expt.tracker.run() elif n == 2: hwrf_expt.trackerd02.run() elif n == 1: hwrf_expt.trackerd01.run() jlogger.info( hwrf_expt.conf.strinterp( 'config', '{stormlabel}: completed domain {dom} tracker job ' 'for {out_prefix}', dom=n))
def main_disk(): """!Main program for disk archiving. Creates an on-disk archiving for one of two cases: * disk:/path/to/archive.tar.gz --- generates an on-disk *.tar.gz archive in a long-term storage disk area * hpsz:/path/to/tape-archive.tar.gz --- generates an on-disk *.tar.gz archive in a temporary location so it can be copied to tape in a later step.""" postmsg('hafs_archive disk step starting') environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs') #conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs) conf = hafs.launcher.load(environ_CONFhafs) logger = conf.log('archive.disk') if conf.has_section('archive'): makethedir = conf.getbool('archive', 'mkdir', False) else: makethedir = False archive = conf.getloc('archive', 'NONE') if archive == 'NONE': logger.info('No archive location specified. Exiting.') postmsg('hafs_archive disk step has nothing to do when archiving is ' 'disabled.') return with NamedDir(conf.getdir('com')): flist = [filename + '\n' for filename in glob.glob('*')] flist.sort() files = ''.join(flist) assert (len(files) > 0) if archive.lower() == 'none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5] == 'disk:': path = archive[5:] if makethedir: adir = os.path.dirname(path) if not os.path.exists(adir): produtil.fileop.makedirs(adir, logger=logger) flags = '-cvp' if path[-3:] == '.gz' or path[-4:] == '.tgz': flags += 'z' cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T', '-'] << files elif archive[0:5] == 'hpss:': logger.info('HPSS archiving enabled.') logger.info('Nothing to do in the disk archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hafs_archive disk step does nothing when using htar ' 'archives.') return elif archive[0:5] == 'hpsz:': path = conf.strinterp('config', '{WORKhafs}/stage-archive.tar.gz') flags = '-cvp' if path[-3:] == '.gz' or path[-4:] == '.tgz': flags += 'z' cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T', '-'] << files else: jlogger.error('Ignoring invalid archive method %s in %s' % (archive[0:4], archive)) return checkrun(cmd, logger=logger) donefile = path + '.done' with open(donefile, 'wt') as f: f.write('hafs_archive disk step completed\n') postmsg('hafs_archive disk step completed')
def main_tape(): """!Main program for tape archiving. Does one of two things: * hpss:/path/to/archive.tar --- will use the htar command to archive COM directory outputs * hpsz:/path/to/archive.tar.gz --- will copy a tar.gz file from a temporary area, made by the disk archiving step, to a tape destination using the "hsi put" command.""" postmsg('hafs_archive tape step starting') environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs') #conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs) conf = hafs.launcher.load(environ_CONFhafs) logger = conf.log('archive.tape') archive = conf.getloc('archive', 'NONE') if conf.has_section('archive'): makethedir = conf.getbool('archive', 'mkdir', False) else: makethedir = False if archive == 'NONE': logger.info('No archive location specified. Exiting.') postmsg('hafs_archive disk step has nothing to do when archiving is ' 'disabled.') return with NamedDir(conf.getdir('com')): flist = [filename + '\n' for filename in glob.glob('*')] flist.sort() files = ''.join(flist) assert (len(files) > 0) if archive.lower() == 'none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5] == 'disk:': logger.info('Disk archiving enabled.') logger.info('Nothing to do in the HPSS archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hafs_archive tape step does nothing when using disk ' 'archives.') return elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:': jlogger.error('Ignoring invalid archive method %s in %s' % (archive[0:4], archive)) return if makethedir: adir = os.path.dirname(archive[5:]) logger.info('%s: make this HPSS directory, even if it exists' % (adir, )) mcmd = batchexe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir] run(mcmd, logger=logger) if archive[0:5] == 'hpss:': path = archive[5:] flags = '-cvp' cmd = batchexe(conf.getexe('htar'))[flags + 'f', path, '-L', '-'] << files elif archive[0:5] == 'hpsz:': topath = archive[5:] frompath = conf.strinterp('config', '{WORKhafs}/stage-archive.tar.gz') cmd = batchexe(conf.getexe('hsi'))['put', frompath, ':', topath] checkrun(cmd, logger=logger) postmsg('hafs_archive tape step completed')