def run(date): """Upload this date's worth of data!""" os.chdir("/mesonet/tmp") tarfn = date.strftime("iem_%Y%m%d.tgz") # Step 1: Create a gzipped tar file cmd = "tar -czf %s /mesonet/ARCHIVE/data/%s" % (tarfn, date.strftime("%Y/%m/%d")) subprocess.call(cmd, shell=True, stderr=subprocess.PIPE) send2box(tarfn, date.strftime("/IEMArchive/%Y/%m"), tmpdir="/mesonet/tmp") os.unlink(tarfn)
def run(date): for nexrad in NEXRADS: os.chdir("/mnt/nexrad3/nexrad/NIDS/%s" % (nexrad, )) cmd = ("tar -czf /mesonet/tmp/%sradar.tgz ???/???_%s_*" ) % (nexrad, date.strftime("%Y%m%d")) subprocess.call(cmd, shell=True) filenames = ['/mesonet/tmp/%sradar.tgz' % (x, ) for x in NEXRADS] remotenames = ['%s_%s.tgz' % (x, date.strftime("%Y%m%d")) for x in NEXRADS] send2box(filenames, "/IowaNexrad3/%s" % (date.strftime("%Y/%m"),), remotenames=remotenames) for filename in filenames: if os.path.isfile(filename): os.unlink(filename)
def main(): """Go Main!""" if len(sys.argv) > 1 and sys.argv[1] == 'debug': logger = logging.getLogger() logger.setLevel(logging.DEBUG) logging.debug("Setting logging to debug...") ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24) os.chdir("/mesonet/share/pickup/ntrans") local_filenames = {} remote_filenames = {} for fn in glob.glob("*.zip"): valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M') if valid >= ceiling: continue rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H") if fn.startswith("fx"): rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m") if rp not in remote_filenames: local_filenames[rp] = [] remote_filenames[rp] = [] # Move file to temp fn so that we don't attempt to upload this twice fn2 = "%s.xfer" % (fn, ) os.rename(fn, fn2) local_filenames[rp].append(fn2) remote_filenames[rp].append(fn) fs = None for rp in remote_filenames: _ftps, ress = util.send2box(local_filenames[rp], rp, remotenames=remote_filenames[rp], fs=fs) for fn, res in zip(local_filenames[rp], ress): if res is not False: os.unlink(fn)
ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24) os.chdir("/mesonet/share/pickup/ntrans") local_filenames = {} remote_filenames = {} for fn in glob.glob("*.zip"): valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M') if valid >= ceiling: continue rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H") if fn.startswith("fx"): rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m") if rp not in remote_filenames: local_filenames[rp] = [] remote_filenames[rp] = [] # Move file to temp fn so that we don't attempt to upload this twice fn2 = "%s.xfer" % (fn, ) os.rename(fn, fn2) local_filenames[rp].append(fn2) remote_filenames[rp].append(fn) fs = None for rp in remote_filenames: ftps, ress = util.send2box(local_filenames[rp], rp, remotenames=remote_filenames[rp], fs=fs) for fn, res in zip(local_filenames[rp], ress): if res is not False: os.unlink(fn)
"""Upload i5 analysis to CyBox""" import os from pyiem import util import datetime import glob ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24) os.chdir("/mesonet/share/pickup/ntrans") remote_paths = {} remote_names = {} for fn in glob.glob("*.zip"): valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M') if valid >= ceiling: continue rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H") if rp not in remote_paths: remote_paths[rp] = [] remote_names[rp] = [] # Move file to temp fn so that we don't attempt to upload this twice fn2 = "%s.xfer" % (fn,) os.rename(fn, fn2) remote_paths[rp].append(fn2) remote_names[rp].append(fn) for rp in remote_paths: util.send2box(remote_paths[rp], rp, remotenames=remote_names[rp]) for fn in remote_paths[rp]: os.unlink(fn)
logger.setLevel(logging.DEBUG) logging.debug("Setting logging to debug...") ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24) os.chdir("/mesonet/share/pickup/ntrans") local_filenames = {} remote_filenames = {} for fn in glob.glob("*.zip"): valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M') if valid >= ceiling: continue rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H") if fn.startswith("fx"): rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m") if rp not in remote_filenames: local_filenames[rp] = [] remote_filenames[rp] = [] # Move file to temp fn so that we don't attempt to upload this twice fn2 = "%s.xfer" % (fn,) os.rename(fn, fn2) local_filenames[rp].append(fn2) remote_filenames[rp].append(fn) fs = None for rp in remote_filenames: fs = util.send2box(local_filenames[rp], rp, remotenames=remote_filenames[rp], fs=fs) for fn in local_filenames[rp]: os.unlink(fn)