def run(date):
"""Upload this date's worth of data!"""
os.chdir("/mesonet/tmp")
tarfn = date.strftime("iem_%Y%m%d.tgz")
# Step 1: Create a gzipped tar file
cmd = "tar -czf %s /mesonet/ARCHIVE/data/%s" % (tarfn,
date.strftime("%Y/%m/%d"))
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
send2box(tarfn, date.strftime("/IEMArchive/%Y/%m"), tmpdir="/mesonet/tmp")
os.unlink(tarfn)
def run(date):
"""Upload this date's worth of data!"""
os.chdir("/mesonet/tmp")
tarfn = date.strftime("iem_%Y%m%d.tgz")
# Step 1: Create a gzipped tar file
cmd = "tar -czf %s /mesonet/ARCHIVE/data/%s" % (tarfn,
date.strftime("%Y/%m/%d"))
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
send2box(tarfn, date.strftime("/IEMArchive/%Y/%m"),
tmpdir="/mesonet/tmp")
os.unlink(tarfn)
def run(date):
for nexrad in NEXRADS:
os.chdir("/mnt/nexrad3/nexrad/NIDS/%s" % (nexrad, ))
cmd = ("tar -czf /mesonet/tmp/%sradar.tgz ???/???_%s_*"
) % (nexrad, date.strftime("%Y%m%d"))
subprocess.call(cmd, shell=True)
filenames = ['/mesonet/tmp/%sradar.tgz' % (x, ) for x in NEXRADS]
remotenames = ['%s_%s.tgz' % (x, date.strftime("%Y%m%d")) for x in NEXRADS]
send2box(filenames, "/IowaNexrad3/%s" % (date.strftime("%Y/%m"),),
remotenames=remotenames)
for filename in filenames:
if os.path.isfile(filename):
os.unlink(filename)
def main():
"""Go Main!"""
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.debug("Setting logging to debug...")
ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")
local_filenames = {}
remote_filenames = {}
for fn in glob.glob("*.zip"):
valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
if valid >= ceiling:
continue
rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
if fn.startswith("fx"):
rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m")
if rp not in remote_filenames:
local_filenames[rp] = []
remote_filenames[rp] = []
# Move file to temp fn so that we don't attempt to upload this twice
fn2 = "%s.xfer" % (fn, )
os.rename(fn, fn2)
local_filenames[rp].append(fn2)
remote_filenames[rp].append(fn)
fs = None
for rp in remote_filenames:
_ftps, ress = util.send2box(local_filenames[rp],
rp,
remotenames=remote_filenames[rp],
fs=fs)
for fn, res in zip(local_filenames[rp], ress):
if res is not False:
os.unlink(fn)
ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")
local_filenames = {}
remote_filenames = {}
for fn in glob.glob("*.zip"):
valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
if valid >= ceiling:
continue
rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
if fn.startswith("fx"):
rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m")
if rp not in remote_filenames:
local_filenames[rp] = []
remote_filenames[rp] = []
# Move file to temp fn so that we don't attempt to upload this twice
fn2 = "%s.xfer" % (fn, )
os.rename(fn, fn2)
local_filenames[rp].append(fn2)
remote_filenames[rp].append(fn)
fs = None
for rp in remote_filenames:
ftps, ress = util.send2box(local_filenames[rp],
rp,
remotenames=remote_filenames[rp],
fs=fs)
for fn, res in zip(local_filenames[rp], ress):
if res is not False:
os.unlink(fn)
"""Upload i5 analysis to CyBox"""
import os
from pyiem import util
import datetime
import glob
ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")
remote_paths = {}
remote_names = {}
for fn in glob.glob("*.zip"):
valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
if valid >= ceiling:
continue
rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
if rp not in remote_paths:
remote_paths[rp] = []
remote_names[rp] = []
# Move file to temp fn so that we don't attempt to upload this twice
fn2 = "%s.xfer" % (fn,)
os.rename(fn, fn2)
remote_paths[rp].append(fn2)
remote_names[rp].append(fn)
for rp in remote_paths:
util.send2box(remote_paths[rp], rp, remotenames=remote_names[rp])
for fn in remote_paths[rp]:
os.unlink(fn)
logger.setLevel(logging.DEBUG)
logging.debug("Setting logging to debug...")
ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")
local_filenames = {}
remote_filenames = {}
for fn in glob.glob("*.zip"):
valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
if valid >= ceiling:
continue
rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
if fn.startswith("fx"):
rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m")
if rp not in remote_filenames:
local_filenames[rp] = []
remote_filenames[rp] = []
# Move file to temp fn so that we don't attempt to upload this twice
fn2 = "%s.xfer" % (fn,)
os.rename(fn, fn2)
local_filenames[rp].append(fn2)
remote_filenames[rp].append(fn)
fs = None
for rp in remote_filenames:
fs = util.send2box(local_filenames[rp], rp,
remotenames=remote_filenames[rp], fs=fs)
for fn in local_filenames[rp]:
os.unlink(fn)
