def calc_dates(thisfile, info):
# Given a type of MOS and its run date and run time, calculate the
# dates (and associated filenames) needed to produce a complete graphic.
# Returns two lists:
# prevruns, a list of datetime objects (including the current run)
# prevfiles, a list of filenames to access for a single station (including the current file)
#
# 'thisfile' is the filename of the file for which to calculate previous dates.
# 'info' is a dictionary returned by find_info.
mostype = info['MOSTYPE']
thisrun = thisrun_as_dt(info)
# start the list of previous runs/files with the first run/file to make looping easier later
prevruns = [thisrun]
# DO NOT initialize 'prevfiles' with 'thisfile', otherwise the first file will end up
# listed twice because of the looping operation below
prevfiles = []
if 'ECMX' in mostype:
# ECMX MOS GUIDANCE = ECE (AWIPS)
# 00z runs for the previous 7 days (7 cycles)
numhrs = [24, 48, 72, 96, 120, 144, 168]
elif 'ECM' in mostype:
# ECM MOS GUIDANCE = ECS (AWIPS)
# 00z runs for the previous 2 days (2 cycles)
numhrs = [24, 48]
elif 'NAM' in mostype:
# NAM MOS GUIDANCE = MET (AWIPS)
# 12z and 00z runs going back 2-ish days (5 cycles)
numhrs = [12, 24, 36, 48, 60]
elif 'GFSX' in mostype:
# GFSX MOS GUIDANCE = MEX (AWIPS)
# 12z and 00z runs going back 7-ish days (15 cycles)
numhrs = [12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 168, 180]
elif 'GFS' in mostype:
# GFS MOS GUIDANCE = MAV (AWIPS)
# 12z, 18z, 00z, 06z runs going back 2-ish days (12 cycles)
numhrs = [6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72]
for hr in numhrs:
wayback = dt.timedelta(hours = hr)
prevruns.append(thisrun - wayback)
dictParms = mosHelper.transformFilename(thisfile)
for item in prevruns:
#Note: change the line below if the file naming convention changes
#prevfiles.append(thisfile[0:9] + item.strftime('%Y%m%d_%H'))
# Use functions from mosHelper for consistency in case of filename convention changes.
appendme = mosHelper.makeFilenames(dictParms['mostype'], dictParms['staname'], item.strftime('%Y'), item.strftime('%m'), item.strftime('%d'), item.strftime('%H'))['proc']
prevfiles.append(appendme)
return prevruns, prevfiles
def cleanHouse():
# Grab a reference to the existing logger.
# This only works if the script calling this function has
# already called mosHelper.setUpTheLogger().
module_logger = logging.getLogger('mosgraphics.cleanHouse')
dictDirNames = mosHelper.getDirNames()
# Can probably rewrite mosplots.calc_dates based on the work here. Perhaps
# in the ample free time with which all forecasters are blessed. Maybe use
# xrange instead, also?
#
# Alright, here's the deal. The UTC date may be in the future compared to the
# local date at times, so we'll toss in some negative numbers just to be sure the
# early day runs (00z, 06z) won't get accidentally deleted.
# Next, grab the current YYYY-MM-DD (local time) and use that as a starting point
# from which to calculate which files to keep. Some of the filenames may not exist
# yet, but that's OK. The important thing is that they won't be deleted. Yeah, that
# makes total sense...
hrsToKeep = {}
hrsToKeep['MEX'] = [-24, -12, 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 168, 180, 192, 204, 216]
hrsToKeep['MAV'] = [-24, -18, -12, -6, 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72]
hrsToKeep['MET'] = [-36, -24, -12, 0, 12, 24, 36, 48, 60, 72, 84]
#hrsToKeep['ECE'] = [-24, 0, 24, 48, 72, 96, 120, 144, 168, 192, 216]
#hrsToKeep['ECS'] = [-24, 0, 24, 48, 72, 96]
# Now. You're looking at now, sir. Everything that happens now, is happening now.
# What happened to then?
# We passed then.
# When?
# Just now. We're at now now.
rightnow = dt.datetime.now()
# But not anymore! Choose 0 o'clock as a baseline. It makes the math easier.
nowish = dt.datetime(year = rightnow.year, month = rightnow.month, day = rightnow.day, hour = 0)
keyIter = hrsToKeep.iterkeys()
# Loop over time to create filenames to keep
for key in keyIter:
keepfiles = []
for hr in hrsToKeep[key]:
mostype = key.lower()
goback = dt.timedelta(hours = hr)
prev = nowish - goback
Y = prev.strftime('%Y')
M = prev.strftime('%m')
D = prev.strftime('%d')
H = prev.strftime('%H')
appendme = mosHelper.makeFilenames(mostype, 'ABCD', Y, M, D, H)['raw']
keepfiles.append(appendme)
keepfiles = set(keepfiles)
# get the contents of the raw files directory for this mostype
rawfiles = set(mosHelper.listRawFiles(mostype))
# Suppose set1 = ([f1, f2, f3, etc.]) contains the filenames to keep, and
# set2 = ([f0, f1, f2, f3, f4, f5, f6]) has the names of all raw files.
# Then set2.difference(set1) is the set of files to delete.
delme = rawfiles.difference(keepfiles)
module_logger.info('%s are marked for deletion from %s', delme, mostype.upper())
for fn in delme:
fullname = os.path.join(dictDirNames['raw'], fn)
os.remove(fullname)
def check_primary(self):
# MDL's FTP site is the primary source. Attempt to access it.
# MDL changed the folder structure for MET:
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/nam/prod/nam_mos.YYYYMMDD/mdl_nammet.tXXz
# YYYYMMDD = year/month/day in UTC
# XX = cycle (00 or 12) in UTC
# This folder also contains files named 'mdl_nammme.tXXz'.
# MDL also changed the folder structure for MAV and MEX:
# MAV:
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfsmos.YYYYMMDD/mdl_gfsmav.tXXz
# MEX:
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfsmos.YYYYMMDD/mdl_gfsmex.tXXz
# YYYYMMDD = year/month/day in UTC
# XX = cycle (00, 06, 12, or 18 for the MAV, 00 or 12 for the MEX and MET) in UTC
dictURLs = {'MAV': 'ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/',
'MEX': 'ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/',
'MET': 'ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/nam/prod/'
}
fileurls = []
localfnames = []
mosname = self.mostype
url = dictURLs[mosname]
try:
response = urllib2.urlopen(url)
contents = response.read()
response.close()
# The FTP directories contain many folders, only 2 of which are of
# interest: 'gfsmos.YYYYMMDD' (MAV, MEX) and 'nam_mos.YYYYMMDD' (MET),
# where YYYMMDD is the UTC date of the MOS run.
if mosname is 'MET':
re_folder = r'nam_mos\.[0-9]+'
elif mosname in ['MAV', 'MEX']:
re_folder = r'gfsmos\.[0-9]+'
# FutureDev: better error handling here?
RD = re.compile(re_folder)
folders = RD.findall(contents)
#print 'Found these folders: %s' % (folders)
for item in folders:
folderurl = '{}/{}/'.format(url, item)
response = urllib2.urlopen(folderurl)
contents = response.read()
response.close()
# Each of these folders contains multiple files (and possibly more folders).
# The files of interest follow these filenames:
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/nam/prod/nam_mos.YYYYMMDD/mdl_nammet.tXXz
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfsmos.YYYYMMDD/mdl_gfsmav.tXXz
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfsmos.YYYYMMDD/mdl_gfsmex.tXXz
# XX = cycle (00, 06, 12, or 18 for the MAV, 00 or 12 for the MEX and MET) in UTC
if mosname is 'MET':
re_filename = r'mdl_nammet\.t[0-9][0-9]z'
elif mosname is 'MAV':
re_filename = r'mdl_gfsmav\.t[0-9][0-9]z'
elif mosname is 'MEX':
re_filename = r'mdl_gfsmex\.t[0-9][0-9]z'
CY = re.compile(re_filename)
mosfiles = CY.findall(contents)
#print '%s has these files: %s' %(item, mosfiles)
for fname in mosfiles:
# Construct the file URL
furl = '{}/{}'.format(folderurl, fname)
fileurls.append(furl)
# Construct the local filename. ABCD is a placeholder value
# because 'staname' is undefined for raw files in
# mosHelper.makeFilenames.
base = item.split('.')[1]
yr = base[0:4]
mon = base[4:6]
dy = base[6:8]
cycle = fname.split('.')[1][1:3]
dictFn = mosHelper.makeFilenames(mosname, 'ABCD', yr, mon, dy, cycle)
localfilename = dictFn['raw']
localfnames.append(localfilename)
# After all file URLs have been found, store them as an object property.
# Ditto on the local filenames.
self.fileurls = fileurls
self.localfnames = localfnames
# There's probably a better way to do this. Let '1' stand for success.
return 1
except urllib2.URLError:
# There's probably a better way to do this. Let '0' stand for failure.
return 0
def check_backup(self):
# MDL's other site is the secondary source. Attempt to access it.
# There is also another source for MAV, MET, MEX, but the date and cycle are not present
# in all filenames. Use mosHelper.getIssuance to determine these elements.
# MAV:
# http://www.nws.noaa.gov/mdl/forecast/text/avnmav.txt
# MET:
# http://www.nws.noaa.gov/mdl/forecast/text/nammet.txt
# MEX:
# http://www.nws.noaa.gov/mdl/forecast/text/mrfmex00.txt
# http://www.nws.noaa.gov/mdl/forecast/text/mrfmex12.txt
backupURLs = {'MAV': ['http://www.nws.noaa.gov/mdl/forecast/text/avnmav.txt'],
'MEX': ['http://www.nws.noaa.gov/mdl/forecast/text/mrfmex00.txt',
'http://www.nws.noaa.gov/mdl/forecast/text/mrfmex12.txt'
],
'MET': ['http://www.nws.noaa.gov/mdl/forecast/text/nammet.txt']
}
mosname = self.mostype
urls = backupURLs[mosname]
bucket = []
fileurls = []
localfnames = []
try:
for u in urls:
response = urllib2.urlopen(u)
contents = response.readlines()
response.close()
bucket.append(contents)
# This seems silly. We've already downloaded the file. Why mark it
# for downloading again?
# FutureDev: Surely there's a better way.
fileurls.append(u)
for item in bucket:
info = mosHelper.getIssuance(item)
yr = info['DATE'].split('/')[2]
mon = info['DATE'].split('/')[0]
dy = info['DATE'].split('/')[1]
cycle = info['CYCLE']
dictFn = mosHelper.makeFilenames(mosname, 'ABCD', yr, mon, dy, cycle)
localfilename = dictFn['raw']
localfnames.append(localfilename)
# After all file URLs have been found, store them as an object property.
# Ditto on the local filenames.
self.fileurls = fileurls
self.localfnames = localfnames
# There's probably a better way to do this. Let '1' stand for success.
return 1
except urllib2.URLError:
# There's probably a better way to do this. Let '0' stand for failure.
return 0
