def clearProcFiles():
# Grab a reference to the existing logger.
# This only works if the script calling this function has
# already called mosHelper.setUpTheLogger().
module_logger = logging.getLogger('mosgraphics.clearProc')
dictDirNames = mosHelper.getDirNames()
contents = os.listdir(dictDirNames['proc'])
module_logger.info('Deleting %s processed files', len(contents))
for fn in contents:
fullname = os.path.join(dictDirNames['proc'], fn)
os.remove(fullname)
def GrabEm():
# Grab a reference to the existing logger.
# This only works if the script calling this function has
# already called mosHelper.setUpTheLogger().
module_logger = logging.getLogger('mosgraphics.GrabEm')
dictDirNames = mosHelper.getDirNames()
moslist = ['MET', 'MEX', 'MAV']
# Use this to keep track of which files were written and still need
# to be processed with mosHelper.parseStations.
rawfiles = []
for mosname in moslist:
tempObj = MOS(mosname)
tempObj.set_filethresh()
module_logger.info('Asking MDL for the {}'.format(mosname))
status = tempObj.check_primary()
if status is not 1:
module_logger.warning('Lost the connection with MDL. File was not downloaded.')
module_logger.info('Trying another server')
status2 = tempObj.check_backup()
if status2 is not 1:
module_logger.warning('Bummer. Unable to download {}'.format(mosname))
elif status2 is 1:
module_logger.info('Success!')
else:
module_logger.info('? ? ? ? ? ?')
if tempObj.fileurls is not None:
for furl, localfilename in zip(tempObj.fileurls, tempObj.localfnames):
# If there already exists a file with the intended localfilename,
# check to see if it has an appropriate size. If the file seems
# too small, then try downloading it again. Otherwise, don't bother
# because it's probably OK.
existingRawFiles = mosHelper.listRawFiles(mosname)
if localfilename in existingRawFiles:
module_logger.info('{} already exists on disk.'.format(localfilename))
fpath = os.path.join(dictDirNames['raw'], localfilename)
# If the file size is too small, then something went wrong the
# last time the file was downloaded. Try to download it again
# now so that it will be available for the next script run.
thresh = tempObj.filethresh * 1000
if os.path.getsize(fpath) > thresh:
module_logger.info('Skipping. It\'s probably OK.')
# 'continue': the current iteration of the loop terminates
# and execution continues with the next iteration of the loop.
continue
else:
module_logger.info('Downloading. The copy on disk seems too small.')
# Note that in order to reach this part of the script, the file
# size must pass the above if-else.
response = urllib2.urlopen(furl)
contents = response.read()
response.close()
module_logger.info('Writing to %s', localfilename)
fullname = os.path.join(dictDirNames['raw'], localfilename)
output = open(fullname, 'w')
output.write(contents)
output.close()
rawfiles.append(localfilename)
else:
module_logger.info('A rolling stone gathers no {} MOS.'.format(mosname))
return(rawfiles)
def cleanHouse():
# Grab a reference to the existing logger.
# This only works if the script calling this function has
# already called mosHelper.setUpTheLogger().
module_logger = logging.getLogger('mosgraphics.cleanHouse')
dictDirNames = mosHelper.getDirNames()
# Can probably rewrite mosplots.calc_dates based on the work here. Perhaps
# in the ample free time with which all forecasters are blessed. Maybe use
# xrange instead, also?
#
# Alright, here's the deal. The UTC date may be in the future compared to the
# local date at times, so we'll toss in some negative numbers just to be sure the
# early day runs (00z, 06z) won't get accidentally deleted.
# Next, grab the current YYYY-MM-DD (local time) and use that as a starting point
# from which to calculate which files to keep. Some of the filenames may not exist
# yet, but that's OK. The important thing is that they won't be deleted. Yeah, that
# makes total sense...
hrsToKeep = {}
hrsToKeep['MEX'] = [-24, -12, 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 168, 180, 192, 204, 216]
hrsToKeep['MAV'] = [-24, -18, -12, -6, 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72]
hrsToKeep['MET'] = [-36, -24, -12, 0, 12, 24, 36, 48, 60, 72, 84]
#hrsToKeep['ECE'] = [-24, 0, 24, 48, 72, 96, 120, 144, 168, 192, 216]
#hrsToKeep['ECS'] = [-24, 0, 24, 48, 72, 96]
# Now. You're looking at now, sir. Everything that happens now, is happening now.
# What happened to then?
# We passed then.
# When?
# Just now. We're at now now.
rightnow = dt.datetime.now()
# But not anymore! Choose 0 o'clock as a baseline. It makes the math easier.
nowish = dt.datetime(year = rightnow.year, month = rightnow.month, day = rightnow.day, hour = 0)
keyIter = hrsToKeep.iterkeys()
# Loop over time to create filenames to keep
for key in keyIter:
keepfiles = []
for hr in hrsToKeep[key]:
mostype = key.lower()
goback = dt.timedelta(hours = hr)
prev = nowish - goback
Y = prev.strftime('%Y')
M = prev.strftime('%m')
D = prev.strftime('%d')
H = prev.strftime('%H')
appendme = mosHelper.makeFilenames(mostype, 'ABCD', Y, M, D, H)['raw']
keepfiles.append(appendme)
keepfiles = set(keepfiles)
# get the contents of the raw files directory for this mostype
rawfiles = set(mosHelper.listRawFiles(mostype))
# Suppose set1 = ([f1, f2, f3, etc.]) contains the filenames to keep, and
# set2 = ([f0, f1, f2, f3, f4, f5, f6]) has the names of all raw files.
# Then set2.difference(set1) is the set of files to delete.
delme = rawfiles.difference(keepfiles)
module_logger.info('%s are marked for deletion from %s', delme, mostype.upper())
for fn in delme:
fullname = os.path.join(dictDirNames['raw'], fn)
os.remove(fullname)
def makeDisplayArrays(filename):
# Given a filename of the expected form, figure out which previous
# files are needed to construct complete arrays of all data needed
# for the display of each configured fcst element. Load those files,
# if they exist, and build the complete arrays. Then, construct display
# arrays for each fcst element by selecting array elements from the complete
# arrays based on mostype and model run date/time.
# Returns the following:
# dispArr, a dictionary of display arrays with keys 'X', 'N', 'P12'
# dtXaxis, a dictionary with datetime objects to be used for creating x-axis labels
# infoDict, a dictionary returned by find_info (needed to construct the title and axes annotations)
# prevruns, a list of datetime objects (including the current run)
#
# 'filename' is a complete filename, including the extension, suitable for passing to load_file.
# 'filename' should not include the file path b/c that is added during this function.
dictDirNames = mosHelper.getDirNames()
fullname = os.path.join(dictDirNames['proc'], filename)
d = load_file(fullname)
infoDict = find_info(d)
prevruns, prevfiles = calc_dates(filename, infoDict)
#allf = []
allxn = []
allp12 = []
allwsp = []
allq12 = []
for fn in prevfiles:
try:
fullname2 = os.path.join(dictDirNames['proc'], fn)
dd = load_file(fullname2)
#fhr = yoinkFromMOS(dd, 'FHR')
xn = yoinkFromMOS(dd, 'XN')
p12 = yoinkFromMOS(dd, 'P12')
wsp = yoinkFromMOS(dd, 'WSP')
q12 = yoinkFromMOS(dd, 'Q12')
# As a side note, fhr, xn, p12 will have length 0 if the
# wxelement was not found in MOS. This matters later.
#allf.append(fhr)
allxn.append(xn)
allp12.append(p12)
allwsp.append(wsp)
allq12.append(q12)
except:
# If the file does not exist, use empty np arrays as placeholders
# On reflection, this may not be an entirely kosher use of try/except. Read up on this.
#allf.append(np.array([]))
allxn.append(np.array([]))
allp12.append(np.array([]))
allwsp.append(np.array([]))
allq12.append(np.array([]))
# At this point, if the wxelement was not found in that MOS type (e.g., NSTU has no
# X/N line in the MAV), then allElem will be an array whose elements each have size 0.
# This matters because we don't want to try constructing a display array with no data.
# Handle this case by checking to see which elements are present and only constructing
# display arrays for those ones. Note that a different situation is a site where
# all entries for a given element are 999 (missing), such as TJMZ's minT, and this is not
# handled here (the plot will be created and it will be empty).
# These variables are used below to construct display arrays for the given wxelements and data
wxelements = []
alldata = {}
# find a way to consolidate these loops to avoid repetition
count = 0
for item in allxn:
if len(item) == 0:
count = count + 1
if count < len(allxn):
wxelements.append('X')
alldata['X'] = allxn
wxelements.append('N')
alldata['N'] = allxn
count = 0
for item in allp12:
if len(item) == 0:
count = count + 1
if count < len(allp12):
wxelements.append('P12')
alldata['P12'] = allp12
count = 0
for item in allwsp:
if len(item) == 0:
count = count + 1
if count < len(allwsp):
wxelements.append('WSP')
alldata['WSP'] = allwsp
count = 0
for item in allq12:
if len(item) == 0:
count = count + 1
if count < len(allq12):
wxelements.append('Q12')
alldata['Q12'] = allq12
# Make life easy by using keys that are constructed from information returned by find_info.
modelKey = infoDict['MOSTYPE'] + ' ' + infoDict['RUNTIME']
# display array size: [row, col]
# start, stop, step, and jump are indices:
# start = index for the first element of the first row
# stop = index for the last element of the first row (may be +/-1 because of Python slicing)
# step = count by this many to go from start to stop (e.g, 0 to 14 by 2 means step = 2)
# jump = count by this many to go from start to the first index of the next row (e.g., ECE X: first row begins with 0, next row begins with 2, therefore jump = 2)
# These indices assume no leading 'X/N', which is OK because the leading label is dropped in the loops below.
# firsthr = the first fcst of this element in the array is fcst at this many hours from the model cycle (e.g., ECE 00z X is 24, ECE 00z N is 36, MEX 12z X is 12 (even though the entry is blank), etc.)
# xstep = number of hours to increment to generate x-axis labels
#
# For the MAV, jump has 2 values: what's needed to get to first index of the next line and
# what's needed to get to the first index of the line after that. MAV is special because
# successive lines sometimes need the same index.
#
# MET, MEX 00z -> first fcst is X of that date
# off-cycle (00z) N plot is the same as the previous (12z) cycle's N plus a special first line
# MET, MEX 12z -> first fcst is N of that night (next date in Z)
# off-cycle (12z) X plot is the same as the previous (00z) cycle's X plus a special first line
# MAV 00z, 06z -> first fcst is X of that date
# MAV 12z, 18z -> first fcst is N of that date (next date in Z)
#
# Handle off-cycle X/N plots as follows:
# display array size is [1,#] where # is the correct number of columns.
# start = index of the first non-nan entry of that first special line
# stop = index of the last element of that first special line (+/-1 for Python slicing)
# step = count by this many to go from start to stop for that first special line
# jump = nan
dictSize = { # ~ahem~
'ECMX MOS GUIDANCE 0000 UTC':{ #ECE 00z
'X':{'size':[8,8], 'start':0, 'stop':15, 'step':2, 'jump':2, 'firsthr':24, 'xstep': 24},
'N':{'size':[7,7], 'start':1, 'stop':14, 'step':2, 'jump':2, 'firsthr':36, 'xstep': 24},
'P12':{'size':[8,15], 'start':0, 'stop':15, 'step':1, 'jump':2, 'firsthr':24, 'xstep': 12}
},
'ECM MOS GUIDANCE 0000 UTC':{ #ECS 00z
'X':{'size':[3,3], 'start':0, 'stop':5, 'step':2, 'jump':2, 'firsthr':24, 'xstep': 24},
'N':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'P12':{'size':[3,5], 'start':0, 'stop':5, 'step':1, 'jump':2, 'firsthr':24, 'xstep': 12}
},
'GFS MOS GUIDANCE 0000 UTC':{ #MAV 00z
'X':{'size':[9,3], 'start':0, 'stop':5, 'step':2, 'jump':[1,0], 'firsthr':24, 'xstep': 24},
'N':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'P12':{'size':[9,5], 'start':0, 'stop':5, 'step':1, 'jump':[1,0], 'firsthr':24, 'xstep': 12},
'WSP':{'size':[10,19], 'start':0, 'stop':19, 'step':1, 'jump':2, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[9,5], 'start':0, 'stop':5, 'step':1, 'jump':[1,0], 'firsthr':24, 'xstep': 12},
},
'GFS MOS GUIDANCE 0600 UTC':{ #MAV 06z
'X':{'size':[10,3], 'start':0, 'stop':5, 'step':2, 'jump':[0,1], 'firsthr':18, 'xstep': 24},
'N':{'size': [1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':6, 'xstep': 24},
'P12':{'size':[10,5], 'start':0, 'stop':5, 'step':1, 'jump':[0,1], 'firsthr':18, 'xstep': 12},
'WSP':{'size':[10,19], 'start':0, 'stop':19, 'step':1, 'jump':2, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[10,5], 'start':0, 'stop':5, 'step':1, 'jump':[0,1], 'firsthr':18, 'xstep': 12},
},
'GFS MOS GUIDANCE 1200 UTC':{ #MAV 12z
'X':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'N':{'size':[9,3], 'start':0, 'stop':5, 'step':2, 'jump':[1,0], 'firsthr':24, 'xstep': 24},
'P12':{'size':[9,5], 'start':0, 'stop':5, 'step':1, 'jump':[1,0], 'firsthr':24, 'xstep': 12},
'WSP':{'size':[10,19], 'start':0, 'stop':19, 'step':1, 'jump':2, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[9,5], 'start':0, 'stop':5, 'step':1, 'jump':[1,0], 'firsthr':24, 'xstep': 12},
},
'GFS MOS GUIDANCE 1800 UTC':{ #MAV 18z
'X':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':6, 'xstep': 24},
'N':{'size':[10,3], 'start':0, 'stop':5, 'step':2, 'jump':[0,1], 'firsthr':18, 'xstep': 24},
'P12':{'size':[10,5], 'start':0, 'stop':5, 'step':1, 'jump':[0,1], 'firsthr':18, 'xstep': 12},
'WSP':{'size':[10,19], 'start':0, 'stop':19, 'step':1, 'jump':2, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[10,5], 'start':0, 'stop':5, 'step':1, 'jump':[0,1], 'firsthr':18, 'xstep': 12},
},
'NAM MOS GUIDANCE 0000 UTC':{ #MET 00z
'X':{'size':[5,3], 'start':0, 'stop':5, 'step':2, 'jump':1, 'firsthr':24, 'xstep': 24},
'N':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'P12':{'size':[5,5], 'start':0, 'stop':5, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'WSP':{'size':[5,19], 'start':0, 'stop':19, 'step':1, 'jump':4, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[5,5], 'start':0, 'stop':5, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
},
'NAM MOS GUIDANCE 1200 UTC':{ #MET 12z
'X':{'size':[1,3], 'start':1, 'stop':4, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'N':{'size':[5,3], 'start':0, 'stop':5, 'step':2, 'jump':1, 'firsthr':24, 'xstep': 24},
'P12':{'size':[5,5], 'start':0, 'stop':5, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'WSP':{'size':[5,19], 'start':0, 'stop':19, 'step':1, 'jump':4, 'firsthr':6, 'xstep': 3},
'Q12':{'size':[5,5], 'start':0, 'stop':5, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
},
'GFSX MOS GUIDANCE 0000 UTC':{ #MEX 00z
'X':{'size':[15,8], 'start':0, 'stop':15, 'step':2, 'jump':1, 'firsthr':24, 'xstep': 24},
'N':{'size':[1,8], 'start':1, 'stop':14, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'P12':{'size':[15,15], 'start':0, 'stop':15, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'WSP':{'size':[15,15], 'start':0, 'stop':15, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'Q12':{'size':[12,12], 'start':0, 'stop':12, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12}
},
'GFSX MOS GUIDANCE 1200 UTC':{ #MEX 12z
'X':{'size':[1,8], 'start':1, 'stop':14, 'step':2, 'jump':np.nan, 'firsthr':12, 'xstep': 24},
'N':{'size':[15,8], 'start':0, 'stop':15, 'step':2, 'jump':1, 'firsthr':24, 'xstep': 24},
'P12':{'size':[15,15], 'start':0, 'stop':15, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'WSP':{'size':[15,15], 'start':0, 'stop':15, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12},
'Q12':{'size':[12,12], 'start':0, 'stop':12, 'step':1, 'jump':1, 'firsthr':24, 'xstep': 12}
}
}
dispArr = {} #init it here, add to it later in the loop below
dtXaxis = {} #ditto
# Recall that wxelements and alldata were defined above to handle missing cases.
for wx in wxelements:
if dictSize[modelKey][wx]['size'][0] == 1:
# Handle the special case of an off-cycle X or N.
#
# As usual, it turns out that this is more complicated than it looks at first.
# PITA cases: ECS 00z N, MAV 06z N, MAV 18z X. Here's what happens:
# 00z ECS N -> look back at the previous run (00z prev day) to get the size of
# the array. Since the 00z prev day has the same dictSize entry as 00z current run,
# it also has size [1,#]. The array is too short.
# 06z MAV N -> look back 6 hrs to the 00z run's N. The size of that array is also [1,#]. The
# array is too short.
# 18z MAV X -> look back 6 hours to the 12z run's X. Same thing happens.
# The solution is to first check dictSize[modelKey][wx]['size'][0] == 1. If so,
# enter a bounded loop (a while loop will cause an infinite loop for the ECS) limited
# to the number of columns given by dictSize[modelKey][wx]['size'][1] since there can't be more
# special leading lines than cols. Within that loop, calculate prevKey and append it to a
# storage list. Break the loop once dictSize[modelKey][wx]['size'][0] != 1. Now we have a list,
# possibly of length 1, containing a list of prevKeys for which to calculate special lines. In
# fact, go ahead and calculate the speciallines within the loop and append to another storage
# list. Once the loop ends, the value of prevKey is the key to use to construct the bulk of
# the array. At least, that's the hope.
thisrun = thisrun_as_dt(infoDict)
storePrevKeys = []
storeSpecialLines = []
origKey = modelKey #save for later to restore, otherwise can't do more than 1 graph at a time
for c in range(dictSize[origKey][wx]['size'][1]):
if dictSize[modelKey][wx]['size'][0] == 1:
if 'ECM' in infoDict['MOSTYPE']:
backtrack = 24 #ECS
elif ('GFS' in infoDict['MOSTYPE'] and 'GFSX' not in infoDict['MOSTYPE']):
backtrack = 6 #MAV
else:
backtrack = 12 #MEX, MET
prevrun = thisrun - dt.timedelta(hours = backtrack)
prevRUNTIME = prevrun.strftime('%H%M UTC')
prevKey = '%s %s' % (infoDict['MOSTYPE'], prevRUNTIME)
storePrevKeys.append(prevKey)
firstline = alldata[wx][c][1:] #toss leading 'X/N', 'P12', etc.
inds = np.arange(1, dictSize[modelKey][wx]['stop'], dictSize[modelKey][wx]['step'])
specialline = np.insert(firstline[inds], [0], np.nan)
storeSpecialLines.append(specialline)
thisrun = prevrun #on the next iteration, go back farther
modelKey = prevKey #on the next iteration, go back farther
else:
break
# 'varResult' is X, N, P12, etc. to return. 'varData' is the working array (allxn, allp12, etc.)
varResult = np.empty(dictSize[modelKey][wx]['size']) * np.nan
# toss the first line(s) from allxn since it's associated with origKey, not prevKey
varData = alldata[wx][c:][:]
else:
origKey = modelKey
varResult = np.empty(dictSize[modelKey][wx]['size']) * np.nan
varData = alldata[wx]
# Secure in the knowledge that an off-cycle case has been handled (if it exists),
# proceed with creating the display array. Note that if it is an off-cycle case,
# the array created below is for the previous cycle, and the special line(s) will be added
# back in afterwards.
flag = 0
jumpindex = dictSize[modelKey][wx]['start']
for row in range(0, len(varResult)):
validline = varData[row][1:] #toss leading 'X/N' or 'N/X'
inds = np.arange(jumpindex, dictSize[modelKey][wx]['stop'], dictSize[modelKey][wx]['step'])
if validline != []:
varResult[row, 0:len(inds)] = validline[inds]
# dictSize[modelKey][wx]['jump'] is a single number unless MOSTYPE is the MAV.
# For the MAV, need to alternate between the two jump indices.
if (np.size(dictSize[modelKey][wx]['jump']) > 1) and (flag == 0):
jumpindex = jumpindex + dictSize[modelKey][wx]['jump'][0]
flag = 1
elif flag == 1:
jumpindex = jumpindex + dictSize[modelKey][wx]['jump'][1]
flag = 0
else:
jumpindex = jumpindex + dictSize[modelKey][wx]['jump']
# Create the fcst valid date/time entries for the x-axis labels.
# The x-axis labels increase differently depending on the wx element
# and the MOS type.
thisrun = thisrun_as_dt(infoDict)
tempdt = []
#if wx in ['P12', 'WSP']:
# fcsthrs = range(dictSize[origKey][wx]['firsthr'], 216, 12)
#else:
# fcsthrs = range(dictSize[origKey][wx]['firsthr'], 216, 24)
fcsthrs = range(dictSize[origKey][wx]['firsthr'], 216, dictSize[origKey][wx]['xstep'])
for hr in fcsthrs:
future = dt.timedelta(hours = hr)
tempdt.append(thisrun + future)
dtXaxis[wx] = tempdt
# For off-cycle plots, add the special line back in.
if dictSize[origKey][wx]['size'][0] == 1:
# insert the special line(s) defined above
storeSpecialLines.reverse()
for c in range(0, len(storeSpecialLines)):
varResult = np.insert(varResult, [0], storeSpecialLines[c], axis = 0)
# Guess what. ECS N plot fails miserably because of the chosen data structure (size
# [1,#] and the off-cycle run is itself). Out of 10 test cases x 3 plots each =
# 30 plots, only 1 doesn't work. Sigh. Get around this by explicitly defining
# each element of varResult for ECS N, which is a 3x3 array.
if 'ECM' in origKey:
varResult = np.array([
[np.nan, alldata[wx][0][2], alldata[wx][0][4]],
[alldata[wx][1][2], alldata[wx][1][4], np.nan],
[alldata[wx][2][5], np.nan, np.nan]
], dtype = 'f')
# ...and all is as it was, for the next value of wx in the loop
varData = alldata[wx]
modelKey = origKey
dispArr[wx] = varResult
return dispArr, dtXaxis, infoDict, prevruns