def retrieve(self, app):
""" Retrieve data for requested RAOB type """
if (self.request.get_type() == 'TEXT:LIST'):
textlist = RAOBtextlist(self.log)
(status, outfile) = textlist.retrieve(app, self.request, self.log)
# If in GUI mode and successfully downloaded a text file, create a
# skewT and display it in the GUI
if status and (app is not None):
self.widget.createSkewt(outfile)
app.processEvents()
elif (self.request.get_type() == 'GIF:SKEWT'):
gifskewt = RAOBgifskewt(self.log)
(status, outfile) = gifskewt.retrieve(app, self.request, self.log)
gifskewt.cleanup()
# If in GUI mode and successfully downloaded a gif image, display
# it in the GUI
if status and (app is not None):
self.widget.setImage(outfile)
app.processEvents()
else:
printmsg(self.log, "RAOB type '" + self.request.get_type() +
"' not implemented yet")
status = False
return(status)
def get_ftp_server(self):
if 'ftp_server' in self.projConfig.keys():
return (self.projConfig['ftp_server'])
else:
printmsg(
self.log, "ERROR: ftp server not defined. " +
"Add 'ftp_server: path' to config file and rerun.")
return (None)
def get_ftp_dir(self):
if 'ftp_dir' in self.projConfig.keys():
return (self.projConfig['ftp_dir'])
else:
printmsg(
self.log, "ERROR: Directory to ftp files to not " +
"defined. Add 'ftp_dir: path' to config file and rerun.")
return (None)
def get_ftp_status(self):
if 'ftp' in self.projConfig.keys():
if str(self.projConfig['ftp']).lower() == "false":
return (False)
if str(self.projConfig['ftp']).lower() == "true":
return (True)
else:
printmsg(
self.log, "ERROR: ftp status not defined. " +
"Add 'ftp: True/False' to config file and rerun.")
return (None)
def read(self, station_list_file):
"""
Read in stations from a master station list.
Return a station list array of dictionary items, one for each stn.
"""
if station_list_file == "":
error = "ERROR: Must select a master\n" + \
"station list to choose\n" + \
"stations from. Close this\n" + \
"window and click\n" + \
"'Create station list' again"
printmsg(self.log, error)
return (error)
else:
# Make sure station_list_file exists
if not os.path.isfile(station_list_file):
error = "ERROR: station list file " + station_list_file + \
" doesn't exist"
printmsg(self.log, error)
return (error)
# Make sure the RAOBstation dictionary and generated station_list
# array are empty
self.station.clear()
self.station_list.clear()
# Open the station list file and read the contents into the
# dictionary
infile = open(station_list_file)
for line in infile:
# Skip HTML/XML
if (line[0] != '<'):
# Parse line and put into dictionary
self.station['id'] = line[0:8]
self.station['number'] = line[10:15]
self.station['description'] = line[16:46]
self.station['state'] = line[49:51]
self.station['country'] = line[52:54]
self.station['lat'] = line[55:60]
self.station['lon'] = line[61:67]
self.station['elev'] = line[68:73]
# Copy dictionary into array (so get a copy, not a pointer)
self.station_list.append(self.station.copy())
# printmsg(self.log, line.rstrip().split())
infile.close()
return (self.station_list)
def set(self):
""" Set the station to the value entered in the input field """
textboxValue = self.stnm.text()
status = self.request.set_stnm(textboxValue)
if not status:
printmsg(self.log, "ERROR: Requested station not valid")
else:
logging.info("Station set to " + textboxValue)
# Since just manually set a station, need to set rsl to '' or
# any defined rsl file will override user station selection
self.request.set_rsl('')
self.reset_loadRSLlabel()
def write(self, request):
""" Write the request metadata to a YAML config file"""
yamlfile = os.path.abspath(request.get_config())
if (request.get_config() == ""):
printmsg(self.log, "WARNING: File not saved")
return (False)
with open(yamlfile, 'w') as outfile:
yaml.dump(request.get_dict(), outfile)
printmsg(self.log, "Successfully saved config to " + yamlfile)
def set_outfile(request, log):
""" Create an output filename, including the full path """
dir = os.path.join(os.getcwd(), request.get_mtp_dir())
# Make sure directory exists. If not, warn user.
if not os.path.exists(dir):
printmsg(
log, "ERROR: Directory to write MTP data does not exist: " + dir +
". Create dir and retry retrieval")
return (False)
# The MTP VB6 code requires that only the begin date be given in the RAOB
# filename.
return (dir + '/' + request.get_stnm() + request.get_year() +
request.get_month() + request.get_begin() + ".txt")
def update_displayed_config(self):
"""
Update the displayed selections in the configedit portion of the GUI
"""
newconfig = self.widget.configGUI()
# Update mode
if self.raob.request.get_mtp() is True:
newconfig.updateMode("MTP")
if self.raob.request.get_catalog() is True:
newconfig.updateMode("CATALOG")
if self.raob.request.get_mtp() is True and \
self.raob.request.get_catalog() is True:
printmsg(self.log, "ERROR: Both mtp and catalog set to true" +
" in config file. Setting to default mode.")
self.raob.request.set_mtp(False)
self.raob.request.set_catalog(False)
newconfig.updateMode("Default")
# update Freq (hours)
newconfig.updateFreq(self.raob.request.get_freq())
# update Type of RAOB data to download
newconfig.updateType(self.raob.request.get_type())
# update displayed station number / id
newconfig.updateStnm(self.raob.request.get_stnm())
# update displayed begin time
newconfig.updateBtime(self.raob.request.get_year() +
self.raob.request.get_month() +
self.raob.request.get_begin())
# update displayed end time
newconfig.updateEtime(self.raob.request.get_year() +
self.raob.request.get_month() +
self.raob.request.get_end())
# update displayed RSL file
if self.raob.request.get_rsl() != '':
newconfig.updateRSL(self.raob.request.get_rsl())
# update displayed time if now
if self.raob.request.get_now() is True:
newconfig.updatenow("Time set to now")
def get_prod(self, request):
""" Build the product name required by the field catalog """
# Open the HTML file and read in the <TITLE> line
out = open(self.get_outfile_html())
line = out.readline()
while '<TITLE>' not in line:
line = out.readline()
if '<TITLE>' in line:
m = re.search(request.get_stnm() + ' (.*) [Sounding]', line)
if m:
prod = m.group(1).replace(" ", "_")
# Get station metadata for use in reformatting product name
station = self.get_station_info(request)
# If we found the station by the number, then the id will still
# be in the title. Remove it.
if station['id'].rstrip() in prod:
prod = prod.replace(station['id'].rstrip() + "_", "")
# I have seen some products with protected shell characters.
# Remove them here.
prod = prod.replace("(", "") # Remove open parenthesis
prod = prod.replace(")", "") # Remove close parenthesis
# ... add more as needed here ...
# For international skewts, set the product name to
# "Station_Name_CC" where CC is the two letter country code.
# For US stations use "Station_Name_ST" where ST it the two
# letter state code.
if (station['country'] == 'US'):
prod += "_" + station['state']
else:
prod += "_" + station['country']
else:
printmsg(
self.log, "WARNING: Couldn't find product name. " +
"Setting to temp")
prod = "temp"
out.close()
return (prod)
def loadRSL(self):
""" Call dialog box to load a station list """
# Call dialog box to select a raob station list (rsl) file
# Once selected, code will loop through contents
self.loader = FileSelector("loadRsl")
self.request.set_rsl(self.loader.get_file())
if (self.request.get_rsl() == ""):
printmsg(self.log, "WARNING: Station list not loaded. Please " +
"select a station/station list.")
else:
self.load.setText(self.request.get_rsl())
logging.info("RAOB station list set to " +
self.request.get_rsl())
# When load an RSL file, set stnm to empty in request and reset display
# text to empty.
self.request.set_stnm('')
self.stnm.setText('')
def set(self):
""" Set the beginning of the time range to download """
# Now that user has entered text, set the color of the text to black
self.btime.setStyleSheet("color: black")
textboxvalue = self.btime.text()
# Validate entered data
time = re.compile(r'[12][0-9][0-9][0-9][0-1][0-9][0-3][0-9][0-9][0-9]')
if not time.match(textboxvalue):
printmsg(
self.log, "ERROR in entered begin time: " + textboxvalue +
". Entered time must be a 4-digit year followed by a " +
"2-digit month, a 2-digit day, and a 2-digit hour, e.g." +
"2019051012 for noon May 10th, 2019")
printmsg(
self.log,
"Begin date/time not set. Please reenter and " + "click 'Set'")
# Data not valid, so return False to indicate setting value failed
self.status = False
else:
# Parse entered date into year, month, day, hr and assign to
# request metadata
year = textboxvalue[0:4]
self.request.set_year(year)
month = textboxvalue[4:6]
self.request.set_month(month)
day = textboxvalue[6:8]
hr = textboxvalue[8:10]
self.request.set_begin(day, hr)
logging.info("year set to " + year + ", month set to " + month +
", begin (ddhh) set to " + day + hr)
# Since sucessfully set a begin date, ensure that now flag is false
self.request.set_now(False)
# Data is valid, so return True to indicate setting value succeeded
self.status = True
def get_station_info(self, request):
""" Read in the station metadata for the given station id/number """
station_list_file = getrootdir() + "/" + request.get_stnlist_file()
stationList = RAOBstation_list(self.log)
stationList.read(station_list_file)
if request.get_stnm().isdigit():
station = stationList.get_by_stnm(request.get_stnm())
else:
station = stationList.get_by_id(request.get_stnm())
# Should only get back one, unique station. If not, warn user
if (len(station) != 1):
printmsg(
self.log, "WARNING: Found more than one station " +
"matching request['stnm']")
for i in range(len(station)):
printmsg(self.log, str(station[i]))
printmsg(self.log, "Returning first station found")
return (station[0])
def get(self, widget, app, log=""):
""" Method to retrieve RAOBS
Parameters:
app: if in GUI mode, a pointer to the QApplication, else
app is None
Returns:
N/A
"""
self.log = log # pointer to GUI log, if running in GUI mode
self.widget = widget
# If args are all empty, warn user.
empty = True
request = self.request.get_request()
for key in request.keys():
if key != 'station_list_file' and key != 'mtpdir':
if str(request[key]).lower() == 'true':
empty = False
elif str(request[key]).lower() != 'false' and \
request[key] != '':
empty = False
if empty:
printmsg(log, "ERROR: Request is empty. Set some metadata or" +
" load a config file and rerun.")
return()
# If option --now is set, set year, month, begin, and end to current
# date/time
if self.request.get_now() is True:
self.request.set_time_now()
# If user has requested more than one RAOB, loop over the requested
# frequency
printmsg(log, "Getting RAOBs from: '" + self.request.get_year() +
self.request.get_month() + self.request.get_begin() +
"' to '" + self.request.get_year() +
self.request.get_month() + self.request.get_end() + "'")
if self.request.get_begin() == self.request.get_end():
# printmsg(log, "Single Day:" + self.request.get_begin() +
# ' - ' + self.request.get_end())
self.stn_loop(app)
else:
if self.request.get_end() < self.request.get_begin():
printmsg(log, "ERROR: Requested end time must be >= " +
"requested begin time")
else:
# Get RAOBs for first day requested
begin = self.request.get_begin()
bday = begin[0:2]
bhr = begin[2:4]
end = self.request.get_end()
eday = end[0:2]
ehr = end[2:4]
for hr in range(int(bhr), 24,
int(self.request.get_freq())):
# get RAOBs
self.request.set_begin(bday, '{:02d}'.format(hr))
self.request.set_end(bday, '{:02d}'.format(hr))
# printmsg(log, "Day 1:" + self.request.get_begin() +
# ' - ' + self.request.get_end())
self.stn_loop(app)
# Get RAOBs for second to second-to-last day
for day in range(int(bday) + 1, int(eday)):
for hr in range(0, 24, int(self.request.get_freq())):
# get RAOBs
self.request.set_begin('{:02d}'.format(day),
'{:02d}'.format(hr))
self.request.set_end('{:02d}'.format(day),
'{:02d}'.format(hr))
# printmsg(log, self.request.get_begin() + ' - ' +
# self.request.get_end())
self.stn_loop(app)
# Get RAOBs for last day requested
for hr in range(0, int(ehr)+1,
int(self.request.get_freq())):
# get RAOBs
self.request.set_begin(eday, '{:02d}'.format(hr))
self.request.set_end(eday, '{:02d}'.format(hr))
# printmsg(log, "Last:" + self.request.get_begin() +
# ' - ' + self.request.get_end())
self.stn_loop(app)
def stn_loop(self, app):
if self.request.get_mtp() is True and \
self.request.get_type() == "GIF:SKEWT":
printmsg(self.log, 'ERROR: Requested GIF:SKEWT plots in ' +
'MTP mode. Check configuration.')
return()
# If TEXT:LIST, change the image to a canvas
if (app is not None) and (self.request.get_type() == 'TEXT:LIST'):
self.widget.resetImageWindow()
# Did user request a single station via --stnm, or a list of stations
# via an RSL file
if (self.request.get_rsl() == ''):
# Stnm already set
# Retrieve requested data/imagery for a single stn
self.retrieve(app)
else:
rslfile = os.path.join(os.getcwd(), self.request.get_rsl())
if self.test_rsl(rslfile):
rsl = RSL()
stnlist = rsl.read_rsl(rslfile)
# If use requests more than 50 stns, break them up so don't
# overwhelm UWyo server.
if len(stnlist) > 30:
count = 0
for stn in stnlist: # Loop through a list of stations
status = self.request.set_stnm(stn)
if not status:
printmsg(self.log, "WARNING: Requested station "
+ stn + " not valid. Update RSL station" +
" list. Skipping and continuing...")
continue
status = self.retrieve(app)
if not status:
printmsg(self.log, "Skipping station and continuing")
if len(stnlist) > 30:
count = count+1
if count % 10 == 0:
printmsg(self.log, 'Sleeping for 30 seconds to ' +
'avoid overwhelming UWyo server')
time.sleep(30)
printmsg(self.log, "Done retrieving RAOBs from: '" +
self.request.get_year() + self.request.get_month() +
self.request.get_begin() + "' to '" +
self.request.get_year() + self.request.get_month() +
self.request.get_end() + "'")
else:
printmsg(self.log, 'ERROR: File ' + rslfile +
' does not exist. Check for typo and rerun.')
def get_data(self, url, outfile):
"""
Send the generated URL to the uwyo website and receive back a file
containing the requested data or imagery.
Parameters:
url: the url containing the request
outfile: the name of the file to which the received data should be
saved
Returns:
boolean: True/False success indicator
"""
# Check if filename already exists. wget will fail if it does.
if os.path.isfile(outfile):
printmsg(self.log, "Already downloaded file with name " + outfile)
return (False) # Did not download new data
else:
# Check if online - if not, exit gracefully
try:
urllib.request.urlopen(url)
except (HTTPError, URLError) as e:
# Get reference to existing QApplication
app = QApplication.instance()
msg = "Can't connect to weather.uwyo.edu. Received error:" + \
"\n" + str(e) + "Use option --test for testing with " + \
"offline sample data files or confirm that you are " + \
"online and try again in a few minutes.\n\n Unable " + \
"to download " + outfile + "\n\n" + \
" When you click OK, code will try to retrieve next RAOB"
if app is None:
print(self.log, msg)
else:
quitButton = QAction('Quit')
quitButton.setShortcut('Ctrl+Q')
quitButton.setToolTip('Exit application')
# quitButton.triggered.connect(QMessageBox.close)
msgBox = QMessageBox()
msgBox.setText(msg)
msgBox.setIcon(QMessageBox.Information)
msgBox.addButton(quitButton, QMessageBox.close)
msgBox.exec()
return (False)
except socket.timeout as e:
printmsg(self.log, "There was an error:")
printmsg(self.log, str(e))
except Exception as e:
printmsg(self.log, "Unknown error connecting to UWyo: " + e)
# Get requested URL.
try:
urllib.request.urlretrieve(url, outfile)
except (HTTPError, URLError) as e:
printmsg(
self.log,
"Error downloading file " + outfile + " Error: " + str(e))
return (False)
except socket.timeout as e:
printmsg(self.log, "There was an error:")
printmsg(self.log, str(e))
# Test if text/html file contains good data
if "gif" not in outfile:
out = open(outfile)
line = out.readline()
while line != '':
if "Can't get" in line:
printmsg(self.log,
'ERROR: Website says "' + line.rstrip() + '"')
out.close()
if os.path.isfile(outfile):
os.remove(outfile)
return (False)
elif 'Sorry, unable to generate' in line:
printmsg(
self.log,
line.rstrip() + ". Retrieved file" +
" contains error message - gif was not " +
"generated")
out.close()
if os.path.isfile(outfile):
os.remove(outfile)
return (False)
else:
line = out.readline()
out.close()
printmsg(self.log, "Retrieved " + outfile)
return (True) # Downloaded new data
def strip_html(request, outfile, log):
# Strip unneeded HTML from the retrieved data.
# The VB6 MTP sofware strips part of the HTML from the downloaded RAOB
# file. We are preserving this format here for backward compatibility
# so RAOBman VB code will still work.
out = open(outfile)
temp = open(outfile + '.temp', 'w')
# Loop over RAOBS in file
line = out.readline()
while line != '':
if (line.rstrip() == '<HTML>'): # Beginning of new RAOB
# Add double quote before <HTML> on first line
temp.write('"' + line)
else:
printmsg(
log, "ERROR: RAOB textlist file " + outfile +
" does not begin with <HTML>")
printmsg(log, line.rstrip())
return ()
# Remove <TITLE>, <LINK>, and <BODY> lines
while line[0:4] != '<H2>' and line != '':
line = out.readline()
# Search for SECOND </PRE>, keep everything before and including it
while line[0:6] != '</PRE>' and line != '':
temp.write(line)
line = out.readline()
# Print the line with the first </PRE>
temp.write(line)
line = out.readline()
# Print until, and including, the line with the second </PRE>
while line[0:6] != '</PRE>' and line != '':
# Check for exact obs time in data
if "Observation time" in line:
obstime = ''.join(filter(str.isdigit, line))
if obstime[0:8] not in outfile:
printmsg(
log, "\nObs time " + obstime + " and filename " +
outfile + " don't match. Fixing...\n")
outfile = request.get_stnm() + "20" + \
obstime[0:8] + ".txt"
printmsg(log, "New filename is " + outfile)
temp.write(line)
line = out.readline()
temp.write(line)
line = out.readline()
# Remove everything until next <HTML> or until end of file.
while line != '<HTML>' and line != '': # not new RAOB and not EOF
line = out.readline()
out.close()
temp.close()
# move temp back to outfile
os.replace(outfile + '.temp', outfile)
def retrieve(self, app, request, log=""):
"""
Retrieves the requested data from the U Wyoming archive
Parameters:
request: A dictionary containing the metadata for the
request.
Returns:
outfile: The name of the retrieved file.
"""
# SKEWT's are downloaded in two steps.
# The first request generates the skewt on the website and downloads an
# HTML wrapper with a reference to the gif image that is still on the
# website. The second request downloads the gif image.
# Create first request URL from request metadata
url = self.get_url(request)
if app is not None: # Force the GUI to redraw so log
app.processEvents() # messages, etc are displayed
# Create output filename from request metadata
self.set_outfile_html(request)
# If in test mode, copy file from data dir to simulate download...
if request.get_test() is True:
shutil.copyfile(
getrootdir() + '/test/data/7267220190528122812.html.ctrl',
'7267220190528122812.html')
status = False
# ...else download data
else:
status = self.rwget.get_data(url, self.get_outfile_html())
if app is not None: # Force the GUI to redraw so log
app.processEvents() # messages, etc are displayed
# If site returned good html file and thus generated gif
if status:
# Download the gif image directly
url = self.get_gif_url(request)
# Create output filename from request metadata
self.set_outfile_gif(request)
outfile = self.get_outfile_gif()
if app is not None: # Force the GUI to redraw so log
app.processEvents() # messages, etc are displayed
# Download gif image
gifstatus = self.rwget.get_data(url, outfile)
if app is not None: # Force the GUI to redraw so log
app.processEvents() # messages, etc are displayed
else:
gifstatus = None
if request.get_test() is True:
shutil.copyfile(
getrootdir() + '/test/data/' +
'upperair.SkewT.201905280000.Riverton_WY.gif' + '.ctrl',
'upperair.SkewT.201905280000.Riverton_WY.gif')
outfile = "upperair.SkewT.201905280000.Riverton_WY.gif"
# If running in catalog mode, ftp files to catalog dir
if request.get_catalog() is True and gifstatus:
ftpstatus = userlib.catalog.to_ftp(outfile, request, self.log)
printmsg(self.log, ftpstatus)
if app is not None: # Force the GUI to redraw so log
app.processEvents() # messages, etc are displayed
return (gifstatus, outfile)
def cleanup(self):
""" Remove now-irrelevant html file """
if os.path.isfile(self.get_outfile_html()):
os.remove(self.get_outfile_html())
printmsg(self.log, 'Removed ' + self.get_outfile_html())