def _get(self):
if self._secret:
raise NotImplementedError(
"No auth protocols currently supported".format(
self._auth_protocol))
else:
if self._key:
params[self._key_param] = self._key
url = self._form_url()
req = urllib.request.Request(url)
try:
resp = urllib.request.urlopen(req, None, self._request_timeout)
except urllib.request.HTTPError as e:
# TODO: should we do this check? any other codes to
# raise BlueSkyUnavailableResourceError for?
if e.getcode() == 404:
raise BlueSkyUnavailableResourceError(
"Unavailable resource for loading: {}".format(url))
raise
# else if URLError (e.g. hostname doesn't exist -
# 'nodename nor servname provided, or not known'), then
# there's no use in retrying
# TODO: handleany other errors?
# - CertificateError (e.g. "hostname 'www.googlesds.com' doesn't match either of '*.dadapro.com', 'dadapro.com'"")
# - other???
body = resp.read().decode('ascii')
return body
def _find():
files = []
for time_window in time_windows:
logging.debug("Findmetdata time window: %s to %s",
time_window['start'], time_window['end'])
files.extend(
met_finder.find(time_window['start'],
time_window['end']).get('files', []))
if not files:
raise BlueSkyUnavailableResourceError("No met files found")
# It's possible that the following filtering by forecast whitelist
# will result in waiting on a forecast that
# already exists, but was passed over by arlfinder.ArlFinder because
# there were newer data. To avoid that situation, we'd need to
# move the forecast whitelist logic into arlfinder.ArlFinder.
# The practical use case for this forecast whitelist logic, however,
# is to wait for the latest applicable met data for the dispersion
# window (e.g. to wait for 2019-07-31 OOZ met data for the
# run starting at 2019-07-31T00:00:00), which should never result
# in waiting for already existing data.
# TODO: move whitelist logic into findmetdata package. support three modes:
# - any met: e.g. allow 7/27 met for 7/28 run if 7/28 met isn't available
# - want only met thats newer than spcified data; e.g. for 7/28 run,
# don't allow use of 7/27 data, but allow met after 7/28 for
# hours in dispersion that are after 7/28 (which can happen if re-running
# past data)
# - want specificc date for run; e.g. for 7/28 run, only allow 7/28 met,
# even for hours after 7/28 in dispersion window
files = _filter_forecasts(accepted_forecasts_config, files)
return files
def __init__(self, **config):
super(BaseFileLoader, self).__init__(**config)
self._filename = config.get('file')
if not self._filename:
raise BlueSkyConfigurationError("Fires file to load not specified")
if not os.path.isfile(self._filename):
raise BlueSkyUnavailableResourceError(
"Fires file to "
"load {} does not exist".format(self._filename))
self._saved_copy_filename = config.get('saved_copy_file')
self._events_filename = None
if config.get('events_file'):
self._events_filename = config['events_file']
if not os.path.isfile(self._events_filename):
raise BlueSkyUnavailableResourceError(
"Fire events file to load {} "
"does not exist".format(self._events_filename))
self._saved_copy_events_filename = config.get('saved_copy_events_file')
def _find():
files = []
for time_window in time_windows:
logging.debug("Findmetdata time window: %s to %s",
time_window['start'], time_window['end'])
files.extend(
met_finder.find(time_window['start'],
time_window['end']).get('files', []))
if not files:
raise BlueSkyUnavailableResourceError("No met files found")
return files
def _filter_forecasts(config, files):
if not config:
return files
filtered_files = []
for init_time in config['init_times']:
to_match = init_time.strftime(config['met_file_name_pattern'])
filtered_files.extend(
[f for f in files if f['file'].find(to_match) > -1])
if not filtered_files:
raise BlueSkyUnavailableResourceError(
"Only accepting forecasts: {}".format(', '.join(
[i.strftime("%Y-%m-%d %HZ") for i in config['init_times']])))
return filtered_files
def get(self, **query):
saved_data_filename = (self._config.get('saved_data_file')
or self._config.get('saved_copy_file'))
if self._secret:
if self._auth_protocol == 'afweb':
url = self._form_url(**query)
url = auth.sign_url(url, self._key, self._secret)
else:
raise NotImplementedError(
"{} auth protocol not supported".format(
self._auth_protocol))
else:
if self._key:
params[self._key_param] = self._key
url = self._form_url(**query)
req = urllib.request.Request(url)
try:
resp = urllib.request.urlopen(req, None, self._request_timeout)
except urllib.request.HTTPError as e:
# TODO: should we do this check? any other codes to
# raise BlueSkyUnavailableResourceError for?
if e.getcode() == 404:
raise BlueSkyUnavailableResourceError(
"Unavailable resource for loading: {}".format(url))
raise
# else if URLError (e.g. hostname doesn't exist -
# 'nodename nor servname provided, or not known'), then
# there's no use in retrying
# TODO: handleany other errors?
# - CertificateError (e.g. "hostname 'www.googlesds.com' doesn't match either of '*.dadapro.com', 'dadapro.com'"")
# - other???
body = resp.read().decode('ascii')
self._write_data(saved_data_filename, body)
return body
def bskur_error():
self.counter += 1
if self.counter <= 2:
raise BlueSkyUnavailableResourceError()
def bskur_error():
raise BlueSkyUnavailableResourceError()