def _get(self): if self._secret: raise NotImplementedError( "No auth protocols currently supported".format( self._auth_protocol)) else: if self._key: params[self._key_param] = self._key url = self._form_url() req = urllib.request.Request(url) try: resp = urllib.request.urlopen(req, None, self._request_timeout) except urllib.request.HTTPError as e: # TODO: should we do this check? any other codes to # raise BlueSkyUnavailableResourceError for? if e.getcode() == 404: raise BlueSkyUnavailableResourceError( "Unavailable resource for loading: {}".format(url)) raise # else if URLError (e.g. hostname doesn't exist - # 'nodename nor servname provided, or not known'), then # there's no use in retrying # TODO: handleany other errors? # - CertificateError (e.g. "hostname 'www.googlesds.com' doesn't match either of '*.dadapro.com', 'dadapro.com'"") # - other??? body = resp.read().decode('ascii') return body
def _find(): files = [] for time_window in time_windows: logging.debug("Findmetdata time window: %s to %s", time_window['start'], time_window['end']) files.extend( met_finder.find(time_window['start'], time_window['end']).get('files', [])) if not files: raise BlueSkyUnavailableResourceError("No met files found") # It's possible that the following filtering by forecast whitelist # will result in waiting on a forecast that # already exists, but was passed over by arlfinder.ArlFinder because # there were newer data. To avoid that situation, we'd need to # move the forecast whitelist logic into arlfinder.ArlFinder. # The practical use case for this forecast whitelist logic, however, # is to wait for the latest applicable met data for the dispersion # window (e.g. to wait for 2019-07-31 OOZ met data for the # run starting at 2019-07-31T00:00:00), which should never result # in waiting for already existing data. # TODO: move whitelist logic into findmetdata package. support three modes: # - any met: e.g. allow 7/27 met for 7/28 run if 7/28 met isn't available # - want only met thats newer than spcified data; e.g. for 7/28 run, # don't allow use of 7/27 data, but allow met after 7/28 for # hours in dispersion that are after 7/28 (which can happen if re-running # past data) # - want specificc date for run; e.g. for 7/28 run, only allow 7/28 met, # even for hours after 7/28 in dispersion window files = _filter_forecasts(accepted_forecasts_config, files) return files
def __init__(self, **config): super(BaseFileLoader, self).__init__(**config) self._filename = config.get('file') if not self._filename: raise BlueSkyConfigurationError("Fires file to load not specified") if not os.path.isfile(self._filename): raise BlueSkyUnavailableResourceError( "Fires file to " "load {} does not exist".format(self._filename)) self._saved_copy_filename = config.get('saved_copy_file') self._events_filename = None if config.get('events_file'): self._events_filename = config['events_file'] if not os.path.isfile(self._events_filename): raise BlueSkyUnavailableResourceError( "Fire events file to load {} " "does not exist".format(self._events_filename)) self._saved_copy_events_filename = config.get('saved_copy_events_file')
def _find(): files = [] for time_window in time_windows: logging.debug("Findmetdata time window: %s to %s", time_window['start'], time_window['end']) files.extend( met_finder.find(time_window['start'], time_window['end']).get('files', [])) if not files: raise BlueSkyUnavailableResourceError("No met files found") return files
def _filter_forecasts(config, files): if not config: return files filtered_files = [] for init_time in config['init_times']: to_match = init_time.strftime(config['met_file_name_pattern']) filtered_files.extend( [f for f in files if f['file'].find(to_match) > -1]) if not filtered_files: raise BlueSkyUnavailableResourceError( "Only accepting forecasts: {}".format(', '.join( [i.strftime("%Y-%m-%d %HZ") for i in config['init_times']]))) return filtered_files
def get(self, **query): saved_data_filename = (self._config.get('saved_data_file') or self._config.get('saved_copy_file')) if self._secret: if self._auth_protocol == 'afweb': url = self._form_url(**query) url = auth.sign_url(url, self._key, self._secret) else: raise NotImplementedError( "{} auth protocol not supported".format( self._auth_protocol)) else: if self._key: params[self._key_param] = self._key url = self._form_url(**query) req = urllib.request.Request(url) try: resp = urllib.request.urlopen(req, None, self._request_timeout) except urllib.request.HTTPError as e: # TODO: should we do this check? any other codes to # raise BlueSkyUnavailableResourceError for? if e.getcode() == 404: raise BlueSkyUnavailableResourceError( "Unavailable resource for loading: {}".format(url)) raise # else if URLError (e.g. hostname doesn't exist - # 'nodename nor servname provided, or not known'), then # there's no use in retrying # TODO: handleany other errors? # - CertificateError (e.g. "hostname 'www.googlesds.com' doesn't match either of '*.dadapro.com', 'dadapro.com'"") # - other??? body = resp.read().decode('ascii') self._write_data(saved_data_filename, body) return body
def bskur_error(): self.counter += 1 if self.counter <= 2: raise BlueSkyUnavailableResourceError()
def bskur_error(): raise BlueSkyUnavailableResourceError()