def __init__(self, **config): super(BaseApiLoader, self).__init__(**config) self._endpoint = config.get('endpoint') if not self._endpoint: raise BlueSkyConfigurationError("Json API not specified") self._key = config.get('key') self._secret = config.get('secret') # you can have a key without a secret, but not vice versa if self._secret and not self._key: raise BlueSkyConfigurationError( "Api key must be specified if secret is specified") self._key_param = config.get('key_param', self.DEFAULT_KEY_PARAM) self._auth_protocol = config.get('auth_protocol', self.DEFAULT_AUTH_PROTOCOL) self._request_timeout = config.get('request_timeout', self.DEFAULT_REQUEST_TIMEOUT) self._query = config.get('query', {}) # Convert datetime.date objects to strings for k in self._query: if isinstance(self._query[k], datetime.date): self._query[k] = self._query[k].strftime(self.DATETIME_FORMAT)
def get_grid_params(met_info={}, fires=None, allow_undefined=False): # defaults to 'LatLon' in config defaults is_deg = config('projection') == 'LatLon' if config("USER_DEFINED_GRID"): # This supports BSF config settings # User settings that can override the default concentration grid info logging.info("User-defined sampling/concentration grid invoked") grid_params = { "center_latitude": config("CENTER_LATITUDE"), "center_longitude": config("CENTER_LONGITUDE"), "height_latitude": config("HEIGHT_LATITUDE"), "width_longitude": config("WIDTH_LONGITUDE"), "spacing_longitude": config("SPACING_LONGITUDE"), "spacing_latitude": config("SPACING_LATITUDE") } # BSF assumed lat/lng if USER_DEFINED_GRID; this support km spacing if not is_deg: grid_params["spacing_longitude"] /= km_per_deg_lng( grid_params["center_latitude"]) grid_params["spacing_latitude"] /= KM_PER_DEG_LAT elif config('grid'): grid_params = grid_params_from_grid(config('grid'), met_info) elif config('compute_grid'): if not fires or len(fires) != 1: # TODO: support multiple fires raise ValueError("Option to compute grid only supported for " "runs with one fire") if (not config('spacing_latitude') or not config('spacing_longitude')): raise BlueSkyConfigurationError( "Config settings " "'spacing_latitude' and 'spacing_longitude' required " "to compute hysplit grid") grid_params = square_grid_from_lat_lng(fires[0]['latitude'], fires[0]['longitude'], config('spacing_latitude'), config('spacing_longitude'), config('grid_length'), input_spacing_in_degrees=is_deg) elif met_info and met_info.get('grid'): grid_params = grid_params_from_grid(met_info['grid'], met_info) elif allow_undefined: grid_params = {} else: raise BlueSkyConfigurationError("Specify hysplit dispersion grid") logging.debug("grid_params: %s", grid_params) return grid_params
def wait_for_availability(config): # config can be undefined, but if it is defined, it must include # strategy, time, and max_attempts. if config: if any( [not config.get(k) for k in ('strategy', 'time', 'max_attempts')]): raise BlueSkyConfigurationError(INVALID_WAIT_CONFIG_MSG) if config['strategy'] not in ('fixed', 'backoff'): raise BlueSkyConfigurationError(INVALID_WAIT_STRATEGY_MSG) if not isinstance(config['time'], (int, float)) or config['time'] <= 0.0: raise BlueSkyConfigurationError(INVALID_WAIT_TIME_MSG) if (not isinstance(config['max_attempts'], int) or config['max_attempts'] <= 0): raise BlueSkyConfigurationError(INVALID_WAIT_MAX_ATTEMPTS_MSG) def decorator(f): if config: def decorated(*args, **kwargs): sleep_time = config['time'] attempts = 0 while True: try: return f(*args, **kwargs) except BlueSkyUnavailableResourceError as e: attempts += 1 if attempts == config['max_attempts']: logging.info( "Resource doesn't exist. Reached max attempts." " (%s)", e) raise logging.info( "Resource doesn't exist. Will attempt " "again in %s seconds. (%s)", sleep_time, e) time.sleep(sleep_time) if config['strategy'] == 'backoff': sleep_time *= 2 else: break return decorated else: return f return decorator
def import_module(module_name): logging.debug("Importing %s", module_name) try: return importlib.import_module(module_name) except ImportError: raise BlueSkyConfigurationError( "Invalid module: '{}'".format(module_name))
def __init__(self, dest_dir): self._filename = Config().get('extrafiles', 'emissionscsv', 'filename') if not self._filename: raise BlueSkyConfigurationError( "Specify destination " "('config' > 'extrafiles' > 'emissionscsv' > 'filename')") self._filename = os.path.join(dest_dir, self._filename)
def run(fires_manager): """Loads fire data from one or more sources Args: - fires_manager -- bluesky.models.fires.FiresManager object """ successfully_loaded_sources = [] sources = Config().get('load', 'sources') if not sources: raise BlueSkyConfigurationError("No sources specified for load module") try: for source in sources: # TODO: use something like with fires_manager.fire_failure_handler(fire) # to optionally skip invalid sources (sources that are insufficiently # configured, don't have corresponding loader class, etc.) and source # that fail to load try: loaded_fires = _load_source(source) fires_manager.add_fires(loaded_fires) # TODO: add fires to fires_manager if source.get("name").lower() == "bsf" and source.get("load_consumption"): datautils.summarize_all_levels(fires_manager, 'consumption') successfully_loaded_sources.append(source) except: # Let skip_failed_sources be defined at top level # or under 'load' in the config if not (Config().get('skip_failed_sources') or Config().get('load', 'skip_failed_sources', allow_missing=True)): raise finally: fires_manager.processed(__name__, __version__, successfully_loaded_sources=successfully_loaded_sources)
def __init__(self, extra_exports): super(LocalSaveExporter, self).__init__(extra_exports) self._dest = self.config('dest_dir') if not self._dest: raise BlueSkyConfigurationError( "Specify destination " "('config' > 'export' > 'localsave' > 'dest_dir')")
def _generate(self, fccs_id): if fccs_id not in self._custom: fuel_loadings = copy.copy(self._all_fuel_loadings[fccs_id]) f = tempfile.NamedTemporaryFile(mode='w') f.write(self.FCCS_LOADINGS_CSV_HEADER) # set fuelbed_id fuel_loadings['fuelbed_number'] = fccs_id # default non-loadings columns to empty string for k in self.NON_LOADINGS_FIELDS: fuel_loadings[k] = fuel_loadings.get(k, "") self._fill_in_defaults(fuel_loadings) # Keep the try/except in case based_on_fccs_id isn't defined and defaults # aren't filled in. try: row = self.FCCS_LOADINGS_CSV_ROW_TEMPLATE.format( **fuel_loadings) except KeyError as e: raise BlueSkyConfigurationError( "Missing fuel loadings field: '{}'".format(str(e))) f.write(row) f.flush() # return temp file object, not just it's name, since file is # deleted once obejct goes out of scope self._custom[fccs_id] = f return self._custom[fccs_id].name
def _apply_settings(fc, location, burn_type): valid_settings = dict(SETTINGS[burn_type], **SETTINGS['all']) for field, d in valid_settings.items(): value = None # If field == 'length_of_ignition', use location.ignition_start # and location.ignition_end, if both defined, else use # value from config d['default'] if field == 'length_of_ignition': if location.get('ignition_start') and location.get('ignition_end'): value = location.ignition_end - location.ignition_start # for backwards compatibility, support length_of_ignition elif location.get('length_of_ignition'): value = location.length_of_ignition else: possible_name = [field] + d.get('synonyms', []) defined_fields = [f for f in possible_name if f in location] if defined_fields: # use first of defined fields - it's not likely that # len(defined_fields) > 1 value = location[defined_fields[0]] if value: setattr(fc, field, value) elif 'default' in d: setattr(fc, field, d['default']) else: raise BlueSkyConfigurationError("Specify {} for {} burns".format( field, burn_type))
def _run_fire(hourly_fractions, fire): active_areas = fire.active_areas if (hourly_fractions and len(active_areas) > 1 and set([len(e) for p, e in hourly_fractions.items()]) != set([24])): # TODO: Support this scenario, but make sure # len(hourly_fractions) equals the total number of hours # represented by all activity objects, and pass the appropriate # slice into each instantiation of StaticTimeProfiler # (or build this into StaticProfiler???) raise BlueSkyConfigurationError( NOT_24_HOURLY_FRACTIONS_W_MULTIPLE_ACTIVE_AREAS_MSG) _validate_fire(fire) for a in active_areas: profiler = _get_profiler(hourly_fractions, fire, a) # convert timeprofile to dict with dt keys a['timeprofile'] = {} fields = list(profiler.hourly_fractions.keys()) for i in range(len(list(profiler.hourly_fractions.values()) [0])): # each phase should have same len hr = profiler.start_hour + (i * profiler.ONE_HOUR) a['timeprofile'][hr.isoformat()] = { p: profiler.hourly_fractions[p][i] for p in fields }
def _run_fire(hourly_fractions, fire): if (hourly_fractions and len(fire.activity) > 1 and set([len(e) for p, e in hourly_fractions.items()]) != set([24])): # TODO: Support this scenario, but make sure # len(hourly_fractions) equals the total number of hours # represented by all activity objects, and pass the appropriate # slice into each instantiation of StaticTimeProfiler # (or build this into StaticProfiler???) raise BlueSkyConfigurationError( "Only 24-hour repeatable time " "profiles supported for fires with multiple activity windows") _validate_fire(fire) for a in fire.activity: tw = parse_datetimes(a, 'start', 'end') profiler = StaticTimeProfiler(tw['start'], tw['end'], hourly_fractions=hourly_fractions) # convert timeprofile to dict with dt keys a['timeprofile'] = {} fields = list(profiler.hourly_fractions.keys()) for i in range(len(list(profiler.hourly_fractions.values()) [0])): # each phase should have same len hr = profiler.start_hour + (i * profiler.ONE_HOUR) a['timeprofile'][hr.isoformat()] = { p: profiler.hourly_fractions[p][i] for p in fields }
def _get_time_window(): start = Config().get('trajectories', 'start') if not start: raise BlueSkyConfigurationError("Missing trajectories 'start' time") num_hours = Config().get('trajectories', 'num_hours') return parse_datetime(start), num_hours
def _read_one_scp_dest_config(self, c): if any([not c.get(k) for k in ['host', 'dest_dir']]): raise BlueSkyConfigurationError( "Specify host and dest_dir for scp'ing") if 'scp' not in self._upload_options: self._upload_options['scp'] = [] self._upload_options['scp'].append(copy.deepcopy(c)) if not self._upload_options['scp'][-1].get('user'): self._upload_options['scp'][-1]['user'] = self._current_user
def __init__(self, implementation='ogr'): try: self._lookup = getattr( self, '_lookup_ecoregion_{}'.format(implementation)) except AttributeError: raise BlueSkyConfigurationError( "Invalid ecoregion lookup implementation: %s", implementation) self._input = None # instantiate when necessary
def _get_met_finder(fires_manager): met_root_dir = _get_met_root_dir(fires_manager) met_format = Config.get('findmetdata', 'met_format').lower() if met_format == "arl": arl_config = Config.get('findmetdata', 'arl') logging.debug("ARL config: %s", arl_config) return arlfinder.ArlFinder(met_root_dir, **arl_config) else: raise BlueSkyConfigurationError( "Invalid or unsupported met data format: '{}'".format(met_format))
def __init__(self, **config): super(BaseFileLoader, self).__init__(**config) self._filename = config.get('file') if not self._filename: raise BlueSkyConfigurationError("Fires file to load not specified") if not os.path.isfile(self._filename): raise BlueSkyUnavailableResourceError( "Fires file to " "load {} does not exist".format(self._filename))
def __init__(self, **config): self._config = config # start and end times, to use in filtering activity windows self._start = self._config.get('start') self._end = self._config.get('end') self._start = self._start and parse_datetime(self._start, 'start') self._end = self._end and parse_datetime(self._end, 'end') if self._start and self._end and self._start > self._end: raise BlueSkyConfigurationError(self.START_AFTER_END_ERROR_MSG)
def _get_dest_dir(): dest_dir = Config().get('extrafiles', 'dest_dir') if not dest_dir: raise BlueSkyConfigurationError("Specify extrafiles destination dir " "('config' > 'extrafiles' > 'dest_dir')") dest_dir = os.path.abspath(dest_dir) if not os.path.exists(dest_dir): os.makedirs(dest_dir) return dest_dir
def _import_loader(source): if any([not source.get(k) for k in ['name', 'format', 'type']]): raise BlueSkyConfigurationError( "Specify 'name', 'format', and 'type' for each source of fire data") try: loader_module = importlib.import_module( 'bluesky.loaders.{}'.format(source['name'].lower())) except ImportError as e: raise BlueSkyConfigurationError( "Invalid source: {}".format(source['name'])) loader = getattr(loader_module, '{}{}Loader'.format( source['format'].capitalize(), source['type'].capitalize())) if not loader: raise BlueSkyConfigurationError( "Invalid source format or type: {},{},{}".format( source['name'], source['format'], source['type'])) return loader
def get_extra_file_writers(file_sets, dest_dir): writers = [] for file_set in file_sets: writer_klass = EXTRA_FILE_WRITERS.get(file_set) if not writer_klass: raise BlueSkyConfigurationError("Invalid writer - {}".format( writer_klass)) writers.append( (file_set, writer_klass(dest_dir)) ) return writers
def import_class(module_name, klass_name): module = import_module(module_name) logging.debug("Loading class %s", klass_name) try: klass = getattr(module, klass_name) except: # TODO: use more appropriate exception class raise BlueSkyConfigurationError("{} does not define class {}".format( module_name, klass_name)) return module, klass
def _get_met_root_dir(fires_manager): # Note: ArlFinder will raise an exception if met_root_dir is undefined # or is not a valid directory # TODO: specify domain instead of met_root_dir, and somehow configure (not # in the code, since this is open source), per domain, the root dir, arl file # name pattern, etc. met_root_dir = Config.get('findmetdata', 'met_root_dir') if not met_root_dir: raise BlueSkyConfigurationError("Config setting 'met_root_dir' " "required by findmetdata module") logging.debug("Met root dir: %s", met_root_dir) return met_root_dir
def run(fires_manager): """Runs timeprofile module Args: - fires_manager -- bluesky.models.fires.FiresManager object """ hourly_fractions = Config().get('timeprofile', 'hourly_fractions') fires_manager.processed(__name__, __version__, timeprofile_version=timeprofile_version) for fire in fires_manager.fires: with fires_manager.fire_failure_handler(fire): try: _run_fire(hourly_fractions, fire) except InvalidHourlyFractionsError as e: raise BlueSkyConfigurationError( "Invalid timeprofile hourly fractions: '{}'".format( str(e))) except InvalidStartEndTimesError as e: raise BlueSkyConfigurationError( "Invalid timeprofile start end times: '{}'".format(str(e)))
def _get_met_finder(fires_manager): met_format = Config().get('findmetdata', 'met_format').lower() finder_klass = MET_FINDERS.get(met_format) if not finder_klass: raise BlueSkyConfigurationError( "Invalid or unsupported met data format: '{}'".format(met_format)) met_config = Config().get('findmetdata', met_format) logging.debug("%s config: %s", met_format, met_config) met_root_dir = _get_met_root_dir(fires_manager) return finder_klass(met_root_dir, **met_config)
def __init__(self, extra_exports): super(EmailExporter, self).__init__(extra_exports) self._recipients = self.config('recipients') if not self._recipients: raise BlueSkyConfigurationError("Specify email recipients.") # TODO: make sure each email address is valid self._sender = self.config('sender') self._subject = self.config('subject') self._server = self.config('smtp_server') self._port = int(self.config('smtp_port')) self._smtp_starttls = self.config('smtp_starttls') self._username = self.config('username') self._password = self.config('password')
def __init__(self): super().__init__() # Note: set date to persist in grow(), since we need fires_manager # to set default to the run's 'today' # cast to int in case it was specified as string in config file or on # command line (really, no excuse for this, since you can use -I option) self._days_to_persist = int(self.config('days_to_persist')) if self._days_to_persist <= 0: raise BlueSkyConfigurationError(DAYS_TO_PERSIST_NOT_INT) self._truncate = self.config('truncate')
def __init__(self, extra_exports): super(UploadExporter, self).__init__(extra_exports) self._upload_options = {} self._current_user = getpass.getuser() # TODO: don't assume scp? maybe look for 'scp' > 'host' & 'user' & ..., # and/or 'ftp' > ..., etc., and upload to all specified self._read_scp_config() # TODO: other upload options if not self._upload_options: raise BlueSkyConfigurationError( "Specify at least one mode of uploads")
def run(fires_manager): """Runs emissions module Args: - fires_manager -- bluesky.models.fires.FiresManager object Config options: - emissions > model -- emissions model to use - emissions > species -- whitelist of species to compute emissions for - emissions > include_emissions_details -- whether or not to include emissions per fuel category per phase, as opposed to just per phase - emissions > fuel_loadings -- - consumption > fuel_loadings -- considered if fuel loadings aren't specified in the emissions config """ model = Config().get('emissions', 'model') include_emissions_details = Config().get('emissions', 'include_emissions_details') fires_manager.processed(__name__, __version__, model=model, emitcalc_version=emitcalc_version, eflookup_version=eflookup_version, consume_version=CONSUME_VERSION_STR) try: klass_name = ''.join([e.capitalize() for e in model.split('-')]) klass = getattr(sys.modules[__name__], klass_name) e = klass(fires_manager.fire_failure_handler) except AttributeError: msg = "Invalid emissions model: '{}'.".format(model) if model == 'urbanski': msg += " The urbanski model has be replaced by prichard-oneill" raise BlueSkyConfigurationError(msg) e.run(fires_manager.fires) # fix keys for fire in fires_manager.fires: with fires_manager.fire_failure_handler(fire): for aa in fire.active_areas: for loc in aa.locations: for fb in loc['fuelbeds']: _fix_keys(fb['emissions']) if include_emissions_details: _fix_keys(fb['emissions_details']) datautils.summarize_all_levels(fires_manager, 'emissions') if include_emissions_details: datautils.summarize_over_all_fires(fires_manager, 'emissions_details')
def _apply_settings(fc, location, burn_type): valid_settings = dict(SETTINGS[burn_type], **SETTINGS['all']) for field, d in valid_settings.items(): possible_name = [field] + d.get('synonyms', []) defined_fields = [f for f in possible_name if f in location] if defined_fields: # use first of defined fields - it's not likely that # len(defined_fields) > 1 setattr(fc, field, location[defined_fields[0]]) elif 'default' in d: setattr(fc, field, d['default']) else: raise BlueSkyConfigurationError("Specify {} for {} burns".format( field, burn_type))
def run(fires_manager): """Loads fire data from one or more sources Args: - fires_manager -- bluesky.models.fires.FiresManager object """ model = Config().get('growth', 'model') # defaults to 'persistence' try: growth_module = importlib.import_module('bluesky.growers.{}'.format( model.lower())) except ImportError as e: raise BlueSkyConfigurationError( "Invalid growth module: {}".format(model)) getattr(growth_module, 'Grower')().grow(fires_manager)