def __init__(self, **config):
super(BaseApiLoader, self).__init__(**config)
self._endpoint = config.get('endpoint')
if not self._endpoint:
raise BlueSkyConfigurationError("Json API not specified")
self._key = config.get('key')
self._secret = config.get('secret')
# you can have a key without a secret, but not vice versa
if self._secret and not self._key:
raise BlueSkyConfigurationError(
"Api key must be specified if secret is specified")
self._key_param = config.get('key_param', self.DEFAULT_KEY_PARAM)
self._auth_protocol = config.get('auth_protocol',
self.DEFAULT_AUTH_PROTOCOL)
self._request_timeout = config.get('request_timeout',
self.DEFAULT_REQUEST_TIMEOUT)
self._query = config.get('query', {})
# Convert datetime.date objects to strings
for k in self._query:
if isinstance(self._query[k], datetime.date):
self._query[k] = self._query[k].strftime(self.DATETIME_FORMAT)
def get_grid_params(met_info={}, fires=None, allow_undefined=False):
# defaults to 'LatLon' in config defaults
is_deg = config('projection') == 'LatLon'
if config("USER_DEFINED_GRID"):
# This supports BSF config settings
# User settings that can override the default concentration grid info
logging.info("User-defined sampling/concentration grid invoked")
grid_params = {
"center_latitude": config("CENTER_LATITUDE"),
"center_longitude": config("CENTER_LONGITUDE"),
"height_latitude": config("HEIGHT_LATITUDE"),
"width_longitude": config("WIDTH_LONGITUDE"),
"spacing_longitude": config("SPACING_LONGITUDE"),
"spacing_latitude": config("SPACING_LATITUDE")
}
# BSF assumed lat/lng if USER_DEFINED_GRID; this support km spacing
if not is_deg:
grid_params["spacing_longitude"] /= km_per_deg_lng(
grid_params["center_latitude"])
grid_params["spacing_latitude"] /= KM_PER_DEG_LAT
elif config('grid'):
grid_params = grid_params_from_grid(config('grid'), met_info)
elif config('compute_grid'):
if not fires or len(fires) != 1:
# TODO: support multiple fires
raise ValueError("Option to compute grid only supported for "
"runs with one fire")
if (not config('spacing_latitude') or not config('spacing_longitude')):
raise BlueSkyConfigurationError(
"Config settings "
"'spacing_latitude' and 'spacing_longitude' required "
"to compute hysplit grid")
grid_params = square_grid_from_lat_lng(fires[0]['latitude'],
fires[0]['longitude'],
config('spacing_latitude'),
config('spacing_longitude'),
config('grid_length'),
input_spacing_in_degrees=is_deg)
elif met_info and met_info.get('grid'):
grid_params = grid_params_from_grid(met_info['grid'], met_info)
elif allow_undefined:
grid_params = {}
else:
raise BlueSkyConfigurationError("Specify hysplit dispersion grid")
logging.debug("grid_params: %s", grid_params)
return grid_params
def wait_for_availability(config):
# config can be undefined, but if it is defined, it must include
# strategy, time, and max_attempts.
if config:
if any(
[not config.get(k) for k in ('strategy', 'time', 'max_attempts')]):
raise BlueSkyConfigurationError(INVALID_WAIT_CONFIG_MSG)
if config['strategy'] not in ('fixed', 'backoff'):
raise BlueSkyConfigurationError(INVALID_WAIT_STRATEGY_MSG)
if not isinstance(config['time'],
(int, float)) or config['time'] <= 0.0:
raise BlueSkyConfigurationError(INVALID_WAIT_TIME_MSG)
if (not isinstance(config['max_attempts'], int)
or config['max_attempts'] <= 0):
raise BlueSkyConfigurationError(INVALID_WAIT_MAX_ATTEMPTS_MSG)
def decorator(f):
if config:
def decorated(*args, **kwargs):
sleep_time = config['time']
attempts = 0
while True:
try:
return f(*args, **kwargs)
except BlueSkyUnavailableResourceError as e:
attempts += 1
if attempts == config['max_attempts']:
logging.info(
"Resource doesn't exist. Reached max attempts."
" (%s)", e)
raise
logging.info(
"Resource doesn't exist. Will attempt "
"again in %s seconds. (%s)", sleep_time, e)
time.sleep(sleep_time)
if config['strategy'] == 'backoff':
sleep_time *= 2
else:
break
return decorated
else:
return f
return decorator
def import_module(module_name):
logging.debug("Importing %s", module_name)
try:
return importlib.import_module(module_name)
except ImportError:
raise BlueSkyConfigurationError(
"Invalid module: '{}'".format(module_name))
def __init__(self, dest_dir):
self._filename = Config().get('extrafiles', 'emissionscsv', 'filename')
if not self._filename:
raise BlueSkyConfigurationError(
"Specify destination "
"('config' > 'extrafiles' > 'emissionscsv' > 'filename')")
self._filename = os.path.join(dest_dir, self._filename)
def run(fires_manager):
"""Loads fire data from one or more sources
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
successfully_loaded_sources = []
sources = Config().get('load', 'sources')
if not sources:
raise BlueSkyConfigurationError("No sources specified for load module")
try:
for source in sources:
# TODO: use something like with fires_manager.fire_failure_handler(fire)
# to optionally skip invalid sources (sources that are insufficiently
# configured, don't have corresponding loader class, etc.) and source
# that fail to load
try:
loaded_fires = _load_source(source)
fires_manager.add_fires(loaded_fires)
# TODO: add fires to fires_manager
if source.get("name").lower() == "bsf" and source.get("load_consumption"):
datautils.summarize_all_levels(fires_manager, 'consumption')
successfully_loaded_sources.append(source)
except:
# Let skip_failed_sources be defined at top level
# or under 'load' in the config
if not (Config().get('skip_failed_sources')
or Config().get('load', 'skip_failed_sources', allow_missing=True)):
raise
finally:
fires_manager.processed(__name__, __version__,
successfully_loaded_sources=successfully_loaded_sources)
def __init__(self, extra_exports):
super(LocalSaveExporter, self).__init__(extra_exports)
self._dest = self.config('dest_dir')
if not self._dest:
raise BlueSkyConfigurationError(
"Specify destination "
"('config' > 'export' > 'localsave' > 'dest_dir')")
def _generate(self, fccs_id):
if fccs_id not in self._custom:
fuel_loadings = copy.copy(self._all_fuel_loadings[fccs_id])
f = tempfile.NamedTemporaryFile(mode='w')
f.write(self.FCCS_LOADINGS_CSV_HEADER)
# set fuelbed_id
fuel_loadings['fuelbed_number'] = fccs_id
# default non-loadings columns to empty string
for k in self.NON_LOADINGS_FIELDS:
fuel_loadings[k] = fuel_loadings.get(k, "")
self._fill_in_defaults(fuel_loadings)
# Keep the try/except in case based_on_fccs_id isn't defined and defaults
# aren't filled in.
try:
row = self.FCCS_LOADINGS_CSV_ROW_TEMPLATE.format(
**fuel_loadings)
except KeyError as e:
raise BlueSkyConfigurationError(
"Missing fuel loadings field: '{}'".format(str(e)))
f.write(row)
f.flush()
# return temp file object, not just it's name, since file is
# deleted once obejct goes out of scope
self._custom[fccs_id] = f
return self._custom[fccs_id].name
def _apply_settings(fc, location, burn_type):
valid_settings = dict(SETTINGS[burn_type], **SETTINGS['all'])
for field, d in valid_settings.items():
value = None
# If field == 'length_of_ignition', use location.ignition_start
# and location.ignition_end, if both defined, else use
# value from config d['default']
if field == 'length_of_ignition':
if location.get('ignition_start') and location.get('ignition_end'):
value = location.ignition_end - location.ignition_start
# for backwards compatibility, support length_of_ignition
elif location.get('length_of_ignition'):
value = location.length_of_ignition
else:
possible_name = [field] + d.get('synonyms', [])
defined_fields = [f for f in possible_name if f in location]
if defined_fields:
# use first of defined fields - it's not likely that
# len(defined_fields) > 1
value = location[defined_fields[0]]
if value:
setattr(fc, field, value)
elif 'default' in d:
setattr(fc, field, d['default'])
else:
raise BlueSkyConfigurationError("Specify {} for {} burns".format(
field, burn_type))
def _run_fire(hourly_fractions, fire):
active_areas = fire.active_areas
if (hourly_fractions and len(active_areas) > 1
and set([len(e)
for p, e in hourly_fractions.items()]) != set([24])):
# TODO: Support this scenario, but make sure
# len(hourly_fractions) equals the total number of hours
# represented by all activity objects, and pass the appropriate
# slice into each instantiation of StaticTimeProfiler
# (or build this into StaticProfiler???)
raise BlueSkyConfigurationError(
NOT_24_HOURLY_FRACTIONS_W_MULTIPLE_ACTIVE_AREAS_MSG)
_validate_fire(fire)
for a in active_areas:
profiler = _get_profiler(hourly_fractions, fire, a)
# convert timeprofile to dict with dt keys
a['timeprofile'] = {}
fields = list(profiler.hourly_fractions.keys())
for i in range(len(list(profiler.hourly_fractions.values())
[0])): # each phase should have same len
hr = profiler.start_hour + (i * profiler.ONE_HOUR)
a['timeprofile'][hr.isoformat()] = {
p: profiler.hourly_fractions[p][i]
for p in fields
}
def _run_fire(hourly_fractions, fire):
if (hourly_fractions and len(fire.activity) > 1
and set([len(e)
for p, e in hourly_fractions.items()]) != set([24])):
# TODO: Support this scenario, but make sure
# len(hourly_fractions) equals the total number of hours
# represented by all activity objects, and pass the appropriate
# slice into each instantiation of StaticTimeProfiler
# (or build this into StaticProfiler???)
raise BlueSkyConfigurationError(
"Only 24-hour repeatable time "
"profiles supported for fires with multiple activity windows")
_validate_fire(fire)
for a in fire.activity:
tw = parse_datetimes(a, 'start', 'end')
profiler = StaticTimeProfiler(tw['start'],
tw['end'],
hourly_fractions=hourly_fractions)
# convert timeprofile to dict with dt keys
a['timeprofile'] = {}
fields = list(profiler.hourly_fractions.keys())
for i in range(len(list(profiler.hourly_fractions.values())
[0])): # each phase should have same len
hr = profiler.start_hour + (i * profiler.ONE_HOUR)
a['timeprofile'][hr.isoformat()] = {
p: profiler.hourly_fractions[p][i]
for p in fields
}
def _get_time_window():
start = Config().get('trajectories', 'start')
if not start:
raise BlueSkyConfigurationError("Missing trajectories 'start' time")
num_hours = Config().get('trajectories', 'num_hours')
return parse_datetime(start), num_hours
def _read_one_scp_dest_config(self, c):
if any([not c.get(k) for k in ['host', 'dest_dir']]):
raise BlueSkyConfigurationError(
"Specify host and dest_dir for scp'ing")
if 'scp' not in self._upload_options:
self._upload_options['scp'] = []
self._upload_options['scp'].append(copy.deepcopy(c))
if not self._upload_options['scp'][-1].get('user'):
self._upload_options['scp'][-1]['user'] = self._current_user
def __init__(self, implementation='ogr'):
try:
self._lookup = getattr(
self, '_lookup_ecoregion_{}'.format(implementation))
except AttributeError:
raise BlueSkyConfigurationError(
"Invalid ecoregion lookup implementation: %s", implementation)
self._input = None # instantiate when necessary
def _get_met_finder(fires_manager):
met_root_dir = _get_met_root_dir(fires_manager)
met_format = Config.get('findmetdata', 'met_format').lower()
if met_format == "arl":
arl_config = Config.get('findmetdata', 'arl')
logging.debug("ARL config: %s", arl_config)
return arlfinder.ArlFinder(met_root_dir, **arl_config)
else:
raise BlueSkyConfigurationError(
"Invalid or unsupported met data format: '{}'".format(met_format))
def __init__(self, **config):
super(BaseFileLoader, self).__init__(**config)
self._filename = config.get('file')
if not self._filename:
raise BlueSkyConfigurationError("Fires file to load not specified")
if not os.path.isfile(self._filename):
raise BlueSkyUnavailableResourceError(
"Fires file to "
"load {} does not exist".format(self._filename))
def __init__(self, **config):
self._config = config
# start and end times, to use in filtering activity windows
self._start = self._config.get('start')
self._end = self._config.get('end')
self._start = self._start and parse_datetime(self._start, 'start')
self._end = self._end and parse_datetime(self._end, 'end')
if self._start and self._end and self._start > self._end:
raise BlueSkyConfigurationError(self.START_AFTER_END_ERROR_MSG)
def _get_dest_dir():
dest_dir = Config().get('extrafiles', 'dest_dir')
if not dest_dir:
raise BlueSkyConfigurationError("Specify extrafiles destination dir "
"('config' > 'extrafiles' > 'dest_dir')")
dest_dir = os.path.abspath(dest_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
return dest_dir
def _import_loader(source):
if any([not source.get(k) for k in ['name', 'format', 'type']]):
raise BlueSkyConfigurationError(
"Specify 'name', 'format', and 'type' for each source of fire data")
try:
loader_module = importlib.import_module(
'bluesky.loaders.{}'.format(source['name'].lower()))
except ImportError as e:
raise BlueSkyConfigurationError(
"Invalid source: {}".format(source['name']))
loader = getattr(loader_module, '{}{}Loader'.format(
source['format'].capitalize(), source['type'].capitalize()))
if not loader:
raise BlueSkyConfigurationError(
"Invalid source format or type: {},{},{}".format(
source['name'], source['format'], source['type']))
return loader
def get_extra_file_writers(file_sets, dest_dir):
writers = []
for file_set in file_sets:
writer_klass = EXTRA_FILE_WRITERS.get(file_set)
if not writer_klass:
raise BlueSkyConfigurationError("Invalid writer - {}".format(
writer_klass))
writers.append(
(file_set, writer_klass(dest_dir))
)
return writers
def import_class(module_name, klass_name):
module = import_module(module_name)
logging.debug("Loading class %s", klass_name)
try:
klass = getattr(module, klass_name)
except:
# TODO: use more appropriate exception class
raise BlueSkyConfigurationError("{} does not define class {}".format(
module_name, klass_name))
return module, klass
def _get_met_root_dir(fires_manager):
# Note: ArlFinder will raise an exception if met_root_dir is undefined
# or is not a valid directory
# TODO: specify domain instead of met_root_dir, and somehow configure (not
# in the code, since this is open source), per domain, the root dir, arl file
# name pattern, etc.
met_root_dir = Config.get('findmetdata', 'met_root_dir')
if not met_root_dir:
raise BlueSkyConfigurationError("Config setting 'met_root_dir' "
"required by findmetdata module")
logging.debug("Met root dir: %s", met_root_dir)
return met_root_dir
def run(fires_manager):
"""Runs timeprofile module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
hourly_fractions = Config().get('timeprofile', 'hourly_fractions')
fires_manager.processed(__name__,
__version__,
timeprofile_version=timeprofile_version)
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
try:
_run_fire(hourly_fractions, fire)
except InvalidHourlyFractionsError as e:
raise BlueSkyConfigurationError(
"Invalid timeprofile hourly fractions: '{}'".format(
str(e)))
except InvalidStartEndTimesError as e:
raise BlueSkyConfigurationError(
"Invalid timeprofile start end times: '{}'".format(str(e)))
def _get_met_finder(fires_manager):
met_format = Config().get('findmetdata', 'met_format').lower()
finder_klass = MET_FINDERS.get(met_format)
if not finder_klass:
raise BlueSkyConfigurationError(
"Invalid or unsupported met data format: '{}'".format(met_format))
met_config = Config().get('findmetdata', met_format)
logging.debug("%s config: %s", met_format, met_config)
met_root_dir = _get_met_root_dir(fires_manager)
return finder_klass(met_root_dir, **met_config)
def __init__(self, extra_exports):
super(EmailExporter, self).__init__(extra_exports)
self._recipients = self.config('recipients')
if not self._recipients:
raise BlueSkyConfigurationError("Specify email recipients.")
# TODO: make sure each email address is valid
self._sender = self.config('sender')
self._subject = self.config('subject')
self._server = self.config('smtp_server')
self._port = int(self.config('smtp_port'))
self._smtp_starttls = self.config('smtp_starttls')
self._username = self.config('username')
self._password = self.config('password')
def __init__(self):
super().__init__()
# Note: set date to persist in grow(), since we need fires_manager
# to set default to the run's 'today'
# cast to int in case it was specified as string in config file or on
# command line (really, no excuse for this, since you can use -I option)
self._days_to_persist = int(self.config('days_to_persist'))
if self._days_to_persist <= 0:
raise BlueSkyConfigurationError(DAYS_TO_PERSIST_NOT_INT)
self._truncate = self.config('truncate')
def __init__(self, extra_exports):
super(UploadExporter, self).__init__(extra_exports)
self._upload_options = {}
self._current_user = getpass.getuser()
# TODO: don't assume scp? maybe look for 'scp' > 'host' & 'user' & ...,
# and/or 'ftp' > ..., etc., and upload to all specified
self._read_scp_config()
# TODO: other upload options
if not self._upload_options:
raise BlueSkyConfigurationError(
"Specify at least one mode of uploads")
def run(fires_manager):
"""Runs emissions module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
Config options:
- emissions > model -- emissions model to use
- emissions > species -- whitelist of species to compute emissions for
- emissions > include_emissions_details -- whether or not to include
emissions per fuel category per phase, as opposed to just per phase
- emissions > fuel_loadings --
- consumption > fuel_loadings -- considered if fuel loadings aren't
specified in the emissions config
"""
model = Config().get('emissions', 'model')
include_emissions_details = Config().get('emissions',
'include_emissions_details')
fires_manager.processed(__name__,
__version__,
model=model,
emitcalc_version=emitcalc_version,
eflookup_version=eflookup_version,
consume_version=CONSUME_VERSION_STR)
try:
klass_name = ''.join([e.capitalize() for e in model.split('-')])
klass = getattr(sys.modules[__name__], klass_name)
e = klass(fires_manager.fire_failure_handler)
except AttributeError:
msg = "Invalid emissions model: '{}'.".format(model)
if model == 'urbanski':
msg += " The urbanski model has be replaced by prichard-oneill"
raise BlueSkyConfigurationError(msg)
e.run(fires_manager.fires)
# fix keys
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
for aa in fire.active_areas:
for loc in aa.locations:
for fb in loc['fuelbeds']:
_fix_keys(fb['emissions'])
if include_emissions_details:
_fix_keys(fb['emissions_details'])
datautils.summarize_all_levels(fires_manager, 'emissions')
if include_emissions_details:
datautils.summarize_over_all_fires(fires_manager, 'emissions_details')
def _apply_settings(fc, location, burn_type):
valid_settings = dict(SETTINGS[burn_type], **SETTINGS['all'])
for field, d in valid_settings.items():
possible_name = [field] + d.get('synonyms', [])
defined_fields = [f for f in possible_name if f in location]
if defined_fields:
# use first of defined fields - it's not likely that
# len(defined_fields) > 1
setattr(fc, field, location[defined_fields[0]])
elif 'default' in d:
setattr(fc, field, d['default'])
else:
raise BlueSkyConfigurationError("Specify {} for {} burns".format(
field, burn_type))
def run(fires_manager):
"""Loads fire data from one or more sources
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
model = Config().get('growth', 'model') # defaults to 'persistence'
try:
growth_module = importlib.import_module('bluesky.growers.{}'.format(
model.lower()))
except ImportError as e:
raise BlueSkyConfigurationError(
"Invalid growth module: {}".format(model))
getattr(growth_module, 'Grower')().grow(fires_manager)
