def _get_dispersion_time_window(fires_manager):
start = Config.get('dispersion', 'start')
num_hours = Config.get('dispersion', 'num_hours')
if start and num_hours:
return {
'start': start,
'end': start + datetime.timedelta(hours=num_hours)
}
def __init__(self, dest_dir, **kwargs):
fl = (kwargs.get('fire_locations_filename') or Config.get(
'extrafiles', 'firescsvs', 'fire_locations_filename'))
self._fire_locations_pathname = os.path.join(dest_dir, fl)
fe = (kwargs.get('fire_events_filename')
or Config.get('extrafiles', 'firescsvs', 'fire_events_filename'))
self._fire_events_pathname = os.path.join(dest_dir, fe)
def log_status(self, status, step, action, **extra_fields):
if not getattr(self, '_status_logger'):
# init_time will be converted to string if it's datetime.date[time]
init_time = (Config.get('dispersion', 'start', allow_missing=True)
or self.today)
sl_config = Config.get('statuslogging')
setattr(self, '_status_logger',
StatusLogger(init_time, **sl_config))
self._status_logger.log(status, step, action, **extra_fields)
def __init__(self, fire_failure_handler):
super(Consume, self).__init__(fire_failure_handler)
self.species = self.species and [e.upper() for e in self.species]
all_fuel_loadings = (Config.get('emissions','fuel_loadings')
or Config.get('consumption','fuel_loadings'))
self.fuel_loadings_manager = FuelLoadingsManager(
all_fuel_loadings=all_fuel_loadings)
def _get_met_finder(fires_manager):
met_root_dir = _get_met_root_dir(fires_manager)
met_format = Config.get('findmetdata', 'met_format').lower()
if met_format == "arl":
arl_config = Config.get('findmetdata', 'arl')
logging.debug("ARL config: %s", arl_config)
return arlfinder.ArlFinder(met_root_dir, **arl_config)
else:
raise BlueSkyConfigurationError(
"Invalid or unsupported met data format: '{}'".format(met_format))
def _get_dirs(fires_manager):
handle_existing = Config.get('dispersion', 'handle_existing')
output_dir = Config.get('dispersion', 'output_dir')
if not output_dir:
raise ValueError("Specify dispersion output directory")
create_dir_or_handle_existing(output_dir, handle_existing)
working_dir = Config.get('dispersion', 'working_dir')
if working_dir:
create_dir_or_handle_existing(working_dir, handle_existing)
return output_dir, working_dir
def _get_accepted_forecasts_config():
accepted_forecasts_config = Config().get('findmetdata',
'accepted_forecasts')
if accepted_forecasts_config:
for k in ('init_times', 'met_file_name_pattern'):
if not accepted_forecasts_config.get(k):
raise BlueSkyConfigurationError(
"Config 'findmetdata' > "
"'accepted_forecasts' settings must define '{}'".format(k))
accepted_forecasts_config['init_times'] = sorted([
to_datetime(i, extra_formats=["%Y%m%d%H"])
for i in accepted_forecasts_config.get('init_times', [])
])
return accepted_forecasts_config
def __init__(self, lookup):
self.lookup = lookup
# Is this necessary?
for attr, val in Config.get('fuelbeds').items():
if attr.startswith('truncation_'):
setattr(self, attr.replace('truncation_', ''), val)
def get_working_and_output_dirs(module_name, require_output_dir=True):
config = Config().get(module_name)
handle_existing = config.get('handle_existing')
output_dir = config.get('output_dir')
if not output_dir and require_output_dir:
raise ValueError("Specify {} output directory".format(module_name))
if output_dir:
create_dir_or_handle_existing(output_dir, handle_existing)
working_dir = config.get('working_dir')
if working_dir:
create_dir_or_handle_existing(working_dir, handle_existing)
return output_dir, working_dir
def run(fires_manager):
"""runs the findmetdata module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
logging.info("Running findmetdata module")
fires_manager.processed(__name__, __version__)
# TODO: For playground, there will be only one file containing data for
# any particular hour. Allow config to specify using simplified logic
# for this scenario. Otherwise, use more complicated logic in
# arlfinder.ArlFinder (or put both code paths in ArlFinder?)
met_finder = _get_met_finder(fires_manager)
time_windows = _get_time_windows(fires_manager)
wait_config = Config.get('findmetdata', 'wait')
@io.wait_for_availability(wait_config)
def _find():
files = []
for time_window in time_windows:
logging.debug("Findmetdata time window: %s to %s",
time_window['start'], time_window['end'])
files.extend(
met_finder.find(time_window['start'],
time_window['end']).get('files', []))
if not files:
raise BlueSkyUnavailableResourceError("No met files found")
return files
fires_manager.met = {"files": _find()}
def run(fires_manager):
"""Runs plumerise module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
logging.info("Running localmet module")
fires_manager.processed(__name__, __version__)
if not fires_manager.met:
raise ValueError("Specify met files to use in localmet")
arl_profiler = ArlProfiler(fires_manager.met.get('files'),
time_step=Config.get('localmet', 'time_step'))
logging.debug("Extracting localmet data for %d fires",
len(fires_manager.fires))
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
if not fire.get('activity'):
raise ValueError("Missing activity data required for localmet")
for a in fire['activity']:
latlng = LatLng(a.get('location'))
# parse_utc_offset makes sure utc offset is defined and valid
utc_offset = parse_utc_offset(
a.get('location', {}).get('utc_offset'))
tw = parse_datetimes(a, 'start', 'end')
a['localmet'] = arl_profiler.profile(latlng.latitude,
latlng.longitude,
tw['start'], tw['end'],
utc_offset)
def _get_profiler(hourly_fractions, fire, active_area):
tw = parse_datetimes(active_area, 'start', 'end')
# Use FepsTimeProfiler for Rx fires and StaticTimeProfiler for WF,
# Unless custom hourly_fractions are specified, in which case
# Static Time Profiler is used for all fires.
# If ignition_start and ignition_end aren't specified for Rx fires,
# FepsTimeProfiler will assume 9am-12pm
# TODO: add config setting to use FEPS for Rx even if custom
# hourly_fractions are specified (or the converse - i.e. alwys use
# FEPS for rx and add setting to turn on use of hourly_fractions,
# if specified, for Rx)
if fire.type == 'rx' and not hourly_fractions:
ig_start = active_area.get('ignition_start') and parse_datetime(
active_area['ignition_start'], k='ignition_start')
ig_end = active_area.get('ignition_end') and parse_datetime(
active_area['ignition_end'], k='ignition_end')
# TODO: pass in duff_fuel_load, total_above_ground_consumption,
# total_below_ground_consumption, moisture_category,
# relative_humidity, wind_speed, and duff_moisture_content,
# if defined?
return FepsTimeProfiler(tw['start'],
tw['end'],
local_ignition_start_time=ig_start,
local_ignition_end_time=ig_end,
fire_type=FireType.RX)
else:
model_name = Config().get("timeprofile", "model").lower()
if model_name == "ubc-bsf-feps":
wfrtConfig = Config().get('timeprofile', 'ubc-bsf-feps')
working_dir = wfrtConfig.get('working_dir')
delete_if_no_error = wfrtConfig.get(
'delete_working_dir_if_no_error')
with osutils.create_working_dir(
working_dir=working_dir,
delete_if_no_error=delete_if_no_error) as wdir:
fire_working_dir = os.path.join(
wdir, "feps-timeprofile-{}".format(fire.id))
if not os.path.exists(fire_working_dir):
os.makedirs(fire_working_dir)
return ubcbsffeps.UbcBsfFEPSTimeProfiler(
active_area, fire_working_dir, wfrtConfig)
else:
return StaticTimeProfiler(tw['start'],
tw['end'],
hourly_fractions=hourly_fractions)
def __init__(self, fires_manager):
"""Constructor
args:
- fires_manager -- FiresManager object whose fires are to be merged
"""
self._fires_manager = fires_manager
self._skip_failures = not not Config.get(self.ACTION, 'skip_failures')
def run(fires_manager):
"""Runs emissions module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
logging.info("Running fuelbeds module")
fires_manager.processed(__name__,
__version__,
fccsmap_version=fccsmap.__version__)
logging.debug('Using FCCS version %s',
Config.get('fuelbeds', 'fccs_version'))
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
if not fire.get('activity'):
raise ValueError(
"Activity information required to look up fuelbeds")
for a in fire['activity']:
if not a.get('location'):
raise ValueError(
"activity location information required to look up fuelbeds"
)
# TODO: instead of instantiating a new FccsLookUp and Estimator
# for each activity object, create AK and non-AK lookup and
# estimator objects that are reused, and set reference to
# correct one here
# TODO: is_alaska from lat,lng, not from 'state'
lookup = FccsLookUp(
is_alaska=a['location'].get('state') == 'AK',
**Config.get('fuelbeds'))
Estimator(lookup).estimate(a)
# TODO: Add fuel loadings data to each fuelbed object (????)
# If we do so here, use bluesky.modules.consumption.FuelLoadingsManager
# (which should maybe be moved to a common module if to be used here)
# fm = FuelLoadingsManager()
# for fire in fires_manager.fires:
# for fb in get_fuel_loadings():
# fb['fuel_loadings'] = fm.get_fuel_loadings(fb['fccs_id'])
# Note: probably no need to do this here since we do it in the
# consumption module
fires_manager.summarize(fuelbeds=summarize(fires_manager.fires))
def get_dispersion_model(fires_manager):
if fires_manager.dispersion and fires_manager.dispersion.get('model'):
return fires_manager.dispersion['model']
model = Config.get('dispersion', 'model')
if model:
return model.lower()
raise ValueError(
"Dispersion model must be specified if visualizing dispersion")
def _get_dest_dir(fires_manager):
dest_dir = Config.get('extrafiles', 'dest_dir')
if not dest_dir:
raise BlueSkyConfigurationError("Specify extrafiles destination dir "
"('config' > 'extrafiles' > 'dest_dir')")
dest_dir = os.path.abspath(dest_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
return dest_dir
def __init__(self, fires_manager):
model = Config.get('plumerising', 'model').lower()
fires_manager.processed(__name__, __version__,
plumerise_version=plumerise_version, model=model)
logging.debug('Generating %s plumerise compution function', model)
generator = getattr(self, '_{}'.format(model), None)
if not generator:
raise BlueSkyConfigurationError(
"Invalid plumerise model: '{}'".format(model))
config = Config.get('plumerising', model)
self._compute_func = generator(config)
if config.get('working_dir'):
fires_manager.plumerising = {
'output': {
'directory': config['working_dir']
}
}
def _get_time(fires_manager):
start = Config.get('dispersion', 'start')
num_hours = Config.get('dispersion', 'num_hours')
if not start or num_hours is None:
s = fires_manager.earliest_start # needed for 'start' and 'num_hours'
if not s:
raise ValueError("Unable to determine dispersion 'start'")
if not start:
start = s
if not num_hours and start == s:
e = fires_manager.latest_end # needed only for num_hours
if e and e > s:
num_hours = int((e - s).total_seconds() / SECONDS_PER_HOUR)
if not num_hours:
raise ValueError("Unable to determine dispersion 'num_hours'")
logging.debug("Dispersion window: %s for %s hours", start, num_hours)
return start, num_hours
def _use_default_ecoregion(fires_manager, a, exc=None):
default_ecoregion = Config.get('consumption', 'default_ecoregion')
if default_ecoregion:
logging.debug('Using default ecoregion %s', default_ecoregion)
a['location']['ecoregion'] = default_ecoregion
else:
logging.debug('No default ecoregion')
if exc:
raise exc
else:
raise ValueError("No default ecoregion specified.")
def _get_met_root_dir(fires_manager):
# Note: ArlFinder will raise an exception if met_root_dir is undefined
# or is not a valid directory
# TODO: specify domain instead of met_root_dir, and somehow configure (not
# in the code, since this is open source), per domain, the root dir, arl file
# name pattern, etc.
met_root_dir = Config.get('findmetdata', 'met_root_dir')
if not met_root_dir:
raise BlueSkyConfigurationError("Config setting 'met_root_dir' "
"required by findmetdata module")
logging.debug("Met root dir: %s", met_root_dir)
return met_root_dir
def _ingest_activity_heat(self, activity, src):
if Config.get('ingestion', 'keep_heat'):
logging.debug("Ingesting heat")
# TODO: look for value in src['heat|HEAT']['summary']['total'] first
heat = self._get_numeric_val(src, "heat", "HEAT")
if heat is not None:
logging.debug("Recording heat in activity object")
activity["heat"] = {
"summary": {
"total": heat
}
}
def __init__(self, dest_dir, **kwargs):
ptinv = (kwargs.get('ptinv_filename')
or Config.get('extrafiles', 'smokeready', 'ptinv_filename'))
self._ptinv_pathname = os.path.join(dest_dir, ptinv)
ptday = (kwargs.get('ptday_filename')
or Config.get('extrafiles', 'smokeready', 'ptday_filename'))
self._ptday_pathname = os.path.join(dest_dir, ptday)
pthour = (kwargs.get('pthour_filename')
or Config.get('extrafiles', 'smokeready', 'pthour_filename'))
self._pthour_pathname = os.path.join(dest_dir, pthour)
separate_smolder = (kwargs.get('separate_smolder') or Config.get(
'extrafiles', 'smokeready', 'separate_smolder'))
self._separate_smolder = separate_smolder
write_ptinv_totals = (kwargs.get('write_ptinv_totals') or Config.get(
'extrafiles', 'smokeready', 'write_ptinv_totals'))
self._write_ptinv_totals = write_ptinv_totals
write_ptday_file = (kwargs.get('write_ptday_file') or Config.get(
'extrafiles', 'smokeready', 'write_ptday_file'))
self._write_ptday_file = write_ptday_file
# Pull the file year out of the dynamically set timestamp
self.file_year = int(pthour.split('-')[1][:4])
def run(fires_manager):
"""runs the export module
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
modes = [m.lower() for m in Config.get('export', 'modes')]
fires_manager.processed(__name__, __version__, modes=modes)
extra_exports = Config.get('export', 'extra_exports')
exporters = []
for mode in modes:
exporter_klass = EXPORTERS.get(mode)
if not exporter_klass:
raise BlueSkyConfigurationError(
"Invalid exporter - {}".format(exporter_klass))
exporters.append(exporter_klass(extra_exports))
# Note: export modules update fires_manager with export info, since that
# info needs to be in the fires_manager before it's dumped to json
for exporter in exporters:
exporter.export(fires_manager)
def __init__(self, fires_manager):
"""Constructor
args:
- fires_manager -- FiresManager object whose fires are to be merged
"""
super(FireActivityFilter, self).__init__(fires_manager)
self._filter_config = Config.get('filter')
self._filter_fields = set(self._filter_config.keys()) - set(
['skip_failures'])
if not self._filter_fields:
if not self._skip_failures:
raise self.FilterError(self.NO_FILTERS_MSG)
# else, just log and return
logging.warning(self.NO_FILTERS_MSG)
def run(fires_manager):
"""Split each of the fire's activity windows to be able to process points
separately.
Args:
- fires_manager -- bluesky.models.fires.FiresManager object
"""
logging.info("Running merge module")
fires_manager.processed(__name__, __version__)
record_original_activity = Config.get(
'splitactivity', 'record_original_activity')
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
_split(fire, record_original_activity)
def _ingest_activity_emissions(self, activity, src):
if Config.get('ingestion', 'keep_emissions'):
logging.debug("Ingesting emissions")
emissions = {"summary": {}}
for e in self.EMISSIONS_SPECIES:
keys = [e] if hasattr(e, "lower") else e
keys = [i for j in [(e, e.lower()) for e in keys] for i in j]
# TODO: look for value in src['emissions']['summary'][k],
# for any k in keys, first
v = self._get_numeric_val(src, *keys)
if v is not None:
emissions["summary"][keys[0]] = v
if emissions["summary"]:
logging.debug("Recording emissions in activity object")
activity["emissions"] = emissions
def _get_configured_time_windows(fires_manager):
time_window = Config.get('findmetdata', 'time_window')
if time_window:
logging.debug("Met time window specified in the config")
time_window = parse_datetimes(time_window, 'first_hour', 'last_hour')
# met data finder doesn't require round hours, but ....
if (not is_round_hour(time_window['first_hour'])
or not is_round_hour(time_window['last_hour'])):
raise BlueSkyConfigurationError(
"Met first and last hours must be round hours")
time_window = {
'start': time_window['first_hour'],
'end':
time_window['last_hour'] # TODO: round this up to the next hour?
}
return time_window
def _validate_input(fires_manager):
ecoregion_lookup = None # instantiate only if necessary
for fire in fires_manager.fires:
with fires_manager.fire_failure_handler(fire):
if not fire.get('activity'):
raise ValueError(
"Missing activity data required for computing consumption")
for a in fire.activity:
for k in ('fuelbeds', 'location'):
if not a.get(k):
raise ValueError("Missing activity '{}' data required "
"for computing consumption".format(k))
# only 'area' is required from location
if not a['location'].get('area'):
raise ValueError("Fire activity location data must "
"define area for computing consumption")
if not a['location'].get('ecoregion'):
# import EcoregionLookup here so that, if fires do have
# ecoregion defined, consumption can be run without mapscript
# and other dependencies installed
try:
latlng = LatLng(a['location'])
if not ecoregion_lookup:
from bluesky.ecoregion.lookup import EcoregionLookup
implemenation = Config.get(
'consumption',
'ecoregion_lookup_implemenation')
ecoregion_lookup = EcoregionLookup(implemenation)
a['location']['ecoregion'] = ecoregion_lookup.lookup(
latlng.latitude, latlng.longitude)
if not a['location']['ecoregion']:
logging.warning("Failed to look up ecoregion for "
"{}, {}".format(
latlng.latitude,
latlng.longitude))
_use_default_ecoregion(fires_manager, a)
except exceptions.MissingDependencyError as e:
_use_default_ecoregion(fires_manager, a, e)
for fb in a['fuelbeds']:
if not fb.get('fccs_id') or not fb.get('pct'):
raise ValueError(
"Each fuelbed must define 'fccs_id' and 'pct'")
def get_targets():
vis_config = Config().get('visualization')
targets = vis_config['targets']
# 'visualization' > 'target' supported for backwards compatibility
if vis_config.get('target'):
# this will only be hit if it's an older config specifying
# 'visualization' > 'target'
targets = [vis_config['target']]
targets = [t.lower() for t in targets]
logging.info("Visualizatin targets '%s'", "', '".join(targets))
invalid = [t for t in targets if t not in vis_config]
if invalid:
raise ValueError("Invalid visualization target(s): '{}'".format(
"','".join(invalid)))
return targets
def dump(self):
# Don't include 'modules' in the output. The modules to be run may have
# been specified on the command line or in the input json. Either way,
# 'processing' contains a record of what modules were run (though it may
# be fewer modules than what were requested, which would be the case
# if there was a failure). We don't want to include 'modules' in the
# output because that breaks the ability to pipe the results into
# another run of bsp.
# TODO: keep track of whether modules were specified in the input
# json or on the command line, and add them to the output if they
# were in the input
return dict(self._meta,
fires=self.fires,
today=self.today,
run_id=self.run_id,
counts=self.counts,
bluesky_version=__version__,
run_config=Config.get())