class FireActivityFilter(FiresActionBase):
"""Class for filtering fire activity windows by various criteria.
Note: The logic in this class is organized as a saparate class primarily
for maintainability, testibility, and readability. Some of it was
originally in the FiresManager class.
"""
def __init__(self, fires_manager, fire_class):
"""Constructor
args:
- fires_manager -- FiresManager object whose fires are to be merged
- fire_class -- Fire class to instantiate new fire objects
"""
super(FireActivityFilter, self).__init__(fires_manager, fire_class)
self._filter_config = Config().get('filter')
self._filter_fields = set(self._filter_config.keys()) - set(
['skip_failures'])
if not self._filter_fields:
if not self._skip_failures:
raise self.FilterError(self.NO_FILTERS_MSG)
# else, just log and return
logging.warning(self.NO_FILTERS_MSG)
ACTION = 'filter'
@property
def _action(self):
return self.ACTION
class FilterError(Exception):
pass
@property
def _error_class(self):
return self.FilterError
##
## Public API
##
NO_FILTERS_MSG = "No filters specified"
def filter(self):
"""Runs all secified filtered
"""
for filter_field in self._filter_fields:
logging.debug('About to run %s filter', filter_field)
# get filter function
try:
filter_func = self._get_filter_func(filter_field)
except self.FilterError as e:
if self._skip_failures:
logging.warning("Failed to initialize %s filter: %s",
filter_field, e)
continue
else:
raise
# run filter
self._filter(filter_func)
logging.info("Number of fires after running %s filter: %d",
filter_field, self._fires_manager.num_fires)
INVALID_FILTER_MSG = "Invalid filter"
MISSING_FILTER_CONFIG_MSG = "Specify config for each filter"
def _get_filter_func(self, filter_field):
"""Filters by specified field
args
- filter_field -- field to filter by (e.g. 'country')
"""
filter_getter = getattr(self, '_get_{}_filter'.format(filter_field),
self._get_filter)
if not filter_getter:
self._fail_or_skip(self.MISSING_FILTER_CONFIG_MSG)
kwargs = self._filter_config.get(filter_field)
if not kwargs:
self._fail_or_skip(self.MISSING_FILTER_CONFIG_MSG)
kwargs.update(filter_field=filter_field)
return filter_getter(**kwargs)
def _filter(self, filter_func):
"""Filter by given filter func
args:
- filter_func -- function that takes fire and active area object
and returns boolean value indicating whether or not to remove
active area object.
"""
for fire in self._fires_manager.fires:
i = 0
while i < len(fire.get('activity', [])):
j = 0
while j < len(fire['activity'][i].get('active_areas', [])):
try:
if filter_func(fire,
fire['activity'][i]['active_areas'][j]):
fire['activity'][i]['active_areas'].pop(j)
logging.debug('Filtered fire %s (%s)', fire.id,
fire._private_id)
# Note: if that was the last active area, then
# `j` must equal zero, and while loop will
# terminate
else:
j += 1
except self.FilterError as e:
if self._skip_failures:
j += 1
# str(e) is already detailed
logging.warning(str(e))
continue
else:
raise
if len(fire['activity'][i].get('active_areas', [])) == 0:
fire['activity'].pop(i)
else:
i += 1
if len(fire.get('activity', [])) == 0:
self._remove_fire(fire)
def _remove_fire(self, fire):
"""Removes fire from fires manager's `fires` list, and adds it
to `filtered_fires`
args
- fire -- fire to remove from active set
"""
self._fires_manager.remove_fire(fire)
if self._fires_manager.filtered_fires is None:
self._fires_manager.filtered_fires = []
# TDOO: add reason for filtering (specify at least filed)
self._fires_manager.filtered_fires.append(fire)
##
## Unterlying filter methods
##
SPECIFY_WHITELIST_OR_BLACKLIST_MSG = "Specify whitelist or blacklist - not both"
SPECIFY_FILTER_FIELD_MSG = "Specify field to filter on"
def _get_filter(self, **kwargs):
whitelist = kwargs.get('whitelist')
blacklist = kwargs.get('blacklist')
if (not whitelist and not blacklist) or (whitelist and blacklist):
raise self.FilterError(self.SPECIFY_WHITELIST_OR_BLACKLIST_MSG)
filter_field = kwargs.get('filter_field')
if not filter_field:
# This will never happen if called internally
raise self.FilterError(self.SPECIFY_FILTER_FIELD_MSG)
def _filter(fire, active_area):
v = active_area.get(filter_field)
if whitelist:
return not v or v not in whitelist
else:
return v and v in blacklist
return _filter
SPECIFY_BOUNDARY_MSG = "Specify boundary to filter by location"
INVALID_BOUNDARY_FIELDS_MSG = (
"Filter boundary must specify"
" 'ne' and 'sw', which each must have 'lat' and 'lng'")
INVALID_BOUNDARY_MSG = "Invalid boundary for filtering"
MISSING_FIRE_LOCATION_INFO_MSG = (
"Fire active areas must"
" have location information to be filtered by location")
def _get_location_filter(self, **kwargs):
"""Returns function that checks if fire activity window is within
boundary, which should be of the form:
{
"ne": {
"lat": 45.25,
"lng": -106.5
},
"sw": {
"lat": 27.75,
"lng": -131.5
}
}
Note: this function does not support boundaries spanning the
international date line. (i.e. NE lng > SW lng)
"""
b = kwargs.get('boundary')
if not b:
raise self.FilterError(self.SPECIFY_BOUNDARY_MSG)
elif (set(b.keys()) != set(["ne", "sw"]) or any(
[set(b[k].keys()) != set(["lat", "lng"]) for k in ["ne", "sw"]])):
raise self.FilterError(self.INVALID_BOUNDARY_FIELDS_MSG)
elif (any([abs(b[k]['lat']) > 90.0 for k in ['ne', 'sw']])
or any([abs(b[k]['lng']) > 180.0 for k in ['ne', 'sw']])
or any([b['ne'][k] < b['sw'][k] for k in ['lat', 'lng']])):
raise self.FilterError(self.INVALID_BOUNDARY_MSG)
def _filter(fire, active_area):
if not isinstance(active_area, dict):
self._fail_fire(fire, self.MISSING_FIRE_LOCATION_INFO_MSG)
try:
latlng = LatLng(active_area)
lat = latlng.latitude
lng = latlng.longitude
except ValueError as e:
self._fail_fire(fire, self.MISSING_FIRE_LOCATION_INFO_MSG)
if not lat or not lng:
self._fail_fire(fire, self.MISSING_FIRE_LOCATION_INFO_MSG)
return (lat < b['sw']['lat'] or lat > b['ne']['lat']
or lng < b['sw']['lng'] or lng > b['ne']['lng'])
return _filter
SPECIFY_MIN_OR_MAX_MSG = "Specify min and/or max area for filtering"
INVALID_MIN_MAX_MUST_BE_POS_MSG = "Min and max areas must be positive for filtering"
INVALID_MIN_MUST_BE_LTE_MAX_MSG = "Min area must be LTE max if both are specified"
MISSING_ACTIVITY_AREA_MSG = "Fire active area must have area information to be filtered by area"
NEGATIVE_ACTIVITY_AREA_MSG = "Fire active area's total can't be negative"
def _get_area_filter(self, **kwargs):
"""Returns funciton that checks if a fire is smaller than some
max threshold and/or larger than some min threshold.
"""
min_area = kwargs.get('min')
max_area = kwargs.get('max')
if min_area is None and max_area is None:
raise self.FilterError(self.SPECIFY_MIN_OR_MAX_MSG)
elif ((min_area is not None and min_area < 0.0)
or (max_area is not None and max_area < 0.0)):
raise self.FilterError(self.INVALID_MIN_MAX_MUST_BE_POS_MSG)
elif (min_area is not None and max_area is not None
and min_area > max_area):
raise self.FilterError(self.INVALID_MIN_MUST_BE_LTE_MAX_MSG)
def _filter(fire, active_area):
try:
total_active_area = active_area.total_area
except:
self._fail_fire(fire, self.MISSING_ACTIVITY_AREA_MSG)
if total_active_area < 0.0:
self._fail_fire(fire, self.NEGATIVE_ACTIVITY_AREA_MSG)
return ((min_area is not None and total_active_area < min_area)
or (max_area is not None and total_active_area > max_area))
return _filter
SPECIFY_TIME_START_AND_OR_END_MSG = "Specify start and/or end to filter by time"
INVALID_TIME_START_OR_END_VAL = "Invalid value for time filter config option '{}'"
INVALID_START_AFTER_END = ("Start must be before end if both are specified"
" for time fileter")
def _get_time_filter(self, **kwargs):
"""Returns function that checks if fire activity window is after
specified start time and/or before specified end time.
"""
s = kwargs.get('start')
e = kwargs.get('end')
if not s and not e:
raise self.FilterError(self.SPECIFY_TIME_START_AND_OR_END_MSG)
def _parse(v, key):
try:
is_local = hasattr(v, 'endswith') and v.endswith('L')
return (
to_datetime(v.rstrip('L') if hasattr(v, 'rstrip') else v),
is_local)
except:
raise self.FilterError(
self.INVALID_TIME_START_OR_END_VAL.format(key))
s, s_is_local = _parse(s, 'start')
e, e_is_local = _parse(e, 'end')
if s and e and s > e:
raise self.FilterError(self.INVALID_START_AFTER_END)
def _filter(fire, active_area):
if not isinstance(active_area, dict):
self._fail_fire(fire, self.MISSING_FIRE_LOCATION_INFO_MSG)
elif not active_area.get('start') or not active_area.get('end'):
self._fail_fire(fire, self.MISSING_FIRE_LOCATION_INFO_MSG)
utc_offset = datetime.timedelta(
hours=parse_utc_offset(active_area.get('utc_offset') or 0))
aa_s = to_datetime(active_area['start'])
# check if e_is_local, since we're comparing aa_s against e
if not e_is_local:
aa_s = aa_s - utc_offset
aa_e = to_datetime(active_area['end'])
# same thing, but s_is_local
if not s_is_local:
aa_e = aa_e - utc_offset
# note that this filters if aa's start/end matches cutoff
# (e.g. if aa's start and filter's end are both 2019-01-01T00:00:00)
return (s and aa_e <= s) or (e and aa_s >= e)
return _filter
def _log_config(self):
# TODO: bail if logging level is less than DEBUG (to avoid list and
# set operations)
_c = Config().get('dispersion', self._model)
for c in sorted(_c.keys()):
logging.debug('Dispersion config setting - %s = %s', c, _c[c])
