def test_with_dispersion_and_configured_time_window_and_three_fires(
self, reset_config):
fm = FiresManager()
fm.load({
"fires": [FIRE_1, FIRE_2, FIRE_3],
})
Config.set({
"dispersion": {
"start": "2014-05-29T19:00:00Z",
"num_hours": 12
},
"findmetdata": {
"time_window": {
"first_hour": "2016-01-04T04:00:00Z",
"last_hour": "2016-01-05T13:00:00Z"
}
}
})
expected = [{
'start': datetime.datetime(2014, 5, 29, 19, 0, 0),
'end': datetime.datetime(2014, 5, 30, 7, 0, 0),
}, {
'start': datetime.datetime(2015, 1, 21, 2, 0, 0),
'end': datetime.datetime(2015, 1, 23, 2, 0, 0),
}, {
'start': datetime.datetime(2015, 2, 2, 0, 0, 0),
'end': datetime.datetime(2015, 2, 3, 0, 0, 0),
}, {
'start': datetime.datetime(2016, 1, 4, 4, 0, 0),
'end': datetime.datetime(2016, 1, 5, 13, 0, 0),
}]
assert expected == findmetdata._get_time_windows(fm)
def test_with_dispersion_window_no_fires(self, reset_config):
fm = FiresManager()
Config.set(
{"dispersion": {
"start": "2014-05-29T19:00:00Z",
"num_hours": 12
}})
expected = [{
'start': datetime.datetime(2014, 5, 29, 19, 0, 0),
'end': datetime.datetime(2014, 5, 30, 7, 0, 0),
}]
assert expected == findmetdata._get_time_windows(fm)
def test_with_details(self, reset_config):
Config.set("consume", 'emissions', "model")
Config.set(True, 'emissions', "include_emissions_details")
emissions.Consume(fire_failure_manager).run(self.fires)
assert self.fires[0]['error'] == (
'Missing fuelbed data required for computing emissions')
assert 'emissions_details' in self.fires[1]['activity'][0]['fuelbeds'][
0]
self._check_emissions(
self.EXPECTED_FIRE1_EMISSIONS,
self.fires[1]['activity'][0]['fuelbeds'][0]['emissions'])
def test_with_configured_time_window_no_fires(self, reset_config):
fm = FiresManager()
Config.set({
"findmetdata": {
"time_window": {
"first_hour": "2016-01-04T04:00:00Z",
"last_hour": "2016-01-05T13:00:00Z"
}
}
})
expected = [{
'start': datetime.datetime(2016, 1, 4, 4, 0, 0),
'end': datetime.datetime(2016, 1, 5, 13, 0, 0),
}]
assert expected == findmetdata._get_time_windows(fm)
def run_input(module, input_file):
output_file = input_file.replace('input/', 'output/').replace(
'.json', '-EXPECTED-OUTPUT.json')
config_file = input_file.replace('input/', 'config/').replace(
'.json', '-CONFIG.json')
with open(config_file) as f:
config = json.loads(f.read()).get('config')
logging.debug('Running bsp on %s', input_file)
try:
Config.set(config)
fires_manager = models.fires.FiresManager()
fires_manager.loads(input_file=input_file)
fires_manager.modules = [module]
fires_manager.run()
except exceptions.BlueSkyModuleError as e:
# The error was added to fires_manager's meta data, and will be
# included in the output data
pass
except Exception as e:
# if output file doesn't exist, it means this expection was expected
# TODO: confirm that this is valid logic
if os.path.isfile(output_file):
logging.error('FAILED - %s - %s', input_file, e)
return False
else:
logging.debug('Caught expected exception')
return True
try:
logging.debug('Loading expected output file %s', output_file)
with open(output_file, 'r') as f:
expected = json.loads(f.read())
except FileNotFoundError as e:
logging.error('FAILED - %s - missing output file', input_file)
return False
# dumps and loads actual to convert datetimest, etc.
actual = json.loads(
json.dumps(fires_manager.dump(), cls=models.fires.FireEncoder))
success = check(expected, actual)
logging.info('PASSED - %s', input_file) if success else logging.error(
'FAILED - %s', input_file)
return success
def test_grid(self, reset_config):
Config.set(
{
"spacing": 6.0,
"boundary": {
"ne": {
"lat": 45.25,
"lng": -106.5
},
"sw": {
"lat": 27.75,
"lng": -131.5
}
}
}, "dispersion", "hysplit", "grid")
expected = {
'center_latitude': 36.5,
'center_longitude': -119.0,
'height_latitude': 17.5,
'spacing_latitude': 0.06705008458605,
'spacing_longitude': 0.06705008458605,
'width_longitude': 25.0
}
assert expected == hysplit_utils.get_grid_params()
def run(args):
pattern = '{}/data/{}/scen_{}.csv'.format(
os.path.abspath(os.path.dirname(__file__)),
args.data_dir, args.scenario_id or '[0-9]')
input_filenames = glob.glob(pattern)
if not input_filenames:
logging.error("No matching scnarios")
sys.exit(1)
logging.info("Scanarios: {}".format(', '.join(
[os.path.basename(n) for n in input_filenames])))
success = True
for input_filename in input_filenames:
fires_manager = load_scenario(input_filename)
Config.set(args.emissions_model, 'emissions', 'model')
Config.set(args.include_emissions_details,
'emissions','include_emissions_details')
fires_manager.modules = ['consumption', 'emissions']
fires_manager.run()
actual = fires_manager.dump()
expected_partials, expected_totals = load_output(
input_filename, args)
success = success and check(actual, expected_partials, expected_totals)
return success
def test_wo_details_PM_only(self, reset_config):
Config.set("consume", 'emissions', "model")
Config.set(False, 'emissions', "include_emissions_details")
Config.set(['PM2.5', 'PM10'], 'emissions', "species")
emissions.Consume(fire_failure_manager).run(self.fires)
assert self.fires[0]['error'] == (
'Missing fuelbed data required for computing emissions')
assert 'emissions_details' not in self.fires[1]['activity'][0][
'fuelbeds'][0]
self._check_emissions(
self.EXPECTED_FIRE1_EMISSIONS_PM_ONLY,
self.fires[1]['activity'][0]['fuelbeds'][0]['emissions'])
def test_with_details_PM_only(self, reset_config):
Config.set("prichard-oneill", 'emissions', "model")
Config.set(True, 'emissions', "include_emissions_details")
Config.set(self.SPECIES, 'emissions', "species")
emissions.PrichardOneill(fire_failure_manager).run(self.fires)
assert self.fires[0]['error'] == (
'Missing fuelbed data required for computing emissions')
assert 'emissions_details' in self.fires[1]['activity'][0]['fuelbeds'][
0]
self._check_emissions(
self.EXPECTED_FIRE1_EMISSIONS,
self.fires[1]['activity'][0]['fuelbeds'][0]['emissions'])
def test_config(self):
Config.set(2.2, "dispersion", "hysplit", "QCYCLE")
Config.set(333, "dispersion", "hysplit", "numpar")
Config.set(34, "dispersion", "hysplit", "FOO")
Config.set(100, "dispersion", "hysplit", "bar")
d = FakeDisperser({})
# non-overridden default
assert d.config('MGMIN') == 10
assert d.config('mgmin') == 10
# user overridden default specified by user as uppercase
assert d.config('QCYCLE') == 2.2
assert d.config('qcycle') == 2.2
# user overridden default specified by user as lowercase
assert d.config('NUMPAR') == 333
assert d.config('numpar') == 333
# user defined config setting without default, specified by user as uppercase
assert d.config('FOO') == 34
assert d.config('foo') == 34
# user defined config setting without default, specified by user as lowercase
assert d.config('BAR') == 100
assert d.config('bar') == 100
def setup(self):
lookup = mock.Mock()
Config.set(75.0, "fuelbeds", "truncation_percentage_threshold")
Config.set(2, "fuelbeds", "truncation_count_threshold")
self.estimator_w_options = fuelbeds.Estimator(lookup)
def setup(self):
Config.set(True, "ingestion", "keep_emissions")
Config.set(True, "ingestion", "keep_heat")
self.ingester = ingestion.FireIngester()
def test_merge_configs(self, reset_config):
Config.merge(
{"foo": {
"A": "{run_id}-{today}",
"b": 222,
"c": 333,
"d": 444
}})
EXPECTED_RAW = dict(
DEFAULTS,
**{"foo": {
"a": "{run_id}-{today}",
"b": 222,
"c": 333,
"d": 444
}})
EXPECTED = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-20160420",
"b": 222,
"c": 333,
"d": 444
}
})
assert Config._RUN_ID == None
assert Config._TODAY == None
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
Config.set_today(datetime.datetime(2019, 2, 4))
EXPECTED_RAW = dict(
DEFAULTS,
**{"foo": {
"a": "{run_id}-{today}",
"b": 222,
"c": 333,
"d": 444
}})
EXPECTED = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-20190204",
"b": 222,
"c": 333,
"d": 444
}
})
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 2, 4)
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
Config.merge({
"foo": {
"B": "{today-1}-{run_id}",
"c": 3333,
"d": 4444,
"bb": "bb"
},
"BAR": {
"b": "b"
},
"b": "b"
})
EXPECTED_RAW = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-{today}",
"b": "{today-1}-{run_id}",
"c": 3333,
"d": 4444,
"bb": "bb"
},
"bar": {
"b": "b"
},
"b": "b"
})
EXPECTED = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-20190204",
"b": "20190203-{run_id}",
"c": 3333,
"d": 4444,
"bb": "bb"
},
"bar": {
"b": "b"
},
"b": "b"
})
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 2, 4)
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
Config.merge({
"foo": {
"c": 33333,
"d": 44444,
"cc": "cc"
},
"baz": {
"c": "c"
},
"c": "c"
})
EXPECTED_RAW = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-{today}",
"b": "{today-1}-{run_id}",
"c": 33333,
"d": 44444,
"bb": "bb",
"cc": "cc"
},
"bar": {
"b": "b"
},
"baz": {
"c": "c"
},
"b": "b",
"c": "c"
})
EXPECTED = dict(
DEFAULTS, **{
"foo": {
"a": "{run_id}-20190204",
"b": "20190203-{run_id}",
"c": 33333,
"d": 44444,
"bb": "bb",
"cc": "cc"
},
"bar": {
"b": "b"
},
"baz": {
"c": "c"
},
"b": "b",
"c": "c"
})
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 2, 4)
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
Config.set("444444", 'foo', 'd')
Config.set("dd", 'foo', 'dd')
Config.set("d", 'boo', 'd')
Config.set("d", 'd')
Config.set(True, 'dbt')
Config.set(False, 'dbf')
Config.set(23, 'di')
Config.set(123.23, 'df')
Config.set('23', 'dci')
Config.set('123.23', 'dcf')
EXPECTED_RAW = dict(
DEFAULTS,
**{
"foo": {
"a": "{run_id}-{today}",
"b": "{today-1}-{run_id}",
"c": 33333,
"bb": "bb",
"cc": "cc",
"dd": "dd",
"d": "444444" # because it was set on command line
},
"bar": {
"b": "b"
},
"baz": {
"c": "c"
},
"boo": {
"d": "d"
},
"b": "b",
"c": "c",
"d": "d",
"dbt": True,
"dbf": False,
"di": 23,
"df": 123.23,
"dci": "23",
"dcf": "123.23"
})
EXPECTED = dict(
DEFAULTS,
**{
"foo": {
"a": "{run_id}-20190204",
"b": "20190203-{run_id}",
"c": 33333,
"bb": "bb",
"cc": "cc",
"dd": "dd",
"d": "444444" # because it was set on command line
},
"bar": {
"b": "b"
},
"baz": {
"c": "c"
},
"boo": {
"d": "d"
},
"b": "b",
"c": "c",
"d": "d",
"dbt": True,
"dbf": False,
"di": 23,
"df": 123.23,
"dci": "23",
"dcf": "123.23"
})
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 2, 4)
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
assert self._ORIGINAL_DEFAULTS == DEFAULTS
def test_compute_grid(self, reset_config):
fires_one = [{'latitude': 40.0, 'longitude': -118.5}]
fires_two = [{
'latitude': 40.0,
'longitude': -118.5
}, {
'latitude': 45.0,
'longitude': -117.5
}]
## Missing spacing
Config.set(True, "dispersion", "hysplit", "compute_grid")
with raises(BlueSkyConfigurationError) as e_info:
hysplit_utils.get_grid_params(fires=fires_one)
assert e_info.value.args[0] == (
"Config settings 'spacing_latitude' "
"and 'spacing_longitude' required to compute hysplit grid")
Config.reset()
Config.set(True, "dispersion", "hysplit", "compute_grid")
Config.set(0.05, 'dispersion', 'hysplit', 'spacing_longitude')
with raises(BlueSkyConfigurationError) as e_info:
hysplit_utils.get_grid_params(fires=fires_one)
assert e_info.value.args[0] == (
"Config settings 'spacing_latitude' "
"and 'spacing_longitude' required to compute hysplit grid")
Config.reset()
Config.set(True, "dispersion", "hysplit", "compute_grid")
Config.set(0.05, 'dispersion', 'hysplit', 'spacing_latitude')
with raises(BlueSkyConfigurationError) as e_info:
hysplit_utils.get_grid_params(fires=fires_one)
assert e_info.value.args[0] == (
"Config settings 'spacing_latitude' "
"and 'spacing_longitude' required to compute hysplit grid")
## no fires or two many fires
Config.reset()
Config.set(True, "dispersion", "hysplit", "compute_grid")
Config.set(0.05, 'dispersion', 'hysplit', 'spacing_latitude')
Config.set(0.05, 'dispersion', 'hysplit', 'spacing_longitude')
with raises(ValueError) as e_info:
hysplit_utils.get_grid_params()
assert e_info.value.args[
0] == 'Option to compute grid only supported for runs with one fire'
with raises(ValueError) as e_info:
hysplit_utils.get_grid_params(fires=fires_two)
assert e_info.value.args[
0] == 'Option to compute grid only supported for runs with one fire'
expected = {
'center_latitude': 40.0,
'center_longitude': -118.5,
'height_latitude': 18.01801801801802,
'spacing_latitude': 0.05,
'spacing_longitude': 0.05,
'width_longitude': 23.453239118438354
}
assert expected == hysplit_utils.get_grid_params(fires=fires_one)
# custom grid length (default is 2000)
Config.set(1000, 'dispersion', 'hysplit', 'grid_length')
expected = {
'center_latitude': 40.0,
'center_longitude': -118.5,
'height_latitude': 9.00900900900901,
'spacing_latitude': 0.05,
'spacing_longitude': 0.05,
'width_longitude': 11.726619559219177
}
assert expected == hysplit_utils.get_grid_params(fires=fires_one)
def test_setting_config_run_id_today(self, reset_config):
# setting
Config.set({"FOO": "{run_id}_{today-2:%Y%m%d}_bar", "bar": "baz"})
assert Config._RUN_ID == None
assert Config._TODAY == None
EXPECTED_RAW = dict(DEFAULTS,
foo="{run_id}_{today-2:%Y%m%d}_bar",
bar="baz")
EXPECTED = dict(DEFAULTS, foo="{run_id}_20160418_bar", bar="baz")
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
# set today
Config.set_today(datetime.datetime(2019, 1, 5, 10, 12, 1))
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 1, 5, 10, 12, 1)
EXPECTED_RAW = dict(DEFAULTS,
foo="{run_id}_{today-2:%Y%m%d}_bar",
bar="baz")
EXPECTED = dict(DEFAULTS, foo="{run_id}_20190103_bar", bar="baz")
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
# set again; datetime wildcard should be filled in
Config.set({"fOO": "{run_id}_{today:%Y%m%d%H}_bar", "bar": "sdfsdf"})
assert Config._RUN_ID == None
assert Config._TODAY == datetime.datetime(2019, 1, 5, 10, 12, 1)
EXPECTED_RAW = dict(DEFAULTS,
foo="{run_id}_{today:%Y%m%d%H}_bar",
bar="sdfsdf")
EXPECTED = dict(DEFAULTS, foo="{run_id}_2019010510_bar", bar="sdfsdf")
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
# set run_id
Config.set_run_id("abc123")
assert Config._RUN_ID == "abc123"
assert Config._TODAY == datetime.datetime(2019, 1, 5, 10, 12, 1)
EXPECTED_RAW = dict(DEFAULTS,
foo="{run_id}_{today:%Y%m%d%H}_bar",
bar="sdfsdf")
EXPECTED = dict(DEFAULTS, foo="abc123_2019010510_bar", bar="sdfsdf")
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
# set again; datetime and run_id wildcards should be filled in
Config.set({"foo": "FOO_{run_id}_{today:%Y%m%d%H}_bar", "bar": "zz"})
assert Config._RUN_ID == "abc123"
assert Config._TODAY == datetime.datetime(2019, 1, 5, 10, 12, 1)
EXPECTED_RAW = dict(DEFAULTS,
foo="FOO_{run_id}_{today:%Y%m%d%H}_bar",
bar="zz")
EXPECTED = dict(DEFAULTS, foo="FOO_abc123_2019010510_bar", bar="zz")
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
# set in individual values
Config.set(100, "bar")
Config.set(200, "BAAAR")
Config.set("sdfsdf{run_id}", "baz", "a")
Config.set("{run_id}", "BAZ", "b")
assert Config._RUN_ID == "abc123"
assert Config._TODAY == datetime.datetime(2019, 1, 5, 10, 12, 1)
EXPECTED_RAW = dict(DEFAULTS,
foo="FOO_{run_id}_{today:%Y%m%d%H}_bar",
bar=100,
baaar=200,
baz={
"a": "sdfsdf{run_id}",
"b": "{run_id}"
})
EXPECTED = dict(DEFAULTS,
foo="FOO_abc123_2019010510_bar",
bar=100,
baaar=200,
baz={
"a": "sdfsdfabc123",
"b": "abc123"
})
assert Config._RAW_CONFIG == EXPECTED_RAW
assert Config._CONFIG == EXPECTED
assert Config.get() == EXPECTED
assert self._ORIGINAL_DEFAULTS == DEFAULTS
def test_user_defined_grid(self, reset_config):
Config.set(True, "dispersion", "hysplit", "USER_DEFINED_GRID")
Config.set(36.5, "dispersion", "hysplit", "CENTER_LATITUDE")
Config.set(-119.0, "dispersion", "hysplit", "CENTER_LONGITUDE")
Config.set(25.0, "dispersion", "hysplit", "WIDTH_LONGITUDE")
Config.set(17.5, "dispersion", "hysplit", "HEIGHT_LATITUDE")
Config.set(0.05, "dispersion", "hysplit", "SPACING_LONGITUDE")
Config.set(0.05, "dispersion", "hysplit", "SPACING_LATITUDE")
expected = {
'center_latitude': 36.5,
'center_longitude': -119.0,
'height_latitude': 17.5,
'spacing_latitude': 0.05,
'spacing_longitude': 0.05,
'width_longitude': 25.0
}
assert expected == hysplit_utils.get_grid_params()
def setup(self):
lookup = mock.Mock()
Config.set(None, "fuelbeds", "truncation_percentage_threshold")
Config.set(None, "fuelbeds", "truncation_count_threshold")
self.estimator_no_truncation = fuelbeds.Estimator(lookup)