def test_save_exposure(self):
# get a file name
f = tempfile.NamedTemporaryFile(
suffix='.npz',
prefix='HAZIMPt_jobs_test_save_exposure',
delete=False)
f.close()
inst = JOBS[SAVEALL]
con = context.Context()
actual = {
'shoes': array([11., 11]),
'depth': array([[5., 3.], [2., 4]]),
misc.INTID: array([0, 1, 2])
}
con.exposure_att = actual
lat = array([1, 22.])
con.exposure_lat = lat
lon = array([100., 22.])
con.exposure_long = lon
test_kwargs = {'file_name': f.name, 'use_parallel': False}
inst(con, **test_kwargs)
with numpy.load(f.name) as exp_dict:
actual[context.EX_LONG] = lon
actual[context.EX_LAT] = lat
for the_key in exp_dict.files:
self.assertTrue(allclose(exp_dict[the_key], actual[the_key]))
os.remove(f.name)
def test_load_raster_clipping(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.txt',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n')
f.write('8.1, 0.1, 1, 4\n')
f.write('7.9, 1.5, 2, -9999\n') # Out of Haz area
f.write('8.9, 2.9, 3, 6\n')
f.write('8.9, 3.1, 4, -9999.\n') # Out of Haz area
f.write('9.9, 2.9, 5, NaN\n') # In no data area
f.close()
inst = JOBS[LOADCSVEXPOSURE]
con_in = context.Context()
con_in.exposure_lat = None
con_in.exposure_long = None
con_in.exposure_att = {}
test_kwargs = {'file_name': f.name, 'use_parallel': False}
inst(con_in, **test_kwargs)
os.remove(f.name)
# Write a hazard file
f = tempfile.NamedTemporaryFile(suffix='.aai',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('ncols 3 \r\n')
f.write('nrows 2 \r\n')
f.write('xllcorner +0. \r\n')
f.write('yllcorner +8. \r\n')
f.write('cellsize 1 \r\n')
f.write('NODATA_value -9999\r\n')
f.write('1 2 -9999 \r\n')
f.write('4 5 6 ')
f.close()
haz_v = 'haz_v'
inst = JOBS[LOADRASTER]
test_kwargs = {
'file_list': [f.name],
'attribute_label': haz_v,
'clip_exposure2all_hazards': True
}
inst(con_in, **test_kwargs)
the_nans = isnan(con_in.exposure_att[haz_v])
con_in.exposure_att.loc[the_nans, (haz_v, )] = numpy.NAN #-9999
msg = "con_in.exposure_att[haz_v] \n" + str(
con_in.exposure_att[haz_v].values)
msg += "\n not = con_in.exposure_att['haz_actual'] \n" + \
str(con_in.exposure_att['haz_actual'].values)
self.assertTrue(
allclose(con_in.exposure_att[haz_v].values,
con_in.exposure_att['haz_actual'].values), msg)
# There should be only 3 exposure points
expected = 3
msg = "Number of exposure points is "
msg += str(len(con_in.exposure_att['ID']))
msg += "\n Expected " + str(expected)
self.assertTrue(len(con_in.exposure_att['ID']) == expected, msg)
os.remove(f.name)
def start(config_list=None, config_file=None, cont_in=None):
"""
Run the HazImp tool, based on the config info.
:param config_list: The configuration info, as a list.
:param config_file: The configuration info, as a file location.
:param cont_in: Only used in testing. A context instance.
:returns: The config dictionary.
"""
if config_file:
config_list = config.read_config_file(config_file)
if isinstance(config_list, dict):
msg = "Bad configuration file. \n"
msg += "Add a dash ( - ) before each variable. e.g. - template: flood"
raise RuntimeError(msg)
if config_list is None:
raise RuntimeError('No configuration information.')
if cont_in is None:
cont_in = context.Context()
calc_jobs = config.instance_builder(config_list)
the_pipeline = pipeline.PipeLine(calc_jobs)
the_pipeline.run(cont_in)
return cont_in
def test_startII(self):
# The config file
f = tempfile.NamedTemporaryFile(suffix='.yaml',
prefix='HAZIMPt_test_hazimp',
delete=False)
print(' - ' + templates.TEMPLATE + ': ' + templates.DEFAULT, file=f)
print(' - constant : ', file=f)
print(' var : c_test', file=f)
print(' value : 7 ', file=f)
print(' - add_test:', file=f)
print(' - multiply_test:', file=f)
f.close()
a_test = 5
b_test = 2
cont_in = context.Context()
cont_in.exposure_long = scipy.asarray([11.0])
cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test}
cont_in = hazimp.start(config_file=f.name, cont_in=cont_in)
os.remove(f.name)
self.assertEqual(cont_in.exposure_att['d_test'], 35) # 7 * 5
self.assertEqual(cont_in.exposure_att['c_test'], 7) # 5 + 2
def test_save_exposure_attsII(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.csv',
prefix='test_save_exposure_atts',
delete=False)
f.close()
con = context.Context()
actual = {'shoes': array([10., 11, 12]),
'depth': array([[5., 4., 3.], [3., 2, 1], [30., 20, 10]]),
misc.INTID: array([0, 1, 2])}
con.exposure_att = actual
lat = array([1, 2., 3])
con.exposure_lat = lat
lon = array([10., 20., 30])
con.exposure_long = lon
con.save_exposure_atts(f.name, use_parallel=False)
exp_dict = misc.csv2dict(f.name)
actual[context.EX_LONG] = lon
actual[context.EX_LAT] = lat
actual['depth'] = array([4, 2, 20])
for key in exp_dict:
self.assertTrue(allclose(exp_dict[key],
actual[key]))
os.remove(f.name)
def test_save_exposure_atts(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.npz',
prefix='test_save_exposure_atts',
delete=False)
f.close()
con = context.Context()
actual = {'shoes': array([10., 11]),
'depth': array([[5., 3.], [2., 4]]),
misc.INTID: array([0, 1, 2])}
con.exposure_att = actual
lat = array([1, 2.])
con.exposure_lat = lat
lon = array([10., 20.])
con.exposure_long = lon
con.save_exposure_atts(f.name, use_parallel=False)
with numpy.load(f.name) as exp_dict:
actual[context.EX_LONG] = lon
actual[context.EX_LAT] = lat
for keyish in exp_dict.files:
self.assertTrue(allclose(exp_dict[keyish],
actual[keyish]))
os.remove(f.name)
def test_Builder(self):
a_test = 5
b_test = 2
calc_list = [CALCS['add_test'], CALCS['multiply_test']]
cont_in = context.Context()
cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test}
the_pipeline = pipeline.PipeLine(calc_list)
the_pipeline.run(cont_in)
self.assertEqual(cont_in.exposure_att['d_test'], 35)
def test_get_site_count(self):
con = context.Context()
actual = {'shoes': array([10., 11]),
'depth': array([[5., 3.], [2., 4]]),
misc.INTID: array([0, 1, 2])}
con.exposure_att = actual
lat = array([1, 2.])
con.exposure_lat = lat
lon = array([10., 20.])
con.exposure_long = lon
self.assertEqual(con.get_site_shape(), (2,))
def test_BuilderII(self):
a_test = 5
b_test = 2
caj = workflow.ConfigAwareJob(CALCS['constant_test'],
atts_to_add={'constant': 5})
calc_list = [CALCS['add_test'], CALCS['multiply_test'], caj]
cont_in = context.Context()
cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test}
the_pipeline = pipeline.PipeLine(calc_list)
the_pipeline.run(cont_in)
self.assertEqual(cont_in.exposure_att['d_test'], 35)
self.assertEqual(cont_in.exposure_att['g_test'], 10)
def test_load_raster(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.txt',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n')
f.write('8.1, 0.1, 1, 4\n')
f.write('7.9, 1.5, 2, -9999\n')
f.write('8.9, 2.9, 3, 6\n')
f.write('8.9, 3.1, 4, -9999\n')
f.write('9.9, 2.9, 5, -9999\n')
f.close()
inst = JOBS[LOADCSVEXPOSURE]
con_in = context.Context()
con_in.exposure_lat = None
con_in.exposure_long = None
con_in.exposure_att = {}
test_kwargs = {'file_name': f.name}
inst(con_in, **test_kwargs)
os.remove(f.name)
# Write a hazard file
f = tempfile.NamedTemporaryFile(suffix='.aai',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('ncols 3 \r\n')
f.write('nrows 2 \r\n')
f.write('xllcorner +0. \r\n')
f.write('yllcorner +8. \r\n')
f.write('cellsize 1 \r\n')
f.write('NODATA_value -9999 \r\n')
f.write('1 2 -9999 \r\n')
f.write('4 5 6 ')
f.close()
haz_v = 'haz_v'
inst = JOBS[LOADRASTER]
test_kwargs = {'file_list': [f.name], 'attribute_label': haz_v}
inst(con_in, **test_kwargs)
the_nans = isnan(con_in.exposure_att[haz_v])
con_in.exposure_att.loc[the_nans, (haz_v, )] = -9999
msg = "con_in.exposure_att[haz_v] " + str(con_in.exposure_att[haz_v])
msg += "\n not = con_in.exposure_att['haz_actual'] " + \
str(con_in.exposure_att['haz_actual'])
self.assertTrue(
allclose(con_in.exposure_att[haz_v],
con_in.exposure_att['haz_actual']), msg)
os.remove(f.name)
def test_LoadCsvExposure(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.txt',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('exposure_latitude, exposure_longitude, ID, haz_0, haz_1\n')
f.write('8.1, 0.1, 1, 4, 40\n')
f.write('7.9, 1.5, 2, -9999, -9999\n')
f.write('8.9, 2.9, 3, 6, 60\n')
f.write('8.9, 3.1, 4, -9999, -9999\n')
f.write('9.9, 2.9, 5, -9999, -9999\n')
f.close()
inst = JOBS[LOADCSVEXPOSURE]
con_in = context.Context()
con_in.exposure_lat = None
con_in.exposure_long = None
con_in.exposure_att = None
test_kwargs = {'file_name': f.name}
inst(con_in, **test_kwargs)
os.remove(f.name)
if parallel.STATE.size == 1:
actual = numpy.arange(1, 6)
msg = "con_in.exposure_att['ID'] " \
+ str(con_in.exposure_att['ID'])
msg += "\n actual " + str(actual)
self.assertTrue(allclose(con_in.exposure_att['ID'], actual), msg)
else:
if parallel.STATE.rank == 0:
actual = numpy.array([1, 3, 5])
msg = "con_in.exposure_att['ID'] " \
+ str(con_in.exposure_att['ID'])
msg += "\n actual " + str(actual)
self.assertTrue(allclose(con_in.exposure_att['ID'], actual),
msg)
elif parallel.STATE.rank == 1:
actual = numpy.array([2, 4])
msg = "con_in.exposure_att['ID'] " \
+ str(con_in.exposure_att['ID'])
msg += "\n actual " + str(actual)
self.assertTrue(allclose(con_in.exposure_att['ID'], actual),
msg)
def test_load_raster_clippingIII(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.txt',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n')
f.write('7.9, 1.5, 2, -9999\n') # Out of Haz area
f.write('8.9, 3.1, 4, -9999\n') # Out of Haz area
f.close()
inst = JOBS[LOADCSVEXPOSURE]
con_in = context.Context()
con_in.exposure_lat = None
con_in.exposure_long = None
con_in.exposure_att = {}
test_kwargs = {'file_name': f.name, 'use_parallel': False}
inst(con_in, **test_kwargs)
os.remove(f.name)
raster = array([[1, 2, -9999], [4, 5, 6]])
upper_left_x = 0
upper_left_y = 10
cell_size = 1
no_data_value = -9999
haz_v = 'haz_v'
inst = JOBS[LOADRASTER]
test_kwargs = {
'attribute_label': haz_v,
'clip_exposure2all_hazards': True,
'raster': raster,
'upper_left_x': upper_left_x,
'upper_left_y': upper_left_y,
'cell_size': cell_size,
'no_data_value': no_data_value
}
inst(con_in, **test_kwargs)
# There should be only no exposure points
expected = 0
msg = "Number of exposure points is "
msg += str(len(con_in.exposure_att['ID']))
msg += "\n Expected " + str(expected)
self.assertTrue(len(con_in.exposure_att['ID']) == expected, msg)
def test_PipeLine_actually(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.csv',
prefix='test_Job_title_fix_Co',
delete=False)
f.write('LAT, LONG, a_test, b_test,BUILDING\n')
f.write('1., 2., 3., 30.,TAB\n')
f.write('4., 5., 6., 60.,DSG\n')
f.close()
f2 = tempfile.NamedTemporaryFile(suffix='.csv',
prefix='test_Job_title_fix_Co',
delete=False)
f2.close()
atts = {
'file_name': f.name,
context.EX_LAT: 'LAT',
context.EX_LONG: 'LONG'
}
caj1 = workflow.ConfigAwareJob(JOBS[LOADCSVEXPOSURE], atts_to_add=atts)
atts = {'var': 'con_test', 'value': 'yeah'}
caj2 = workflow.ConfigAwareJob(JOBS[CONSTANT], atts_to_add=atts)
atts = {'var': 'con2_test', 'value': 30}
caj3 = workflow.ConfigAwareJob(JOBS[CONSTANT], atts_to_add=atts)
calc_list = [caj1, caj2, caj3, CALCS['add_test']]
cont_in = context.Context()
the_pipeline = pipeline.PipeLine(calc_list)
the_pipeline.run(cont_in)
cont_dict = cont_in.save_exposure_atts(f2.name)
os.remove(f2.name)
if parallel.STATE.rank == 0:
self.assertTrue(allclose(cont_dict['c_test'], asarray([33., 66.])))
self.assertEqual(cont_dict['BUILDING'].tolist(), ['TAB', 'DSG'])
self.assertTrue(
allclose(cont_dict['con2_test'], asarray([30., 30.])))
self.assertEqual(cont_dict['con_test'].tolist(), ['yeah', 'yeah'])
os.remove(f.name)
def test_start(self):
config_list = [{
templates.TEMPLATE: templates.DEFAULT
}, {
'constant': {
'var': 'c_test',
'value': 7
}
}, {
'add_test': None
}, {
'multiply_test': None
}]
a_test = 5
b_test = 2
cont_in = context.Context()
cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test}
cont_in.exposure_long = scipy.asarray([11.0])
cont_in = hazimp.start(config_list=config_list, cont_in=cont_in)
self.assertEqual(cont_in.exposure_att['d_test'], 35)
self.assertEqual(cont_in.exposure_att['c_test'], 7)
def test_clip_exposure(self):
# These points are in the HazImp notebook.
lat_long = array([[-23, 110], [-23, 130], [-23, 145],
[-30, 110], [-35, 121], [-25, 139], [-30, 145],
[-37, 130]])
num_points = lat_long.shape[0]
shoes_array = arange(num_points * 2).reshape((-1, 2))
d3_array = arange(num_points * 2 * 3).reshape((-1, 2, 3))
id_array = arange(num_points)
con = context.Context()
sub_set = (4, 5)
initial = {'shoes': shoes_array,
'd3': d3_array,
misc.INTID: id_array}
con.exposure_att = initial
con.exposure_lat = lat_long[:, 0]
con.exposure_long = lat_long[:, 1]
# After this clip the only points that remain are;
# [-35, 121] & [-25, 139], indexed as 4 & 5
con.clip_exposure(min_lat=-36, max_lat=-24,
min_long=120, max_long=140)
actual = {}
actual[context.EX_LAT] = lat_long[:, 0][sub_set, ...]
actual[context.EX_LONG] = lat_long[:, 1][sub_set, ...]
actual['shoes'] = shoes_array[sub_set, ...]
actual['d3'] = d3_array[sub_set, ...]
actual[misc.INTID] = id_array[sub_set, ...]
for key in con.exposure_att:
self.assertTrue(allclose(con.exposure_att[key],
actual[key]))
def test_load_rasters(self):
# Write a file to test
f = tempfile.NamedTemporaryFile(suffix='.txt',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('exposure_latitude, exposure_longitude, ID, haz_0, haz_1\n')
f.write('8.1, 0.1, 1, 4, 40\n')
f.write('7.9, 1.5, 2, -9999, -9999\n')
f.write('8.9, 2.9, 3, 6, 60\n')
f.write('8.9, 3.1, 4, -9999, -9999\n')
f.write('9.9, 2.9, 5, -9999, -9999\n')
f.close()
inst = JOBS[LOADCSVEXPOSURE]
con_in = context.Context()
con_in.exposure_lat = con_in.exposure_long = None
con_in.exposure_att = {}
test_kwargs = {'file_name': f.name, 'use_parallel': False}
inst(con_in, **test_kwargs)
os.remove(f.name)
# Write a hazard file
f = tempfile.NamedTemporaryFile(suffix='.aai',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('ncols 3 \r\n')
f.write('nrows 2 \r\n')
f.write('xllcorner +0. \r\n')
f.write('yllcorner +8. \r\n')
f.write('cellsize 1 \r\n')
f.write('NODATA_value -9999 \r\n')
f.write('1 2 -9999 \r\n')
f.write('4 5 6 ')
f.close()
files = [f.name]
# Write another hazard file
f = tempfile.NamedTemporaryFile(suffix='.aai',
prefix='HAZIMPtest_jobs',
delete=False)
f.write('ncols 3 \r\n')
f.write('nrows 2 \r\n')
f.write('xllcorner +0. \r\n')
f.write('yllcorner +8. \r\n')
f.write('cellsize 1 \r\n')
f.write('NODATA_value -9999 \r\n')
f.write('10 20 -9999 \r\n')
f.write('40 50 60 ')
f.close()
files.append(f.name)
haz_v = 'haz_v'
inst = JOBS[LOADRASTER]
test_kwargs = {'file_list': files, 'attribute_label': haz_v}
inst(con_in, **test_kwargs)
the_nans = isnan(con_in.exposure_att[haz_v])
con_in.exposure_att[haz_v][the_nans] = -9999
actual = asarray(
[con_in.exposure_att['haz_0'], con_in.exposure_att['haz_1']])
actual = rollaxis(actual, 1)
msg = "con_in.exposure_att[haz_av] " \
+ str(con_in.exposure_att[haz_v])
msg += "\n actual " + str(actual)
self.assertTrue(allclose(con_in.exposure_att[haz_v], actual), msg)
for a_file in files:
os.remove(a_file)
