def test_save_exposure(self): # get a file name f = tempfile.NamedTemporaryFile( suffix='.npz', prefix='HAZIMPt_jobs_test_save_exposure', delete=False) f.close() inst = JOBS[SAVEALL] con = context.Context() actual = { 'shoes': array([11., 11]), 'depth': array([[5., 3.], [2., 4]]), misc.INTID: array([0, 1, 2]) } con.exposure_att = actual lat = array([1, 22.]) con.exposure_lat = lat lon = array([100., 22.]) con.exposure_long = lon test_kwargs = {'file_name': f.name, 'use_parallel': False} inst(con, **test_kwargs) with numpy.load(f.name) as exp_dict: actual[context.EX_LONG] = lon actual[context.EX_LAT] = lat for the_key in exp_dict.files: self.assertTrue(allclose(exp_dict[the_key], actual[the_key])) os.remove(f.name)
def test_load_raster_clipping(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='HAZIMPtest_jobs', delete=False) f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n') f.write('8.1, 0.1, 1, 4\n') f.write('7.9, 1.5, 2, -9999\n') # Out of Haz area f.write('8.9, 2.9, 3, 6\n') f.write('8.9, 3.1, 4, -9999.\n') # Out of Haz area f.write('9.9, 2.9, 5, NaN\n') # In no data area f.close() inst = JOBS[LOADCSVEXPOSURE] con_in = context.Context() con_in.exposure_lat = None con_in.exposure_long = None con_in.exposure_att = {} test_kwargs = {'file_name': f.name, 'use_parallel': False} inst(con_in, **test_kwargs) os.remove(f.name) # Write a hazard file f = tempfile.NamedTemporaryFile(suffix='.aai', prefix='HAZIMPtest_jobs', delete=False) f.write('ncols 3 \r\n') f.write('nrows 2 \r\n') f.write('xllcorner +0. \r\n') f.write('yllcorner +8. \r\n') f.write('cellsize 1 \r\n') f.write('NODATA_value -9999\r\n') f.write('1 2 -9999 \r\n') f.write('4 5 6 ') f.close() haz_v = 'haz_v' inst = JOBS[LOADRASTER] test_kwargs = { 'file_list': [f.name], 'attribute_label': haz_v, 'clip_exposure2all_hazards': True } inst(con_in, **test_kwargs) the_nans = isnan(con_in.exposure_att[haz_v]) con_in.exposure_att.loc[the_nans, (haz_v, )] = numpy.NAN #-9999 msg = "con_in.exposure_att[haz_v] \n" + str( con_in.exposure_att[haz_v].values) msg += "\n not = con_in.exposure_att['haz_actual'] \n" + \ str(con_in.exposure_att['haz_actual'].values) self.assertTrue( allclose(con_in.exposure_att[haz_v].values, con_in.exposure_att['haz_actual'].values), msg) # There should be only 3 exposure points expected = 3 msg = "Number of exposure points is " msg += str(len(con_in.exposure_att['ID'])) msg += "\n Expected " + str(expected) self.assertTrue(len(con_in.exposure_att['ID']) == expected, msg) os.remove(f.name)
def start(config_list=None, config_file=None, cont_in=None): """ Run the HazImp tool, based on the config info. :param config_list: The configuration info, as a list. :param config_file: The configuration info, as a file location. :param cont_in: Only used in testing. A context instance. :returns: The config dictionary. """ if config_file: config_list = config.read_config_file(config_file) if isinstance(config_list, dict): msg = "Bad configuration file. \n" msg += "Add a dash ( - ) before each variable. e.g. - template: flood" raise RuntimeError(msg) if config_list is None: raise RuntimeError('No configuration information.') if cont_in is None: cont_in = context.Context() calc_jobs = config.instance_builder(config_list) the_pipeline = pipeline.PipeLine(calc_jobs) the_pipeline.run(cont_in) return cont_in
def test_startII(self): # The config file f = tempfile.NamedTemporaryFile(suffix='.yaml', prefix='HAZIMPt_test_hazimp', delete=False) print(' - ' + templates.TEMPLATE + ': ' + templates.DEFAULT, file=f) print(' - constant : ', file=f) print(' var : c_test', file=f) print(' value : 7 ', file=f) print(' - add_test:', file=f) print(' - multiply_test:', file=f) f.close() a_test = 5 b_test = 2 cont_in = context.Context() cont_in.exposure_long = scipy.asarray([11.0]) cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test} cont_in = hazimp.start(config_file=f.name, cont_in=cont_in) os.remove(f.name) self.assertEqual(cont_in.exposure_att['d_test'], 35) # 7 * 5 self.assertEqual(cont_in.exposure_att['c_test'], 7) # 5 + 2
def test_save_exposure_attsII(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.csv', prefix='test_save_exposure_atts', delete=False) f.close() con = context.Context() actual = {'shoes': array([10., 11, 12]), 'depth': array([[5., 4., 3.], [3., 2, 1], [30., 20, 10]]), misc.INTID: array([0, 1, 2])} con.exposure_att = actual lat = array([1, 2., 3]) con.exposure_lat = lat lon = array([10., 20., 30]) con.exposure_long = lon con.save_exposure_atts(f.name, use_parallel=False) exp_dict = misc.csv2dict(f.name) actual[context.EX_LONG] = lon actual[context.EX_LAT] = lat actual['depth'] = array([4, 2, 20]) for key in exp_dict: self.assertTrue(allclose(exp_dict[key], actual[key])) os.remove(f.name)
def test_save_exposure_atts(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.npz', prefix='test_save_exposure_atts', delete=False) f.close() con = context.Context() actual = {'shoes': array([10., 11]), 'depth': array([[5., 3.], [2., 4]]), misc.INTID: array([0, 1, 2])} con.exposure_att = actual lat = array([1, 2.]) con.exposure_lat = lat lon = array([10., 20.]) con.exposure_long = lon con.save_exposure_atts(f.name, use_parallel=False) with numpy.load(f.name) as exp_dict: actual[context.EX_LONG] = lon actual[context.EX_LAT] = lat for keyish in exp_dict.files: self.assertTrue(allclose(exp_dict[keyish], actual[keyish])) os.remove(f.name)
def test_Builder(self): a_test = 5 b_test = 2 calc_list = [CALCS['add_test'], CALCS['multiply_test']] cont_in = context.Context() cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test} the_pipeline = pipeline.PipeLine(calc_list) the_pipeline.run(cont_in) self.assertEqual(cont_in.exposure_att['d_test'], 35)
def test_get_site_count(self): con = context.Context() actual = {'shoes': array([10., 11]), 'depth': array([[5., 3.], [2., 4]]), misc.INTID: array([0, 1, 2])} con.exposure_att = actual lat = array([1, 2.]) con.exposure_lat = lat lon = array([10., 20.]) con.exposure_long = lon self.assertEqual(con.get_site_shape(), (2,))
def test_BuilderII(self): a_test = 5 b_test = 2 caj = workflow.ConfigAwareJob(CALCS['constant_test'], atts_to_add={'constant': 5}) calc_list = [CALCS['add_test'], CALCS['multiply_test'], caj] cont_in = context.Context() cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test} the_pipeline = pipeline.PipeLine(calc_list) the_pipeline.run(cont_in) self.assertEqual(cont_in.exposure_att['d_test'], 35) self.assertEqual(cont_in.exposure_att['g_test'], 10)
def test_load_raster(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='HAZIMPtest_jobs', delete=False) f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n') f.write('8.1, 0.1, 1, 4\n') f.write('7.9, 1.5, 2, -9999\n') f.write('8.9, 2.9, 3, 6\n') f.write('8.9, 3.1, 4, -9999\n') f.write('9.9, 2.9, 5, -9999\n') f.close() inst = JOBS[LOADCSVEXPOSURE] con_in = context.Context() con_in.exposure_lat = None con_in.exposure_long = None con_in.exposure_att = {} test_kwargs = {'file_name': f.name} inst(con_in, **test_kwargs) os.remove(f.name) # Write a hazard file f = tempfile.NamedTemporaryFile(suffix='.aai', prefix='HAZIMPtest_jobs', delete=False) f.write('ncols 3 \r\n') f.write('nrows 2 \r\n') f.write('xllcorner +0. \r\n') f.write('yllcorner +8. \r\n') f.write('cellsize 1 \r\n') f.write('NODATA_value -9999 \r\n') f.write('1 2 -9999 \r\n') f.write('4 5 6 ') f.close() haz_v = 'haz_v' inst = JOBS[LOADRASTER] test_kwargs = {'file_list': [f.name], 'attribute_label': haz_v} inst(con_in, **test_kwargs) the_nans = isnan(con_in.exposure_att[haz_v]) con_in.exposure_att.loc[the_nans, (haz_v, )] = -9999 msg = "con_in.exposure_att[haz_v] " + str(con_in.exposure_att[haz_v]) msg += "\n not = con_in.exposure_att['haz_actual'] " + \ str(con_in.exposure_att['haz_actual']) self.assertTrue( allclose(con_in.exposure_att[haz_v], con_in.exposure_att['haz_actual']), msg) os.remove(f.name)
def test_LoadCsvExposure(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='HAZIMPtest_jobs', delete=False) f.write('exposure_latitude, exposure_longitude, ID, haz_0, haz_1\n') f.write('8.1, 0.1, 1, 4, 40\n') f.write('7.9, 1.5, 2, -9999, -9999\n') f.write('8.9, 2.9, 3, 6, 60\n') f.write('8.9, 3.1, 4, -9999, -9999\n') f.write('9.9, 2.9, 5, -9999, -9999\n') f.close() inst = JOBS[LOADCSVEXPOSURE] con_in = context.Context() con_in.exposure_lat = None con_in.exposure_long = None con_in.exposure_att = None test_kwargs = {'file_name': f.name} inst(con_in, **test_kwargs) os.remove(f.name) if parallel.STATE.size == 1: actual = numpy.arange(1, 6) msg = "con_in.exposure_att['ID'] " \ + str(con_in.exposure_att['ID']) msg += "\n actual " + str(actual) self.assertTrue(allclose(con_in.exposure_att['ID'], actual), msg) else: if parallel.STATE.rank == 0: actual = numpy.array([1, 3, 5]) msg = "con_in.exposure_att['ID'] " \ + str(con_in.exposure_att['ID']) msg += "\n actual " + str(actual) self.assertTrue(allclose(con_in.exposure_att['ID'], actual), msg) elif parallel.STATE.rank == 1: actual = numpy.array([2, 4]) msg = "con_in.exposure_att['ID'] " \ + str(con_in.exposure_att['ID']) msg += "\n actual " + str(actual) self.assertTrue(allclose(con_in.exposure_att['ID'], actual), msg)
def test_load_raster_clippingIII(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='HAZIMPtest_jobs', delete=False) f.write('exposure_latitude, exposure_longitude, ID, haz_actual\n') f.write('7.9, 1.5, 2, -9999\n') # Out of Haz area f.write('8.9, 3.1, 4, -9999\n') # Out of Haz area f.close() inst = JOBS[LOADCSVEXPOSURE] con_in = context.Context() con_in.exposure_lat = None con_in.exposure_long = None con_in.exposure_att = {} test_kwargs = {'file_name': f.name, 'use_parallel': False} inst(con_in, **test_kwargs) os.remove(f.name) raster = array([[1, 2, -9999], [4, 5, 6]]) upper_left_x = 0 upper_left_y = 10 cell_size = 1 no_data_value = -9999 haz_v = 'haz_v' inst = JOBS[LOADRASTER] test_kwargs = { 'attribute_label': haz_v, 'clip_exposure2all_hazards': True, 'raster': raster, 'upper_left_x': upper_left_x, 'upper_left_y': upper_left_y, 'cell_size': cell_size, 'no_data_value': no_data_value } inst(con_in, **test_kwargs) # There should be only no exposure points expected = 0 msg = "Number of exposure points is " msg += str(len(con_in.exposure_att['ID'])) msg += "\n Expected " + str(expected) self.assertTrue(len(con_in.exposure_att['ID']) == expected, msg)
def test_PipeLine_actually(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.csv', prefix='test_Job_title_fix_Co', delete=False) f.write('LAT, LONG, a_test, b_test,BUILDING\n') f.write('1., 2., 3., 30.,TAB\n') f.write('4., 5., 6., 60.,DSG\n') f.close() f2 = tempfile.NamedTemporaryFile(suffix='.csv', prefix='test_Job_title_fix_Co', delete=False) f2.close() atts = { 'file_name': f.name, context.EX_LAT: 'LAT', context.EX_LONG: 'LONG' } caj1 = workflow.ConfigAwareJob(JOBS[LOADCSVEXPOSURE], atts_to_add=atts) atts = {'var': 'con_test', 'value': 'yeah'} caj2 = workflow.ConfigAwareJob(JOBS[CONSTANT], atts_to_add=atts) atts = {'var': 'con2_test', 'value': 30} caj3 = workflow.ConfigAwareJob(JOBS[CONSTANT], atts_to_add=atts) calc_list = [caj1, caj2, caj3, CALCS['add_test']] cont_in = context.Context() the_pipeline = pipeline.PipeLine(calc_list) the_pipeline.run(cont_in) cont_dict = cont_in.save_exposure_atts(f2.name) os.remove(f2.name) if parallel.STATE.rank == 0: self.assertTrue(allclose(cont_dict['c_test'], asarray([33., 66.]))) self.assertEqual(cont_dict['BUILDING'].tolist(), ['TAB', 'DSG']) self.assertTrue( allclose(cont_dict['con2_test'], asarray([30., 30.]))) self.assertEqual(cont_dict['con_test'].tolist(), ['yeah', 'yeah']) os.remove(f.name)
def test_start(self): config_list = [{ templates.TEMPLATE: templates.DEFAULT }, { 'constant': { 'var': 'c_test', 'value': 7 } }, { 'add_test': None }, { 'multiply_test': None }] a_test = 5 b_test = 2 cont_in = context.Context() cont_in.exposure_att = {'a_test': a_test, 'b_test': b_test} cont_in.exposure_long = scipy.asarray([11.0]) cont_in = hazimp.start(config_list=config_list, cont_in=cont_in) self.assertEqual(cont_in.exposure_att['d_test'], 35) self.assertEqual(cont_in.exposure_att['c_test'], 7)
def test_clip_exposure(self): # These points are in the HazImp notebook. lat_long = array([[-23, 110], [-23, 130], [-23, 145], [-30, 110], [-35, 121], [-25, 139], [-30, 145], [-37, 130]]) num_points = lat_long.shape[0] shoes_array = arange(num_points * 2).reshape((-1, 2)) d3_array = arange(num_points * 2 * 3).reshape((-1, 2, 3)) id_array = arange(num_points) con = context.Context() sub_set = (4, 5) initial = {'shoes': shoes_array, 'd3': d3_array, misc.INTID: id_array} con.exposure_att = initial con.exposure_lat = lat_long[:, 0] con.exposure_long = lat_long[:, 1] # After this clip the only points that remain are; # [-35, 121] & [-25, 139], indexed as 4 & 5 con.clip_exposure(min_lat=-36, max_lat=-24, min_long=120, max_long=140) actual = {} actual[context.EX_LAT] = lat_long[:, 0][sub_set, ...] actual[context.EX_LONG] = lat_long[:, 1][sub_set, ...] actual['shoes'] = shoes_array[sub_set, ...] actual['d3'] = d3_array[sub_set, ...] actual[misc.INTID] = id_array[sub_set, ...] for key in con.exposure_att: self.assertTrue(allclose(con.exposure_att[key], actual[key]))
def test_load_rasters(self): # Write a file to test f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='HAZIMPtest_jobs', delete=False) f.write('exposure_latitude, exposure_longitude, ID, haz_0, haz_1\n') f.write('8.1, 0.1, 1, 4, 40\n') f.write('7.9, 1.5, 2, -9999, -9999\n') f.write('8.9, 2.9, 3, 6, 60\n') f.write('8.9, 3.1, 4, -9999, -9999\n') f.write('9.9, 2.9, 5, -9999, -9999\n') f.close() inst = JOBS[LOADCSVEXPOSURE] con_in = context.Context() con_in.exposure_lat = con_in.exposure_long = None con_in.exposure_att = {} test_kwargs = {'file_name': f.name, 'use_parallel': False} inst(con_in, **test_kwargs) os.remove(f.name) # Write a hazard file f = tempfile.NamedTemporaryFile(suffix='.aai', prefix='HAZIMPtest_jobs', delete=False) f.write('ncols 3 \r\n') f.write('nrows 2 \r\n') f.write('xllcorner +0. \r\n') f.write('yllcorner +8. \r\n') f.write('cellsize 1 \r\n') f.write('NODATA_value -9999 \r\n') f.write('1 2 -9999 \r\n') f.write('4 5 6 ') f.close() files = [f.name] # Write another hazard file f = tempfile.NamedTemporaryFile(suffix='.aai', prefix='HAZIMPtest_jobs', delete=False) f.write('ncols 3 \r\n') f.write('nrows 2 \r\n') f.write('xllcorner +0. \r\n') f.write('yllcorner +8. \r\n') f.write('cellsize 1 \r\n') f.write('NODATA_value -9999 \r\n') f.write('10 20 -9999 \r\n') f.write('40 50 60 ') f.close() files.append(f.name) haz_v = 'haz_v' inst = JOBS[LOADRASTER] test_kwargs = {'file_list': files, 'attribute_label': haz_v} inst(con_in, **test_kwargs) the_nans = isnan(con_in.exposure_att[haz_v]) con_in.exposure_att[haz_v][the_nans] = -9999 actual = asarray( [con_in.exposure_att['haz_0'], con_in.exposure_att['haz_1']]) actual = rollaxis(actual, 1) msg = "con_in.exposure_att[haz_av] " \ + str(con_in.exposure_att[haz_v]) msg += "\n actual " + str(actual) self.assertTrue(allclose(con_in.exposure_att[haz_v], actual), msg) for a_file in files: os.remove(a_file)