def test_download_file_from_s3_if_needed(self):
s3 = get_s3_client()
s3.create_bucket(Bucket='bucket')
file_path = download_file_from_s3_if_needed('/local/path/to/file')
self.assertEqual(file_path, '/local/path/to/file')
s3.put_object(Bucket='bucket', Key='subdir/file.ext', Body='')
file_path = \
download_file_from_s3_if_needed('/vsis3/bucket/subdir/file.ext')
self.assertTrue('file.ext' in file_path)
s3.put_object(Bucket='bucket', Key='subdir/file.shp', Body='')
s3.put_object(Bucket='bucket', Key='subdir/file.shx', Body='')
s3.put_object(Bucket='bucket', Key='subdir/file.dbf', Body='')
s3.put_object(Bucket='bucket', Key='subdir/file.prj', Body='')
file_path = \
download_file_from_s3_if_needed('/vsis3/bucket/subdir/file.shp')
self.assertTrue('file.shp' in file_path)
# Check zip file
directory_path = get_temporary_directory()
zip_file_path = os.path.join(directory_path, 'file.zip')
with ZipFile(zip_file_path, 'w') as zipObj:
for file_name in ['test.shp', 'test.shx', 'test.dbf',
'test.prj', 'test.cpg']:
file_path = os.path.join(directory_path, file_name)
print('Test file', file=open(file_path, 'w'))
zipObj.write(file_path)
s3.upload_file(zip_file_path, 'bucket', 'subdir/file.zip')
file_path = \
download_file_from_s3_if_needed('/vsis3/bucket/subdir/file.zip')
self.assertTrue('test.shp' in file_path)
def start(config_list=None, config_file=None, cont_in=None):
"""
Run the HazImp tool, based on the config info.
:param config_list: The configuration info, as a list.
:param config_file: The configuration info, as a file location.
:param cont_in: Only used in testing. A context instance.
:returns: The config dictionary.
"""
if config_file:
config_file = misc.download_file_from_s3_if_needed(config_file)
config_list = config.read_config_file(config_file)
if isinstance(config_list, dict):
msg = "Bad configuration file. \n"
msg += "Add a dash ( - ) before each variable. e.g. - template: flood"
raise RuntimeError(msg)
if config_list is None:
raise RuntimeError('No configuration information.')
if cont_in is None:
cont_in = context.Context()
# TODO: Make the entity name a tailored variable
cont_in.set_prov_label("HazImp_analysis")
calc_jobs = config.instance_builder(config_list)
the_pipeline = pipeline.PipeLine(calc_jobs)
the_pipeline.run(cont_in)
return cont_in
def __call__(self,
context,
file_name,
exposure_latitude=None,
exposure_longitude=None,
use_parallel=True):
"""
Read a csv exposure file into the context object.
:param context: The context instance, used to move data around.
:param file_name: The csv file to load.
:param exposure_latitude: the title string of the latitude column.
:param exposure_longitude: the title string of the longitude column.
Content return:
exposure_att: Add the file values into this dictionary.
key: column titles
value: column values, except the title
"""
file_name = misc.download_file_from_s3_if_needed(file_name)
dt = misc.get_file_mtime(file_name)
expent = context.prov.entity(
":Exposure data", {
'dcterms:title': 'Exposure data',
'prov:type': 'void:Dataset',
'prov:generatedAtTime': dt,
'prov:atLocation': os.path.basename(file_name)
})
context.prov.used(context.provlabel, expent)
data_frame = parallel.csv2dict(file_name, use_parallel=use_parallel)
# FIXME Need to do better error handling
# FIXME this function can only be called once.
# Multiple calls will corrupt the context data.
if exposure_latitude is None:
lat_key = EX_LAT
else:
lat_key = exposure_latitude
try:
context.exposure_lat = data_frame[lat_key].values
del data_frame[lat_key]
except KeyError:
msg = "No Exposure latitude column labelled '%s'." % lat_key
raise RuntimeError(msg)
if exposure_longitude is None:
long_key = EX_LONG
else:
long_key = exposure_longitude
try:
context.exposure_long = data_frame[long_key].values
del data_frame[long_key]
except KeyError:
msg = "No Exposure longitude column labelled '%s'." % long_key
raise RuntimeError(msg)
context.exposure_att = data_frame
def save_aggregation(self,
filename,
boundaries,
impactcode,
boundarycode,
categories,
fields,
use_parallel=True):
"""
Save data aggregated to geospatial regions
:param str filename: Destination filename
:param bool use_parallel: True for parallel behaviout, which
is only node 0 writing to file
"""
LOGGER.info("Saving aggregated data")
boundaries = misc.download_file_from_s3_if_needed(boundaries)
[filename, bucket_name, bucket_key] = \
misc.create_temp_file_path_for_s3(filename)
write_dict = self.exposure_att.copy()
dt = datetime.now().strftime(DATEFMT)
atts = {
"prov:type": "void:Dataset",
"prov:atLocation": os.path.basename(boundaries),
"prov:generatedAtTime": misc.get_file_mtime(boundaries),
"void:boundary_code": boundarycode
}
bdyent = self.prov.entity(":Aggregation boundaries", atts)
aggact = self.prov.activity(":AggregationByRegions", dt, None, {
'prov:type': "Spatial aggregation",
'void:functions': repr(fields)
})
aggatts = {
"prov:type": "void:Dataset",
"prov:atLocation": os.path.basename(filename),
"prov:generatedAtTime": dt
}
aggfileent = self.prov.entity(":AggregationFile", aggatts)
self.prov.used(aggact, bdyent)
self.prov.wasInformedBy(aggact, self.provlabel)
self.prov.wasGeneratedBy(aggfileent, aggact)
if parallel.STATE.rank == 0 or not use_parallel:
aggregate.choropleth(write_dict, boundaries, impactcode,
boundarycode, filename, fields, categories)
misc.upload_to_s3_if_applicable(filename, bucket_name, bucket_key)
if (bucket_name is not None and bucket_key is not None
and bucket_key.endswith('.shp')):
[rootname, ext] = os.path.splitext(filename)
base_bucket_key = bucket_key[:-len(ext)]
misc.upload_to_s3_if_applicable(rootname + '.dbf', bucket_name,
base_bucket_key + '.dbf')
misc.upload_to_s3_if_applicable(rootname + '.shx', bucket_name,
base_bucket_key + '.shx')
misc.upload_to_s3_if_applicable(rootname + '.prj', bucket_name,
base_bucket_key + '.prj')
misc.upload_to_s3_if_applicable(rootname + '.cpg', bucket_name,
base_bucket_key + '.cpg', True)
def __call__(self,
context,
attribute_label,
file_list,
clip_exposure2all_hazards=False,
file_format=None,
variable=None,
no_data_value=None):
"""
Load one or more files and get the value for all the
exposure points. All files have to be of the same attribute.
Alternatively a numeric array of the raster data can be passed in.
:param context: The context instance, used to move data around.
:param attribute_label: The string to be associated with this data.
:param clip_exposure2all_hazards: True if the exposure data is
clippped to the hazard data, so no hazard values are ignored.
:param file_list: A list of files or a single file to be loaded.
:param no_data_value: Values in the raster that represent no data.
Context return:
exposure_att: Add the file values into this dictionary.
key: column titles
value: column values, except the title
"""
if isinstance(file_list, str):
file_list = [file_list]
for f in file_list:
f = misc.download_file_from_s3_if_needed(f)
dt = misc.get_file_mtime(f)
atts = {
"dcterms:title": "Source hazard data",
"prov:type": "prov:Dataset",
"prov:atLocation": os.path.basename(f),
"prov:format": os.path.splitext(f)[1].replace('.', ''),
"prov:generatedAtTime": dt,
}
if file_format == 'nc' and variable:
atts['prov:variable'] = variable
hazent = context.prov.entity(":Hazard data", atts)
context.prov.used(context.provlabel, hazent)
if file_format == 'nc' and variable:
file_list = misc.mod_file_list(file_list, variable)
file_data, extent = raster_module.files_raster_data_at_points(
context.exposure_long, context.exposure_lat, file_list)
file_data[file_data == no_data_value] = np.NAN
context.exposure_att[attribute_label] = file_data
if clip_exposure2all_hazards:
# Clipping the exposure points after the data has been added.
# Not optimised for speed, but easy to implement.
context.clip_exposure(*extent)
def __call__(self,
context,
attribute_label,
clip_exposure2all_hazards=False,
file_list=None,
file_format=None,
variable=None,
raster=None,
upper_left_x=None,
upper_left_y=None,
cell_size=None,
no_data_value=None):
"""
Load one or more files and get the value for all the
exposure points. All files have to be of the same attribute.
Alternatively a numeric array of the raster data can be passed in.
:param context: The context instance, used to move data around.
:param attribute_label: The string to be associated with this data.
:param clip_exposure2all_hazards: True if the exposure data is
clippped to the hazard data, so no hazard values are ignored.
:param file_list: A list of files or a single file to be loaded.
OR
:param raster: A 2D numeric array of the raster values, North is up.
:param upper_left_x: The longitude at the upper left corner.
:param upper_left_y: The latitude at the upper left corner.
:param cell_size: The cell size.
:param no_data_value: Values in the raster that represent no data.
Context return:
exposure_att: Add the file values into this dictionary.
key: column titles
value: column values, except the title
"""
# We need a file or a full set of raster info.
if file_list is None:
# The raster info is being passed as an array
assert raster is not None
assert upper_left_x is not None
assert upper_left_y is not None
assert cell_size is not None
assert no_data_value is not None
a_raster = raster_module.Raster.from_array(raster, upper_left_x,
upper_left_y, cell_size,
no_data_value)
if clip_exposure2all_hazards:
# Reduce the context to the hazard area
# before the raster info has been added to the context
extent = a_raster.extent()
context.clip_exposure(*extent)
file_data = a_raster.raster_data_at_points(context.exposure_long,
context.exposure_lat)
file_data = np.where(file_data == no_data_value, np.NAN, file_data)
context.exposure_att[attribute_label] = file_data
else:
if isinstance(file_list, str):
file_list = [file_list]
for f in file_list:
f = misc.download_file_from_s3_if_needed(f)
dt = misc.get_file_mtime(f)
atts = {
"dcterms:title": "Source hazard data",
"prov:type": "prov:Dataset",
"prov:atLocation": os.path.basename(f),
"prov:format": os.path.splitext(f)[1].replace('.', ''),
"prov:generatedAtTime": dt,
}
if file_format == 'nc' and variable:
atts['prov:variable'] = variable
hazent = context.prov.entity(":Hazard data", atts)
context.prov.used(context.provlabel, hazent)
if file_format == 'nc' and variable:
file_list = misc.mod_file_list(file_list, variable)
file_data, extent = raster_module.files_raster_data_at_points(
context.exposure_long, context.exposure_lat, file_list)
file_data[file_data == no_data_value] = np.NAN
context.exposure_att[attribute_label] = file_data
if clip_exposure2all_hazards:
# Clipping the exposure points after the data has been added.
# Not optimised for speed, but easy to implement.
context.clip_exposure(*extent)