def save_aggregation(self,
filename,
boundaries,
impactcode,
boundarycode,
categories,
fields,
use_parallel=True):
"""
Save data aggregated to geospatial regions
:param str filename: Destination filename
:param bool use_parallel: True for parallel behaviout, which
is only node 0 writing to file
"""
LOGGER.info("Saving aggregated data")
boundaries = misc.download_file_from_s3_if_needed(boundaries)
[filename, bucket_name, bucket_key] = \
misc.create_temp_file_path_for_s3(filename)
write_dict = self.exposure_att.copy()
dt = datetime.now().strftime(DATEFMT)
atts = {
"prov:type": "void:Dataset",
"prov:atLocation": os.path.basename(boundaries),
"prov:generatedAtTime": misc.get_file_mtime(boundaries),
"void:boundary_code": boundarycode
}
bdyent = self.prov.entity(":Aggregation boundaries", atts)
aggact = self.prov.activity(":AggregationByRegions", dt, None, {
'prov:type': "Spatial aggregation",
'void:functions': repr(fields)
})
aggatts = {
"prov:type": "void:Dataset",
"prov:atLocation": os.path.basename(filename),
"prov:generatedAtTime": dt
}
aggfileent = self.prov.entity(":AggregationFile", aggatts)
self.prov.used(aggact, bdyent)
self.prov.wasInformedBy(aggact, self.provlabel)
self.prov.wasGeneratedBy(aggfileent, aggact)
if parallel.STATE.rank == 0 or not use_parallel:
aggregate.choropleth(write_dict, boundaries, impactcode,
boundarycode, filename, fields, categories)
misc.upload_to_s3_if_applicable(filename, bucket_name, bucket_key)
if (bucket_name is not None and bucket_key is not None
and bucket_key.endswith('.shp')):
[rootname, ext] = os.path.splitext(filename)
base_bucket_key = bucket_key[:-len(ext)]
misc.upload_to_s3_if_applicable(rootname + '.dbf', bucket_name,
base_bucket_key + '.dbf')
misc.upload_to_s3_if_applicable(rootname + '.shx', bucket_name,
base_bucket_key + '.shx')
misc.upload_to_s3_if_applicable(rootname + '.prj', bucket_name,
base_bucket_key + '.prj')
misc.upload_to_s3_if_applicable(rootname + '.cpg', bucket_name,
base_bucket_key + '.cpg', True)
def test_choropleth_invalid_boundary(self):
with self.assertRaises(SystemExit) as context:
data_frame = pd.read_csv(str(CWD / 'data/exposure_with_loss.csv'))
choropleth(data_frame, str(CWD / 'data/boundaries.json'),
'MESHBLOCK_CODE_2011', 'INVALID', 'output.json',
{'structural': ['mean']}, True, None)
self.assertEqual(context.exception.code, 1)
def test_choropleth_creates_output(self):
for output in outputs_to_test:
with self.subTest(output):
data_frame = pd.read_csv(
str(CWD / 'data/exposure_with_loss.csv'))
temp_dir = tempfile.TemporaryDirectory()
filename = f'{temp_dir.name}/{output}'
try:
choropleth(data_frame, str(CWD / 'data/boundaries.json'),
'MESHBLOCK_CODE_2011', 'MB_CODE11', filename,
{'structural': ['mean']}, True, None)
self.assert_file_exists(filename)
finally:
temp_dir.cleanup()
def test_choropleth_invalid_driver(self):
data_frame = pd.read_csv(str(CWD / 'data/exposure_with_loss.csv'))
status = choropleth(data_frame, str(CWD / 'data/boundaries.json'),
'MESHBLOCK_CODE_2011', 'MB_CODE11',
'output.invalid', {'structural': ['mean']}, True,
None)
self.assertFalse(status)