def calculate(self, logger):
"""
Calculate this damage event.
"""
from lizard_damage import calc
logger.info("event %s" % (self,))
logger.info(" - month %s, floodtime %s" % (
self.floodmonth, self.floodtime))
# Read damage table
dt_path, damage_table = self.scenario.read_damage_table()
logger.info('damage table: %s' % dt_path)
calc_type = self.scenario.calc_type or calculation.CALC_TYPE_MAX
# Use the calculator from lizard-damage-calculation for the
# actual calculation.
calculator = calculation.DamageCalculator(
ahn_data_dir=os.path.join(
settings.LIZARD_DAMAGE_DATA_ROOT, 'data_ahn'),
lgn_data_dir=os.path.join(
settings.LIZARD_DAMAGE_DATA_ROOT, 'data_lgn'),
alternative_heights_dataset=(
self.scenario.alternative_heights_dataset),
alternative_landuse_dataset=(
self.scenario.alternative_landuse_dataset),
table=damage_table,
get_roads_flooded_for_tile_and_code=
Roads.get_roads_flooded_for_tile_and_code,
calc_type=calc_type,
road_grid_codes=Roads.ROAD_GRIDCODE,
logger=logger)
waterlevel_ascfiles = [
dewl.waterlevel_path for dewl in
self.damageeventwaterlevel_set.all()]
calculator.set_waterlevel_datafiles(waterlevel_ascfiles)
# Track global results
overall_area = collections.defaultdict(float)
overall_damage = collections.defaultdict(float)
roads_flooded_global = {i: collections.defaultdict(float)
for i in Roads.ROAD_GRIDCODE}
all_leaves = calculator.get_ahn_leaves()
result_collector = results.ResultCollector(
self.workdir, all_leaves, logger)
result_collector.save_file_for_zipfile(dt_path, 'dt.cfg')
for (ahn_name, extent, ds_height, landuse_ma, depth_ma, damage,
area, result, roads_flooded_for_tile) in (
calculator.calculate_for_all_leaves(
month=self.floodmonth,
floodtime=self.floodtime,
repairtime_roads=self.repairtime_roads,
repairtime_buildings=self.repairtime_buildings)):
logger.info("Recording results for tile {}...".format(ahn_name))
# Keep track of flooded roads
for code, roads_flooded in roads_flooded_for_tile.iteritems():
for road, flooded_m2 in roads_flooded.iteritems():
roads_flooded_global[code][road] += flooded_m2
logger.debug("result sum: %f" % result.sum())
arcname = 'schade_{}'.format(ahn_name)
if self.repetition_time:
arcname += '_T%.1f' % self.repetition_time
result_collector.save_ma(
ahn_name, result, result_type='damage', ds_template=ds_height,
repetition_time=self.repetition_time)
result_collector.save_csv_data_for_zipfile(
'schade_{}.csv'.format(ahn_name), dict(
damage=damage, area=area, damage_table=damage_table,
meta=[
['schade module versie', tools.version()],
['waterlevel', waterlevel_ascfiles[0]],
['damage table', dt_path],
['maand', str(self.floodmonth)],
['duur overstroming (s)', str(self.floodtime)],
['hersteltijd wegen (s)', str(self.repairtime_roads)],
['hersteltijd bebouwing (s)',
str(self.repairtime_buildings)],
['berekening',
{1: 'Minimum', 2: 'Maximum',
3: 'Gemiddelde'}[calc_type]],
['ahn_name', ahn_name],
]))
for k in damage.keys():
overall_damage[k] += damage[k]
for k in area.keys():
overall_area[k] += area[k]
# Generate vrt + geotiff out of the .asc files.
result_collector.build_damage_geotiff()
# Only after all tiles have been processed, calculate overall indirect
# Road damage. This is not visible in the per-tile-damagetable.
roads_flooded_over_threshold = []
for code, roads_flooded in roads_flooded_global.iteritems():
damage_data = damage_table.data[code]
for road, area in roads_flooded.iteritems():
if area >= 100:
roads_flooded_over_threshold.append(road)
indirect_road_damage = (
damage_data.to_indirect_damage(
calculation.CALC_TYPES[calc_type]) *
damage_data.to_gamma_repairtime(self.repairtime_roads))
logger.info(
'%s - %s - %s: %.2f ind' %
(
damage_data.code,
damage_data.source,
damage_data.description,
indirect_road_damage,
),
)
logger.info(
('track indirect road damage: scenario slug {}, ' +
'roadid {}, damage {}').format(
self.scenario.slug, road, indirect_road_damage,
))
overall_damage[code] += indirect_road_damage
result_collector.save_csv_data_for_zipfile(
'schade_totaal.csv', dict(
damage=overall_damage,
area=overall_area,
damage_table=damage_table,
meta=[
['schade module versie', tools.version()],
['waterlevel', waterlevel_ascfiles[0]],
['damage table', dt_path],
['maand', str(self.floodmonth)],
['duur overstroming (s)', str(self.floodtime)],
['hersteltijd wegen (s)', str(self.repairtime_roads)],
['hersteltijd bebouwing (s)',
str(self.repairtime_buildings)],
['berekening',
{1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]],
],
include_total=True))
result_collector.finalize()
DamageEventResult.create_from_result_collector(self, result_collector)
# Save a table in a JSON string to show in the interface
self.parsed_table = calc.result_as_dict(
damage=overall_damage,
area=overall_area,
damage_table=damage_table)
# Save min and max height, for legend
self.min_height = result_collector.mins.get('height')
self.max_height = result_collector.maxes.get('height')
self.save()
result_collector.cleanup_tmp_dir()
return True, result_collector.riskmap_data # success
def version(self):
return tools.version()
def calc_damage_for_waterlevel(
repetition_time,
ds_wl_filenames,
dt_path=None,
month=9, floodtime=20*3600,
repairtime_roads=None, repairtime_buildings=None,
calc_type=CALC_TYPE_MAX,
logger=logger):
"""
Calculate damage for provided waterlevel file.
in:
- waterlevel file (provided by user)
- damage table (optionally provided by user, else default)
- AHN: models.AhnIndex refer to ahn tiles available on
<settings.DATA_ROOT>/...
- month, floodtime (s), repairtime_roads/buildings (s): provided
by user, used by calc.calculate
out:
- per ahn tile an .asc and .csv (see write_result and write_table)
- schade_totaal.csv (see write_table)
"""
cdict_water_depth = {
'red': ((0.0, 170./256, 170./256),
(0.5, 65./256, 65./256),
(1.0, 4./256, 4./256)),
'green': ((0.0, 200./256, 200./256),
(0.5, 120./256, 120./256),
(1.0, 65./256, 65./256)),
'blue': ((0.0, 255./256, 255./256),
(0.5, 221./256, 221./256),
(1.0, 176./256, 176./256)),
}
zip_result = [] # store all the file references for zipping. {'filename': .., 'arcname': ...}
img_result = []
landuse_slugs = [] # slugs for landuse geo images
height_slugs = [] # slugs for height geo images
depth_slugs = [] # slugs for depth geo images
logger.info('water level: %s' % ds_wl_filenames)
logger.info('damage table: %s' % dt_path)
output_zipfile = mkstemp_and_close()
waterlevel_ascfiles = ds_wl_filenames
correct_ascfiles(waterlevel_ascfiles) # TODO: do it elsewhere
waterlevel_datasets = [raster.import_dataset(waterlevel_ascfile, 'AAIGrid')
for waterlevel_ascfile in waterlevel_ascfiles]
logger.info('waterlevel_ascfiles: %r' % waterlevel_ascfiles)
logger.info('waterlevel_datasets: %r' % waterlevel_datasets)
for fn, ds in zip(waterlevel_ascfiles, waterlevel_datasets):
if ds is None:
logger.error('data source is not available,'
' please check folder %s' % fn)
return
if dt_path is None:
damage_table_path = 'data/damagetable/dt.cfg'
dt_path = os.path.join(settings.BUILDOUT_DIR, damage_table_path)
with open(dt_path) as cfg:
dt = table.DamageTable.read_cfg(cfg)
zip_result.append({'filename': dt_path, 'arcname': 'dt.cfg'})
overall_area = collections.defaultdict(float)
overall_damage = collections.defaultdict(float)
roads_flooded_global = {i: {} for i in ROAD_GRIDCODE}
result_images = [] # Images to be modified for indirect road damage.
min_height = None
max_height = None
min_depth = 0.0 # Set defaults for testing... depth is always >= 0
max_depth = 0.1
ahn_indices = raster.get_ahn_indices(waterlevel_datasets[0])
for ahn_index in ahn_indices:
ahn_name = ahn_index.bladnr
logger.info("Preparing calculation for tile %s..." % ahn_name)
# Prepare data for calculation
try:
alldata = raster.get_calc_data(
waterlevel_datasets=waterlevel_datasets,
method=settings.RASTER_SOURCE,
floodtime=floodtime,
ahn_name=ahn_name,
logger=logger,
)
if alldata is None:
logger.warning(
'Skipping damage calculation for {}'.format(ahn_name),
)
continue
landuse, depth, geo, floodtime_px, ds_height, height = alldata
except:
# Log this error and all previous normal logs, instead of hard crashing
logger.error('Exception')
for exception_line in traceback.format_exc().split('\n'):
logger.error(exception_line)
return
extent = ahn_index.the_geom.extent # 1000x1250 meters = 2000x2500 pixels
# For height map
new_min_height = np.amin(height)
if min_height is None or new_min_height < min_height:
min_height = new_min_height
new_max_height = np.amax(height)
if max_height is None or new_max_height < max_height:
max_height = new_max_height
# For depth map
new_min_depth = np.amin(depth)
if min_depth is None or new_min_depth < min_depth:
min_depth = new_min_depth
new_max_depth = np.amax(depth)
if max_depth is None or new_max_depth < max_depth:
max_depth = new_max_depth
# For landuse map
landuse_slug = slug_for_landuse(ahn_name)
landuse_slugs.append(landuse_slug) # part of result
# note: multiple objects with the same slug can exist if they
# enter this function at the same time
if models.GeoImage.objects.filter(slug=landuse_slug).count() == 0:
logger.info("Generating landuse GeoImage: %s" % landuse_slug)
models.GeoImage.from_data_with_legend(landuse_slug, landuse.data, extent, landuse_legend())
# Result is a np array
damage, count, area, result, roads_flooded_for_tile = calculate(
use=landuse, depth=depth, geo=geo, calc_type=calc_type,
table=dt, month=month, floodtime=floodtime_px,
repairtime_roads=repairtime_roads,
repairtime_buildings=repairtime_buildings,
logger=logger,
)
for code, roads_flooded in roads_flooded_for_tile.iteritems():
for road, flooded_m2 in roads_flooded.iteritems():
if road in roads_flooded_global[code]:
roads_flooded_global[code][road]['area'] += flooded_m2
else:
roads_flooded_global[code][road] = dict(
shape=depth.shape,
area=flooded_m2,
geo=geo,
)
logger.debug("result sum: %f" % result.sum())
arcname = 'schade_{}'.format(ahn_name)
if repetition_time:
arcname += '_T%.1f' % repetition_time
asc_result = {'filename': mkstemp_and_close(), 'arcname': arcname + '.asc',
'delete_after': True}
write_result(
name=asc_result['filename'],
ma_result=result,
ds_template=ds_height,
)
zip_result.append(asc_result)
# Generate image in .png
# Subdivide tiles
x_tiles = 1
y_tiles = 1
tile_x_size = (extent[2] - extent[0]) / x_tiles
tile_y_size = (extent[3] - extent[1]) / y_tiles
result_tile_size_x = result.shape[1] / x_tiles
result_tile_size_y = result.shape[0] / y_tiles
#print ('result tile size: %r %r' % (result_tile_size_x, result_tile_size_y))
for tile_x in range(x_tiles):
for tile_y in range(y_tiles):
e = (extent[0] + tile_x * tile_x_size, extent[1] + tile_y * tile_y_size,
extent[0] + (tile_x + 1) * tile_x_size, extent[1] + (tile_y + 1) * tile_y_size)
# We are writing a png + pgw now, but in the task a tiff will be created and uploaded
base_filename = mkstemp_and_close()
image_result = {
'filename_tif': base_filename + '.tif',
'filename_png': base_filename + '.png',
'filename_pgw': base_filename + '.pgw',
'dstname': 'schade_%s_' + ahn_name + '.png',
'extent': ahn_index.extent_wgs84(e=e)} # %s is for the damage_event.slug
write_image(
name=image_result['filename_png'],
values=result[(y_tiles-tile_y-1)*result_tile_size_y:(y_tiles-tile_y)*result_tile_size_y,
(tile_x)*result_tile_size_x:(tile_x+1)*result_tile_size_x])
result_images.append({
'extent': e,
'path': image_result['filename_png'],
})
models.write_pgw(
name=image_result['filename_pgw'],
extent=e)
img_result.append(image_result)
csv_result = {'filename': mkstemp_and_close(), 'arcname': arcname + '.csv',
'delete_after': True}
meta = [
['schade module versie', tools.version()],
['waterlevel', waterlevel_ascfiles[0]],
['damage table', dt_path],
['maand', str(month)],
['duur overstroming (s)', str(floodtime)],
['hersteltijd wegen (s)', str(repairtime_roads)],
['hersteltijd bebouwing (s)', str(repairtime_buildings)],
['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]],
['ahn_name', ahn_name],
]
write_table(
name=csv_result['filename'],
damage=damage,
area=area,
dt=dt,
meta=meta,
)
zip_result.append(csv_result)
for k in damage.keys():
if k in overall_damage:
overall_damage[k] += damage[k]
else:
overall_damage[k] = damage[k]
for k in area.keys():
if k in overall_area:
overall_area[k] += area[k]
else:
overall_area[k] = area[k]
add_to_zip(output_zipfile, zip_result, logger)
zip_result = []
def generate_height_tiles():
"""
This is in a subroutine because it
must be possible to not use it.
"""
logger.info('Generating height and depth tiles...')
logger.debug(
'height min max=%f, %f, depth min max=%f, %f' %
(min_height, max_height, min_depth, max_depth),
)
for ahn_index in ahn_indices:
ahn_name = ahn_index.bladnr
height_slug = slug_for_height(ahn_name, min_height, max_height)
height_slugs.append(height_slug) # part of result
geo_image_depth_count = -1
try:
depth_slug = slug_for_depth(ahn_name, min_depth, max_depth)
depth_slugs.append(depth_slug) # part of result
geo_image_depth_count = models.GeoImage.objects.filter(
slug=depth_slug,
).count()
except:
logger.warning('GeoImage for depth failed because of fully masked')
geo_image_height_count = models.GeoImage.objects.filter(
slug=height_slug,
).count()
if (geo_image_height_count == 0 or geo_image_depth_count == 0):
# Copied from above
try:
alldata = raster.get_calc_data(
waterlevel_datasets=waterlevel_datasets,
method=settings.RASTER_SOURCE,
floodtime=floodtime,
ahn_name=ahn_name,
logger=logger,
)
if alldata is None:
logger.warning(
'Skipping height tiles generation for {}'
.format(ahn_name),
)
continue
landuse, depth, geo, floodtime_px, ds_height, height = alldata
except:
# Log this error and all previous normal logs,
# instead of hard crashing
logger.error('Exception')
for exception_line in traceback.format_exc().split('\n'):
logger.error(exception_line)
return
if geo_image_height_count == 0:
# 1000x1250 meters = 2000x2500 pixels
extent = ahn_index.the_geom.extent
# Actually create tile
logger.info("Generating height GeoImage: %s" % height_slug)
models.GeoImage.from_data_with_min_max(
height_slug, height, extent, min_height, max_height)
if geo_image_depth_count == 0:
# 1000x1250 meters = 2000x2500 pixels
extent = ahn_index.the_geom.extent
# Actually create tile
logger.info("Generating depth GeoImage: %s" % depth_slug)
try: #if isinstance(min_depth, float) and isinstance(max_depth, float):
models.GeoImage.from_data_with_min_max(
depth_slug, depth, extent, min_depth, max_depth,
cdict=cdict_water_depth)
depth_slugs.append(depth_slug) # part of result
except:
logger.info("Skipped depth GeoImage because of masked only or unknown error")
if ((min_height is not None) and
(max_height is not None) and
(min_depth is not None) and
(max_depth is not None)):
generate_height_tiles()
# Only after all tiles have been processed, calculate overall indirect
# Road damage. This is not visible in the per-tile-damagetable.
roads_flooded_over_threshold = []
for code, roads_flooded in roads_flooded_global.iteritems():
for road, info in roads_flooded.iteritems():
if info['area'] >= 100:
roads_flooded_over_threshold.append(road)
overall_damage[code] += (
dt.data[code].to_indirect_damage(CALC_TYPES[calc_type]) *
dt.data[code].to_gamma_repairtime(repairtime_roads)
)
def add_roads_to_image(roads, image_path, extent):
""" This function could be moved to top level. """
# Get old image that needs indirect road damage visualized
image = Image.open(result_image['path'])
# Rasterize all roads that have indirect damage
size = image.size
geotransform = [
extent[0],
(extent[2] - extent[0]) / size[0],
0,
extent[3],
0,
(extent[1] - extent[3]) / size[1],
]
roadgrid = raster.get_mask(
roads=road_objects,
shape=image.size[::-1],
geo=(b'', geotransform),
)
# Paste it into the old image and overwrite the image file
rgba = np.uint8([[[0, 0, 0, 153]]]) * roadgrid.reshape(
roadgrid.shape[0], roadgrid.shape[1], 1
)
image_roads_rgb = Image.fromarray(rgba[:, :, 0:3])
image_roads_mask = Image.fromarray(rgba[:, :, 3])
image.paste(image_roads_rgb, None, image_roads_mask)
image.save(result_image['path'])
road_objects = models.Roads.objects.filter(
pk__in=roads_flooded_over_threshold,
)
for result_image in result_images:
add_roads_to_image(
roads=road_objects,
image_path=result_image['path'],
extent=result_image['extent'],
)
csv_result = {'filename': mkstemp_and_close(), 'arcname': 'schade_totaal.csv',
'delete_after': True}
meta = [
['schade module versie', tools.version()],
['waterlevel', waterlevel_ascfiles[0]],
['damage table', dt_path],
['maand', str(month)],
['duur overstroming (s)', str(floodtime)],
['hersteltijd wegen (s)', str(repairtime_roads)],
['hersteltijd bebouwing (s)', str(repairtime_buildings)],
['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]],
]
write_table(
name=csv_result['filename'],
damage=overall_damage,
area=overall_area,
dt=dt,
meta=meta,
include_total=True,
)
result_table = result_as_dict(
name=csv_result['filename'],
damage=overall_damage,
area=overall_area,
damage_table=dt
)
zip_result.append(csv_result)
add_to_zip(output_zipfile, zip_result, logger)
zip_result = []
logger.info('zipfile: %s' % output_zipfile)
return output_zipfile, img_result, result_table, landuse_slugs, height_slugs, depth_slugs
