def calculate(self, logger): """ Calculate this damage event. """ from lizard_damage import calc logger.info("event %s" % (self,)) logger.info(" - month %s, floodtime %s" % ( self.floodmonth, self.floodtime)) # Read damage table dt_path, damage_table = self.scenario.read_damage_table() logger.info('damage table: %s' % dt_path) calc_type = self.scenario.calc_type or calculation.CALC_TYPE_MAX # Use the calculator from lizard-damage-calculation for the # actual calculation. calculator = calculation.DamageCalculator( ahn_data_dir=os.path.join( settings.LIZARD_DAMAGE_DATA_ROOT, 'data_ahn'), lgn_data_dir=os.path.join( settings.LIZARD_DAMAGE_DATA_ROOT, 'data_lgn'), alternative_heights_dataset=( self.scenario.alternative_heights_dataset), alternative_landuse_dataset=( self.scenario.alternative_landuse_dataset), table=damage_table, get_roads_flooded_for_tile_and_code= Roads.get_roads_flooded_for_tile_and_code, calc_type=calc_type, road_grid_codes=Roads.ROAD_GRIDCODE, logger=logger) waterlevel_ascfiles = [ dewl.waterlevel_path for dewl in self.damageeventwaterlevel_set.all()] calculator.set_waterlevel_datafiles(waterlevel_ascfiles) # Track global results overall_area = collections.defaultdict(float) overall_damage = collections.defaultdict(float) roads_flooded_global = {i: collections.defaultdict(float) for i in Roads.ROAD_GRIDCODE} all_leaves = calculator.get_ahn_leaves() result_collector = results.ResultCollector( self.workdir, all_leaves, logger) result_collector.save_file_for_zipfile(dt_path, 'dt.cfg') for (ahn_name, extent, ds_height, landuse_ma, depth_ma, damage, area, result, roads_flooded_for_tile) in ( calculator.calculate_for_all_leaves( month=self.floodmonth, floodtime=self.floodtime, repairtime_roads=self.repairtime_roads, repairtime_buildings=self.repairtime_buildings)): logger.info("Recording results for tile {}...".format(ahn_name)) # Keep track of flooded roads for code, roads_flooded in roads_flooded_for_tile.iteritems(): for road, flooded_m2 in roads_flooded.iteritems(): roads_flooded_global[code][road] += flooded_m2 logger.debug("result sum: %f" % result.sum()) arcname = 'schade_{}'.format(ahn_name) if self.repetition_time: arcname += '_T%.1f' % self.repetition_time result_collector.save_ma( ahn_name, result, result_type='damage', ds_template=ds_height, repetition_time=self.repetition_time) result_collector.save_csv_data_for_zipfile( 'schade_{}.csv'.format(ahn_name), dict( damage=damage, area=area, damage_table=damage_table, meta=[ ['schade module versie', tools.version()], ['waterlevel', waterlevel_ascfiles[0]], ['damage table', dt_path], ['maand', str(self.floodmonth)], ['duur overstroming (s)', str(self.floodtime)], ['hersteltijd wegen (s)', str(self.repairtime_roads)], ['hersteltijd bebouwing (s)', str(self.repairtime_buildings)], ['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]], ['ahn_name', ahn_name], ])) for k in damage.keys(): overall_damage[k] += damage[k] for k in area.keys(): overall_area[k] += area[k] # Generate vrt + geotiff out of the .asc files. result_collector.build_damage_geotiff() # Only after all tiles have been processed, calculate overall indirect # Road damage. This is not visible in the per-tile-damagetable. roads_flooded_over_threshold = [] for code, roads_flooded in roads_flooded_global.iteritems(): damage_data = damage_table.data[code] for road, area in roads_flooded.iteritems(): if area >= 100: roads_flooded_over_threshold.append(road) indirect_road_damage = ( damage_data.to_indirect_damage( calculation.CALC_TYPES[calc_type]) * damage_data.to_gamma_repairtime(self.repairtime_roads)) logger.info( '%s - %s - %s: %.2f ind' % ( damage_data.code, damage_data.source, damage_data.description, indirect_road_damage, ), ) logger.info( ('track indirect road damage: scenario slug {}, ' + 'roadid {}, damage {}').format( self.scenario.slug, road, indirect_road_damage, )) overall_damage[code] += indirect_road_damage result_collector.save_csv_data_for_zipfile( 'schade_totaal.csv', dict( damage=overall_damage, area=overall_area, damage_table=damage_table, meta=[ ['schade module versie', tools.version()], ['waterlevel', waterlevel_ascfiles[0]], ['damage table', dt_path], ['maand', str(self.floodmonth)], ['duur overstroming (s)', str(self.floodtime)], ['hersteltijd wegen (s)', str(self.repairtime_roads)], ['hersteltijd bebouwing (s)', str(self.repairtime_buildings)], ['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]], ], include_total=True)) result_collector.finalize() DamageEventResult.create_from_result_collector(self, result_collector) # Save a table in a JSON string to show in the interface self.parsed_table = calc.result_as_dict( damage=overall_damage, area=overall_area, damage_table=damage_table) # Save min and max height, for legend self.min_height = result_collector.mins.get('height') self.max_height = result_collector.maxes.get('height') self.save() result_collector.cleanup_tmp_dir() return True, result_collector.riskmap_data # success
def version(self): return tools.version()
def calc_damage_for_waterlevel( repetition_time, ds_wl_filenames, dt_path=None, month=9, floodtime=20*3600, repairtime_roads=None, repairtime_buildings=None, calc_type=CALC_TYPE_MAX, logger=logger): """ Calculate damage for provided waterlevel file. in: - waterlevel file (provided by user) - damage table (optionally provided by user, else default) - AHN: models.AhnIndex refer to ahn tiles available on <settings.DATA_ROOT>/... - month, floodtime (s), repairtime_roads/buildings (s): provided by user, used by calc.calculate out: - per ahn tile an .asc and .csv (see write_result and write_table) - schade_totaal.csv (see write_table) """ cdict_water_depth = { 'red': ((0.0, 170./256, 170./256), (0.5, 65./256, 65./256), (1.0, 4./256, 4./256)), 'green': ((0.0, 200./256, 200./256), (0.5, 120./256, 120./256), (1.0, 65./256, 65./256)), 'blue': ((0.0, 255./256, 255./256), (0.5, 221./256, 221./256), (1.0, 176./256, 176./256)), } zip_result = [] # store all the file references for zipping. {'filename': .., 'arcname': ...} img_result = [] landuse_slugs = [] # slugs for landuse geo images height_slugs = [] # slugs for height geo images depth_slugs = [] # slugs for depth geo images logger.info('water level: %s' % ds_wl_filenames) logger.info('damage table: %s' % dt_path) output_zipfile = mkstemp_and_close() waterlevel_ascfiles = ds_wl_filenames correct_ascfiles(waterlevel_ascfiles) # TODO: do it elsewhere waterlevel_datasets = [raster.import_dataset(waterlevel_ascfile, 'AAIGrid') for waterlevel_ascfile in waterlevel_ascfiles] logger.info('waterlevel_ascfiles: %r' % waterlevel_ascfiles) logger.info('waterlevel_datasets: %r' % waterlevel_datasets) for fn, ds in zip(waterlevel_ascfiles, waterlevel_datasets): if ds is None: logger.error('data source is not available,' ' please check folder %s' % fn) return if dt_path is None: damage_table_path = 'data/damagetable/dt.cfg' dt_path = os.path.join(settings.BUILDOUT_DIR, damage_table_path) with open(dt_path) as cfg: dt = table.DamageTable.read_cfg(cfg) zip_result.append({'filename': dt_path, 'arcname': 'dt.cfg'}) overall_area = collections.defaultdict(float) overall_damage = collections.defaultdict(float) roads_flooded_global = {i: {} for i in ROAD_GRIDCODE} result_images = [] # Images to be modified for indirect road damage. min_height = None max_height = None min_depth = 0.0 # Set defaults for testing... depth is always >= 0 max_depth = 0.1 ahn_indices = raster.get_ahn_indices(waterlevel_datasets[0]) for ahn_index in ahn_indices: ahn_name = ahn_index.bladnr logger.info("Preparing calculation for tile %s..." % ahn_name) # Prepare data for calculation try: alldata = raster.get_calc_data( waterlevel_datasets=waterlevel_datasets, method=settings.RASTER_SOURCE, floodtime=floodtime, ahn_name=ahn_name, logger=logger, ) if alldata is None: logger.warning( 'Skipping damage calculation for {}'.format(ahn_name), ) continue landuse, depth, geo, floodtime_px, ds_height, height = alldata except: # Log this error and all previous normal logs, instead of hard crashing logger.error('Exception') for exception_line in traceback.format_exc().split('\n'): logger.error(exception_line) return extent = ahn_index.the_geom.extent # 1000x1250 meters = 2000x2500 pixels # For height map new_min_height = np.amin(height) if min_height is None or new_min_height < min_height: min_height = new_min_height new_max_height = np.amax(height) if max_height is None or new_max_height < max_height: max_height = new_max_height # For depth map new_min_depth = np.amin(depth) if min_depth is None or new_min_depth < min_depth: min_depth = new_min_depth new_max_depth = np.amax(depth) if max_depth is None or new_max_depth < max_depth: max_depth = new_max_depth # For landuse map landuse_slug = slug_for_landuse(ahn_name) landuse_slugs.append(landuse_slug) # part of result # note: multiple objects with the same slug can exist if they # enter this function at the same time if models.GeoImage.objects.filter(slug=landuse_slug).count() == 0: logger.info("Generating landuse GeoImage: %s" % landuse_slug) models.GeoImage.from_data_with_legend(landuse_slug, landuse.data, extent, landuse_legend()) # Result is a np array damage, count, area, result, roads_flooded_for_tile = calculate( use=landuse, depth=depth, geo=geo, calc_type=calc_type, table=dt, month=month, floodtime=floodtime_px, repairtime_roads=repairtime_roads, repairtime_buildings=repairtime_buildings, logger=logger, ) for code, roads_flooded in roads_flooded_for_tile.iteritems(): for road, flooded_m2 in roads_flooded.iteritems(): if road in roads_flooded_global[code]: roads_flooded_global[code][road]['area'] += flooded_m2 else: roads_flooded_global[code][road] = dict( shape=depth.shape, area=flooded_m2, geo=geo, ) logger.debug("result sum: %f" % result.sum()) arcname = 'schade_{}'.format(ahn_name) if repetition_time: arcname += '_T%.1f' % repetition_time asc_result = {'filename': mkstemp_and_close(), 'arcname': arcname + '.asc', 'delete_after': True} write_result( name=asc_result['filename'], ma_result=result, ds_template=ds_height, ) zip_result.append(asc_result) # Generate image in .png # Subdivide tiles x_tiles = 1 y_tiles = 1 tile_x_size = (extent[2] - extent[0]) / x_tiles tile_y_size = (extent[3] - extent[1]) / y_tiles result_tile_size_x = result.shape[1] / x_tiles result_tile_size_y = result.shape[0] / y_tiles #print ('result tile size: %r %r' % (result_tile_size_x, result_tile_size_y)) for tile_x in range(x_tiles): for tile_y in range(y_tiles): e = (extent[0] + tile_x * tile_x_size, extent[1] + tile_y * tile_y_size, extent[0] + (tile_x + 1) * tile_x_size, extent[1] + (tile_y + 1) * tile_y_size) # We are writing a png + pgw now, but in the task a tiff will be created and uploaded base_filename = mkstemp_and_close() image_result = { 'filename_tif': base_filename + '.tif', 'filename_png': base_filename + '.png', 'filename_pgw': base_filename + '.pgw', 'dstname': 'schade_%s_' + ahn_name + '.png', 'extent': ahn_index.extent_wgs84(e=e)} # %s is for the damage_event.slug write_image( name=image_result['filename_png'], values=result[(y_tiles-tile_y-1)*result_tile_size_y:(y_tiles-tile_y)*result_tile_size_y, (tile_x)*result_tile_size_x:(tile_x+1)*result_tile_size_x]) result_images.append({ 'extent': e, 'path': image_result['filename_png'], }) models.write_pgw( name=image_result['filename_pgw'], extent=e) img_result.append(image_result) csv_result = {'filename': mkstemp_and_close(), 'arcname': arcname + '.csv', 'delete_after': True} meta = [ ['schade module versie', tools.version()], ['waterlevel', waterlevel_ascfiles[0]], ['damage table', dt_path], ['maand', str(month)], ['duur overstroming (s)', str(floodtime)], ['hersteltijd wegen (s)', str(repairtime_roads)], ['hersteltijd bebouwing (s)', str(repairtime_buildings)], ['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]], ['ahn_name', ahn_name], ] write_table( name=csv_result['filename'], damage=damage, area=area, dt=dt, meta=meta, ) zip_result.append(csv_result) for k in damage.keys(): if k in overall_damage: overall_damage[k] += damage[k] else: overall_damage[k] = damage[k] for k in area.keys(): if k in overall_area: overall_area[k] += area[k] else: overall_area[k] = area[k] add_to_zip(output_zipfile, zip_result, logger) zip_result = [] def generate_height_tiles(): """ This is in a subroutine because it must be possible to not use it. """ logger.info('Generating height and depth tiles...') logger.debug( 'height min max=%f, %f, depth min max=%f, %f' % (min_height, max_height, min_depth, max_depth), ) for ahn_index in ahn_indices: ahn_name = ahn_index.bladnr height_slug = slug_for_height(ahn_name, min_height, max_height) height_slugs.append(height_slug) # part of result geo_image_depth_count = -1 try: depth_slug = slug_for_depth(ahn_name, min_depth, max_depth) depth_slugs.append(depth_slug) # part of result geo_image_depth_count = models.GeoImage.objects.filter( slug=depth_slug, ).count() except: logger.warning('GeoImage for depth failed because of fully masked') geo_image_height_count = models.GeoImage.objects.filter( slug=height_slug, ).count() if (geo_image_height_count == 0 or geo_image_depth_count == 0): # Copied from above try: alldata = raster.get_calc_data( waterlevel_datasets=waterlevel_datasets, method=settings.RASTER_SOURCE, floodtime=floodtime, ahn_name=ahn_name, logger=logger, ) if alldata is None: logger.warning( 'Skipping height tiles generation for {}' .format(ahn_name), ) continue landuse, depth, geo, floodtime_px, ds_height, height = alldata except: # Log this error and all previous normal logs, # instead of hard crashing logger.error('Exception') for exception_line in traceback.format_exc().split('\n'): logger.error(exception_line) return if geo_image_height_count == 0: # 1000x1250 meters = 2000x2500 pixels extent = ahn_index.the_geom.extent # Actually create tile logger.info("Generating height GeoImage: %s" % height_slug) models.GeoImage.from_data_with_min_max( height_slug, height, extent, min_height, max_height) if geo_image_depth_count == 0: # 1000x1250 meters = 2000x2500 pixels extent = ahn_index.the_geom.extent # Actually create tile logger.info("Generating depth GeoImage: %s" % depth_slug) try: #if isinstance(min_depth, float) and isinstance(max_depth, float): models.GeoImage.from_data_with_min_max( depth_slug, depth, extent, min_depth, max_depth, cdict=cdict_water_depth) depth_slugs.append(depth_slug) # part of result except: logger.info("Skipped depth GeoImage because of masked only or unknown error") if ((min_height is not None) and (max_height is not None) and (min_depth is not None) and (max_depth is not None)): generate_height_tiles() # Only after all tiles have been processed, calculate overall indirect # Road damage. This is not visible in the per-tile-damagetable. roads_flooded_over_threshold = [] for code, roads_flooded in roads_flooded_global.iteritems(): for road, info in roads_flooded.iteritems(): if info['area'] >= 100: roads_flooded_over_threshold.append(road) overall_damage[code] += ( dt.data[code].to_indirect_damage(CALC_TYPES[calc_type]) * dt.data[code].to_gamma_repairtime(repairtime_roads) ) def add_roads_to_image(roads, image_path, extent): """ This function could be moved to top level. """ # Get old image that needs indirect road damage visualized image = Image.open(result_image['path']) # Rasterize all roads that have indirect damage size = image.size geotransform = [ extent[0], (extent[2] - extent[0]) / size[0], 0, extent[3], 0, (extent[1] - extent[3]) / size[1], ] roadgrid = raster.get_mask( roads=road_objects, shape=image.size[::-1], geo=(b'', geotransform), ) # Paste it into the old image and overwrite the image file rgba = np.uint8([[[0, 0, 0, 153]]]) * roadgrid.reshape( roadgrid.shape[0], roadgrid.shape[1], 1 ) image_roads_rgb = Image.fromarray(rgba[:, :, 0:3]) image_roads_mask = Image.fromarray(rgba[:, :, 3]) image.paste(image_roads_rgb, None, image_roads_mask) image.save(result_image['path']) road_objects = models.Roads.objects.filter( pk__in=roads_flooded_over_threshold, ) for result_image in result_images: add_roads_to_image( roads=road_objects, image_path=result_image['path'], extent=result_image['extent'], ) csv_result = {'filename': mkstemp_and_close(), 'arcname': 'schade_totaal.csv', 'delete_after': True} meta = [ ['schade module versie', tools.version()], ['waterlevel', waterlevel_ascfiles[0]], ['damage table', dt_path], ['maand', str(month)], ['duur overstroming (s)', str(floodtime)], ['hersteltijd wegen (s)', str(repairtime_roads)], ['hersteltijd bebouwing (s)', str(repairtime_buildings)], ['berekening', {1: 'Minimum', 2: 'Maximum', 3: 'Gemiddelde'}[calc_type]], ] write_table( name=csv_result['filename'], damage=overall_damage, area=overall_area, dt=dt, meta=meta, include_total=True, ) result_table = result_as_dict( name=csv_result['filename'], damage=overall_damage, area=overall_area, damage_table=dt ) zip_result.append(csv_result) add_to_zip(output_zipfile, zip_result, logger) zip_result = [] logger.info('zipfile: %s' % output_zipfile) return output_zipfile, img_result, result_table, landuse_slugs, height_slugs, depth_slugs