def process_result(
logger, damage_event, damage_event_index, result, scenario_name):
errors = 0
# result[0] is the result zip file name in temp dir.
with open(result[0], 'rb') as doc_file:
try:
if damage_event.result:
logger.warning('Deleting existing results...')
damage_event.result.delete() # Delete old results
logger.info('Saving results...')
damage_event.result.save(
'%s%i.zip' % (
slugify(scenario_name),
damage_event_index + 1),
File(doc_file), save=True)
damage_event.save()
except:
logger.error('Exception saving zipfile. Too big?')
for exception_line in traceback.format_exc().split('\n'):
logger.error(exception_line)
errors = 1
os.remove(result[0]) # remove temp file, whether it was saved
# or not
# result[2] is the table in a data structure
damage_event.table = json.dumps(result[2])
# Store references to GeoImage objects
damage_event.set_slugs(
landuse_slugs=','.join(result[3]),
height_slugs=','.join(result[4]),
depth_slugs=','.join(result[5]))
damage_event.save()
# result[1] is a list of png files to be uploaded to the django db.
if damage_event.damageeventresult_set.count() >= 0:
logger.warning("Removing old images...")
for damage_event_result in (
damage_event.damageeventresult_set.all()):
damage_event_result.image.delete()
damage_event_result.delete()
for img in result[1]:
# convert filename_png to geotiff,
#import pdb; pdb.set_trace()
logger.info('Warping png to tif... %s' % img['filename_png'])
command = (
'gdalwarp %s %s -t_srs "+proj=latlong '
'+datum=WGS83" -s_srs "%s"' % (
img['filename_png'], img['filename_tif'], RD.strip()))
logger.info(command)
# Warp png file, output is tif.
subprocess.call([
'gdalwarp', img['filename_png'], img['filename_tif'],
'-t_srs', "+proj=latlong +datum=WGS84",
'-s_srs', RD.strip()])
img['extent'] = extent_from_geotiff(img['filename_tif'])
# Convert it back to png
convert_tif_to_png(img['filename_tif'], img['filename_png'])
damage_event_result = DamageEventResult(
damage_event=damage_event,
west=img['extent'][0],
south=img['extent'][1],
east=img['extent'][2],
north=img['extent'][3])
logger.info('Uploading %s...' % img['filename_png'])
with open(img['filename_png'], 'rb') as img_file:
damage_event_result.image.save(
img['dstname'] % damage_event.slug,
File(img_file), save=True)
damage_event_result.save()
os.remove(img['filename_png'])
os.remove(img['filename_pgw'])
os.remove(img['filename_tif'])
logger.info('Result has %d images' % len(result[1]))
return errors
def calculate_damage(damage_scenario_id, username=None, taskname=None, loglevel=20):
"""
Main calculation task.
"""
logger = logging.getLogger(taskname)
logger.info("calculate damage")
damage_scenario = DamageScenario.objects.get(pk=damage_scenario_id)
logger.info("scenario: %d, %s" % (damage_scenario.id, str(damage_scenario)))
logger.info("calculating...")
logger.info("scenario %s" % (damage_scenario.name))
damage_scenario.status = damage_scenario.SCENARIO_STATUS_INPROGRESS
damage_scenario.save()
errors = 0
for damage_event_index, damage_event in enumerate(
damage_scenario.damageevent_set.all(),
):
# ds_wl_filename = os.path.join(
# settings.DATA_ROOT, 'waterlevel', 'ws_test1.asc',
# )
# damage_event.waterlevel.path
ds_wl_filenames = [dewl.waterlevel.path for dewl in
damage_event.damageeventwaterlevel_set.all()]
logger.info("event %s" % (damage_event))
#logger.info(" - waterlevel: %s" % (damage_event.waterlevel))
logger.info(" - month %s, floodtime %s" % (
damage_event.floodmonth, damage_event.floodtime))
if damage_scenario.damagetable:
dt_path = damage_scenario.damagetable.path
else:
# Default
dt_path = os.path.join(settings.BUILDOUT_DIR, 'data/damagetable/dt.cfg')
result = calc.calc_damage_for_waterlevel(
repetition_time=damage_event.repetition_time,
ds_wl_filenames=ds_wl_filenames,
dt_path=dt_path,
month=damage_event.floodmonth,
floodtime=damage_event.floodtime,
repairtime_roads=damage_event.repairtime_roads,
repairtime_buildings=damage_event.repairtime_buildings,
calc_type=damage_scenario.calc_type,
logger=logger)
if result:
# result[0] is the result zip file name in temp dir.
with open(result[0], 'rb') as doc_file:
try:
if damage_event.result:
logger.warning('Deleting existing results...')
damage_event.result.delete() # Delete old results
logger.info('Saving results...')
damage_event.result.save('%s%i.zip' % (
slugify(damage_scenario.name), damage_event_index + 1),
File(doc_file),save=True)
damage_event.save()
except:
logger.error('Exception saving zipfile. Too big?')
for exception_line in traceback.format_exc().split('\n'):
logger.error(exception_line)
errors += 1
os.remove(result[0]) # remove temp file, whether it was saved or not
# result[2] is the table in a data structure
damage_event.table = json.dumps(result[2])
damage_event.landuse_slugs = ','.join(result[3]) # Store references to GeoImage objects
damage_event.height_slugs = ','.join(result[4]) # Store references to GeoImage objects
damage_event.depth_slugs = ','.join(result[5]) # Store references to GeoImage objects
damage_event.save()
# result[1] is a list of png files to be uploaded to the django db.
if damage_event.damageeventresult_set.count() >= 0:
logger.warning("Removing old images...")
for damage_event_result in damage_event.damageeventresult_set.all():
damage_event_result.image.delete()
damage_event_result.delete()
for img in result[1]:
# convert filename_png to geotiff,
#import pdb; pdb.set_trace()
logger.info('Warping png to tif... %s' % img['filename_png'])
command = 'gdalwarp %s %s -t_srs "+proj=latlong +datum=WGS83" -s_srs "%s"' % (
img['filename_png'], img['filename_tif'], RD.strip())
logger.info(command)
# Warp png file, output is tif.
subprocess.call([
'gdalwarp', img['filename_png'], img['filename_tif'],
'-t_srs', "+proj=latlong +datum=WGS84", '-s_srs', RD.strip()])
img['extent'] = extent_from_geotiff(img['filename_tif'])
# Convert it back to png
#subprocess.call([
# 'convert', img['filename_tif'], img['filename_png']])
convert_tif_to_png(img['filename_tif'], img['filename_png'])
#logger.info('Creating damage event results...')
#for img in result[1]:
damage_event_result = DamageEventResult(
damage_event=damage_event,
west=img['extent'][0],
south=img['extent'][1],
east=img['extent'][2],
north=img['extent'][3])
logger.info('Uploading %s...' % img['filename_png'])
with open(img['filename_png'], 'rb') as img_file:
damage_event_result.image.save(img['dstname'] % damage_event.slug,
File(img_file), save=True)
damage_event_result.save()
os.remove(img['filename_png'])
os.remove(img['filename_pgw'])
os.remove(img['filename_tif'])
logger.info('Result has %d images' % len(result[1]))
else:
errors += 1
# Calculate risk maps
if damage_scenario.scenario_type == 4:
risk.create_risk_map(damage_scenario=damage_scenario, logger=logger)
# Roundup
damage_scenario.status = damage_scenario.SCENARIO_STATUS_DONE
damage_scenario.save()
if errors == 0:
logger.info("creating email task for scenario %d" % damage_scenario.id)
subject = 'STOWA Schade Calculator: Resultaten beschikbaar voor scenario %s ' % damage_scenario.name
send_email_to_task(damage_scenario.id, 'email_ready', subject, username=username)
logger.info("finished")
else:
logger.info("there were errors in scenario %d" % damage_scenario.id)
logger.info("creating email task for error")
subject = 'STOWA Schade Calculator: scenario %s heeft fouten' % damage_scenario.name
send_email_to_task(damage_scenario.id, 'email_error', subject, username=username)
send_email_to_task(damage_scenario.id, 'email_error', subject, username=username,
email='*****@*****.**')
logger.info("finished with errors")
return 'failure'