def read_his_file(sobek_id, presentationlayer): """ - Return [time_steps, values] read in the his-file and belonging to the sobek_id and result_id """ pl = presentationlayer his = cache.get('his_' + str(presentationlayer.id)) if his == None: zip_name = external_file_location( pl.presentationshape.value_source.file_location) input_file = ZipFile(zip_name, "r") if presentationlayer.presentationtype.geo_source_filter: filename = pl.presentationtype.geo_source_filter else: filename = input_file.filelist[0].filename his = HISFile( Stream(input_file.read(external_file_location(filename)))) input_file.close() cache.set('his_' + str(presentationlayer.id), his, 30) sobek_id = ( presentationlayer.presentationtype.value_source_id_prefix + sobek_id) #else: # sobek_id = 'p_' + sobek_id field = presentationlayer.presentationtype.field_set.get( source_type=Field.SOURCE_TYPE_VALUE_SOURCE_PARAM) timeserie = his.get_timeseries( sobek_id, field.name_in_source, None, None, list) t0 = timeserie[0][0] time_steps = [timedelta_to_float(t - t0) * 24 for (t, v) in timeserie] if presentationlayer.presentationtype.absolute: values = [abs(v) for (t, v) in timeserie] else: values = [v for (t, v) in timeserie] #Return data return [time_steps, values]
def read_his_file(sobek_id, presentationlayer): """ - Return [time_steps, values] read in the his-file and belonging to the sobek_id and result_id """ pl = presentationlayer his = cache.get('his_' + str(presentationlayer.id)) if his == None: zip_name = external_file_location( pl.presentationshape.value_source.file_location) input_file = ZipFile(zip_name, "r") if presentationlayer.presentationtype.geo_source_filter: filename = pl.presentationtype.geo_source_filter else: filename = input_file.filelist[0].filename his = HISFile(Stream(input_file.read( external_file_location(filename)))) input_file.close() cache.set('his_' + str(presentationlayer.id), his, 30) sobek_id = (presentationlayer.presentationtype.value_source_id_prefix + sobek_id) #else: # sobek_id = 'p_' + sobek_id field = presentationlayer.presentationtype.field_set.get( source_type=Field.SOURCE_TYPE_VALUE_SOURCE_PARAM) timeserie = his.get_timeseries(sobek_id, field.name_in_source, None, None, list) t0 = timeserie[0][0] time_steps = [timedelta_to_float(t - t0) * 24 for (t, v) in timeserie] if presentationlayer.presentationtype.absolute: values = [abs(v) for (t, v) in timeserie] else: values = [v for (t, v) in timeserie] #Return data return [time_steps, values]
def calc_damage(scenario_id): log.debug('read django and lizard settings') source_dir = Setting.objects.get(key='source_dir').value result_dir = Setting.objects.get(key='destination_dir').value presentation_dir = Setting.objects.get(key='presentation_dir').value log.debug('read scenario and file settings') dest_shape = os.path.join('flooding', 'scenario', str(scenario_id), 'embank_damage.shp') scenario = Scenario.objects.get(pk=scenario_id) pt = PresentationType.objects.get(code='damage_embankments') try: #damage settings extwater_models = SobekModel.objects.filter(scenariobreach__scenario=scenario, embankment_damage_shape__isnull=False) if extwater_models.count() == 0: log.info('no externalwatermodel or embankment_damage_shape for this scenario. Scenario skipped') return True, 'skipped 1 ' else: model = extwater_models[0] result_waterlevel = Result.objects.filter(scenario=scenario, resulttype=13) if result_waterlevel.count() > 0: his_zip_name = os.path.join(result_dir, result_waterlevel[0].resultloc) else: log.warning('Not enough results for calculation of embankment damage. Scenario skipped') return True, 'skipped 2 ' except Result.DoesNotExist: log.warning('Not enough results for calculation of embankment damage. Scenario skipped') return True, 'skipped 3 ' shapefile_name = os.path.join(source_dir, model.embankment_damage_shape) log.debug('read damage settings') damage = {'onvhard': 500, # for each m 'verhard': 1000, # for each m 'huizen': 3000, # for each m2 'woonboot': 50000} # each start_damage = {'zonder verdediging': 0.5, 'met beschoeiing': 0.5, 'stalen damwand': 0.5, 'stortsteen': 0.5, 'woonboot': 1.0, 'kleidijk': 3.0, 'veendijk': 2.5, 'zanddijk': 2.0, 'Klei': 3.0, 'Veen': 2.5, 'Zand': 2.0, None: 0.5} full_damage = {'zonder verdediging': 1.0, 'met beschoeiing': 1.0, 'stalen damwand': 1.0, 'stortsteen': 2.0, 'woonboot': 1.0, 'kleidijk': 3.0, 'veendijk': 2.5, 'zanddijk': 2.0, 'Klei': 3.0, 'Veen': 2.5, 'Zand': 2.0, None: 1.0} log.debug('open source shapefile') drv = ogr.GetDriverByName('ESRI Shapefile') source = drv.Open(str(shapefile_name)) source_layer = source.GetLayer() log.debug('create dest shapefile') output_dir = os.path.dirname(os.path.join(presentation_dir, dest_shape)) if not os.path.isdir(output_dir): log.debug('create output dir' + output_dir) os.makedirs(output_dir) dest_shape_full = os.path.join(presentation_dir, dest_shape) if os.path.isfile(dest_shape_full): log.debug('shapefile already exist, delete shapefile.') os.remove(dest_shape_full) os.remove(dest_shape_full.replace('.shp', '.shx')) os.remove(dest_shape_full.replace('.shp', '.dbf')) os.remove(dest_shape_full.replace('.shp', '.prj')) dest = drv.CreateDataSource(str(dest_shape_full)) dest_srs = osr.SpatialReference() dest_srs.ImportFromProj4('+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') dest_layer = dest.CreateLayer( dest.GetName(), geom_type=ogr.wkbLineString, srs=dest_srs) log.debug('create geo-transformation') #source_srs = source_layer.GetSpatialRef() #transformation = ogr.osr.CoordinateTransformation(source_srs,dest_srs) log.debug('get field ids') if (source_layer.GetFeatureCount() > 0): feature = source_layer.next() id_index = feature.GetFieldIndex('ID_1') length_index = feature.GetFieldIndex('LENGTE') onverhard_index = feature.GetFieldIndex('ONVERH') verhard_index = feature.GetFieldIndex('VERHARD') huizen_index = feature.GetFieldIndex('HUIZEN') woonboot_index = feature.GetFieldIndex('WOONBOOT') kade_type_index = feature.GetFieldIndex('BSOORT') source_layer.ResetReading() log.debug("create 'source' fields in new shapefile") dest_layer.CreateField(ogr.FieldDefn('id', ogr.OFTString)) dest_layer.CreateField(ogr.FieldDefn('length', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('onverh', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('verhard', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('huizen', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('woonboot', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('kade_type', ogr.OFTString)) log.debug("create 'result' fields in new shapefile") dest_layer.CreateField(ogr.FieldDefn('daling', ogr.OFTReal)) dest_layer.CreateField(ogr.FieldDefn('perc_bezw', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_kade', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_bebouw', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_woonboot', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_totaal', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_max', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_perc', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_per_m', ogr.OFTReal)) #openen hisfile log.debug('read hisfile') input_file = ZipFile(his_zip_name, "r") for filename in input_file.namelist(): if filename.lower() == 'calcpnt.his': his = HISFile(StringIO.StringIO(input_file.read(filename))) break input_file.close() def get_perc_bezw(daling, start_damage, full_damage): if daling <= start_damage: return 0.0 elif daling >= full_damage: return 100.0 else: return 100 * (daling - start_damage) / (full_damage - start_damage) sum_damage = 0 for layer in source_layer: #opzetten feature feat = ogr.Feature(feature_def=dest_layer.GetLayerDefn()) feat.SetFID(layer.GetFID()) #omzetten geometry geom = layer.geometry() #geom.TransformTo(dest_srs) feat.SetGeometry(geom) #omzetten basisgegevens feat.SetField('id', layer.GetField(id_index)) feat.SetField('length', layer.GetField(length_index)) feat.SetField('onverh', layer.GetField(onverhard_index)) feat.SetField('verhard', layer.GetField(verhard_index)) feat.SetField('huizen', layer.GetField(huizen_index)) feat.SetField('woonboot', layer.GetField(woonboot_index)) feat.SetField('kade_type', layer.GetField(kade_type_index)) #uitlezen peildaling uit hisfile try: daling = his.get_timeseries('c_' + layer.GetField(id_index), 'Waterlevel', None, None, list)[0][1] - min([value for date, value in his.get_timeseries('c_' + layer.GetField(id_index), 'Waterlevel', None, None, list)]) except KeyError: log.debug('no value for %s' % layer.GetField(id_index)) daling = 0. feat.SetField('daling', float(daling)) #uitvoeren berekeningen type_besch = layer.GetField(kade_type_index) perc_bezw = get_perc_bezw( daling, start_damage[type_besch], full_damage[type_besch]) feat.SetField('perc_bezw', int(perc_bezw)) max_sch_kade = layer.GetField(onverhard_index) * damage['onvhard'] + layer.GetField(verhard_index) * damage['verhard'] feat.SetField('sch_kade', int(max_sch_kade * perc_bezw / 100)) max_sch_bebouwing = layer.GetField(huizen_index) * damage['huizen'] feat.SetField('sch_bebouw', int(max_sch_bebouwing * perc_bezw / 100)) max_sch_woonboot = layer.GetField(woonboot_index) * damage['woonboot'] sch_woonboot = max_sch_woonboot * get_perc_bezw( daling, start_damage['woonboot'], full_damage['woonboot']) / 100 feat.SetField('sch_woonboot', int(sch_woonboot)) sch_totaal = sch_woonboot + ( max_sch_bebouwing + max_sch_kade) * perc_bezw / 100 feat.SetField('sch_totaal', int(sch_totaal)) sum_damage += sch_totaal max_schade = max_sch_woonboot + max_sch_bebouwing + max_sch_kade feat.SetField('sch_max', int(max_schade)) if max_schade > 0: sch_perc = 100 * sch_totaal / max_schade else: sch_perc = -1 feat.SetField('sch_perc', int(sch_perc)) length = layer.GetField(length_index) if length > 0: sch_per_m = sch_totaal / length else: sch_per_m = -1 feat.SetField('sch_per_m', int(sch_per_m)) #wegschrijven dest_layer.CreateFeature(feat) dest_layer.SyncToDisk() # Clean up dest.Destroy() source.Destroy() pl, pl_new = PresentationLayer.objects.get_or_create( presentationtype=pt, scenario_presentationlayer__scenario=scenario, defaults={"value": sum_damage}) if pl_new: Scenario_PresentationLayer.objects.create(scenario=scenario, presentationlayer=pl) else: pl.value = sum_damage pl.save() link_type, _ = SourceLinkType.objects.get_or_create( name='flooding_scenario_embankment_damage') source_link, new = SourceLink.objects.get_or_create( link_id=str(scenario.id), sourcelinktype=link_type, type='fl_emb_dam_' + str(scenario.id)) source, new = source_link.presentationsource.get_or_create( type=PresentationSource.SOURCE_TYPE_SHAPEFILE) source.file_location = dest_shape source.t_original = datetime.datetime.fromtimestamp( os.stat(str(his_zip_name))[8]) source.t_source = datetime.datetime.now() source.save() ps, ps_new = PresentationShape.objects.get_or_create(presentationlayer=pl) ps.geo_source = source ps.save() log.debug("Finish task.") log.debug("close db connection to avoid an idle process.") db.close_connection() return True, "ok "
def get_or_create_value_presentation_source( scenario, pt, get_animation_info, check_timestamp_original_sourcefile): ''' create PresentationSource objects for given Flooding.Scenario and PresentationType input: scenario: Flooding.Scenario object pt: PresentationType object check_timestamp_original_sourcefile output: source object if there are no Results to fill the presentationLayer, this function throws an exception In case of an error, an exeption is thrown. ''' log.debug('!!!get_or_create_value_presentation_source') dest_dir = Setting.objects.get(key='DESTINATION_DIR').value presentation_dir = Setting.objects.get(key='PRESENTATION_DIR').value #source_dir = Setting.objects.get(key='SOURCE_DIR').value try: animation = {} result = scenario.result_set.get(resulttype__presentationtype=pt) link_type, _ = SourceLinkType.objects.get_or_create( name='flooding_scenario_value') source_link, new = SourceLink.objects.get_or_create( link_id=str(scenario.id), sourcelinktype=link_type, type='fl_rlt_' + str(result.id)) source, new = source_link.presentationsource.get_or_create( type=PresentationSource.SOURCE_TYPE_ZIPPED_HISFILE) if new or source.file_location is None: new = True elif not os.path.isfile( os.path.join(presentation_dir, rel_path( source.file_location))): new = True #source resultloc = result.resultloc.replace('\\', '/') source_file_name = os.path.join(dest_dir, resultloc) if check_timestamp_original_sourcefile and not new: log.debug('check timestamps') if os.path.isfile(source_file_name): source_file_last_modified = datetime.datetime.fromtimestamp( os.stat(source_file_name.encode('utf8'))[8]) #adding timestamps where this was not filled before if source.t_origin == None or source.t_source == None: source.t_origin = source_file_last_modified source.t_source = datetime.datetime.now() source.save() elif (source_file_last_modified > source.t_origin): log.info('source file has changed, recreate') new = True output_dir_name = os.path.join('flooding', 'scenario', str(scenario.id)) filename = source_file_name.replace('\\', '/').split('/')[-1] output_file_name = os.path.join(output_dir_name, filename) if new: log.debug('copy file') #output output_path = os.path.join(presentation_dir, output_dir_name) if not os.path.isdir(output_path): os.makedirs(output_path) destination_file_name = os.path.join(presentation_dir, output_file_name) log.debug('source file is {0}'.format(source_file_name)) log.debug('destination is {0}'.format(destination_file_name)) try: if filename[-3:].lower() == 'zip': copyfile(source_file_name, destination_file_name) else: dest = ZipFile(destination_file_name.encode('utf8')[:-3] + '.zip', mode="w", compression=ZIP_DEFLATED) dest.writestr(filename, file(source_file_name, 'rb').read()) dest.close() source.file_location = output_file_name source.t_original = datetime.datetime.fromtimestamp( os.stat(source_file_name.encode('utf8'))[8]) source.t_source = datetime.datetime.now() source.save() get_animation_info = True except IOError as e: source.delete() raise IOError(e) if get_animation_info: log.debug('get animation information') zip_name = os.path.join(presentation_dir, output_file_name) input_file = ZipFile(zip_name, "r") his = HISFile( Stream(input_file.read(input_file.filelist[0].filename))) input_file.close() delta = (his.get_datetime_of_timestep(1) - his.get_datetime_of_timestep(0)) animation['delta_timestep'] = (delta.days + float(delta.seconds) / (24 * 60 * 60)) animation['firstnr'] = 0 animation['lastnr'] = his.size() - 1 animation['startnr'] = 0 except Exception as e: log.error('Error generating %s: %s', pt, e) return False, None, None, None return True, source, animation, new
def service_get_wms_of_shape( request, width, height, bbox, presentationlayer_id, legend_id, timestep): """ width = int height = int bbox = tuple """ pl = get_object_or_404(PresentationLayer, pk=presentationlayer_id) if legend_id == -1: legend_id = pl.presentationtype.default_legend_id #presentation_dir = Setting.objects.get( key = 'presentation_dir' ).value #################### set up map ################################### log.debug('start setting up map ' + str(datetime.datetime.now())) m = mapnik.Map(width, height) spherical_mercator = ( '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 ' '+y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') m.srs = spherical_mercator m.background = mapnik.Color('transparent') #p = mapnik.Projection(spherical_mercator) log.debug('start setting up legend ' + str(datetime.datetime.now())) #################### set up legend ################################### mpl = cache.get('legend_' + str(legend_id)) if mpl == None: sdl = get_object_or_404(ShapeDataLegend, pk=legend_id) if pl.presentationtype.geo_type in [ PresentationType.GEO_TYPE_POLYGON, PresentationType.GEO_TYPE_LINE, PresentationType.GEO_TYPE_POINT]: sm = SymbolManager('media/flooding_presentation/symbols/') mpl = MapnikPointLegend(sdl, sm) cache.set('legend_' + str(legend_id), mpl, 300) mapnik_style = mpl.get_style() # get mapnik style with a set of rules. fields = mpl.get_presentationtype_fields() m.append_style('1', mapnik_style) log.debug('start setting up lijntje ' + str(datetime.datetime.now())) #################### supportive layers ################################### if SupportLayers.objects.filter( presentationtype=pl.presentationtype).count() > 0: supportive_layers = (pl.presentationtype.supported_presentationtype. supportive_presentationtype.all()) sl = mapnik.Style() rule_l = mapnik.Rule() rule_stk = mapnik.Stroke() rule_stk.color = mapnik.Color(3, 158, 137) rule_stk.line_cap = mapnik.line_cap.ROUND_CAP rule_stk.width = 2.0 rule_l.symbols.append(mapnik.LineSymbolizer(rule_stk)) sl.rules.append(rule_l) m.append_style('Line Style', sl) for spt in supportive_layers: log.debug('supportive layer with id: ' + str(spt.id)) # warning! works only for flooding scenarios if pl.scenario_set.count() > 0: scenario = pl.scenario_set.get() layers = PresentationLayer.objects.filter( presentationtype=spt, scenario=scenario) if len(layers) > 0: log.debug( 'supportive layer found for this presentationlayer') layer = layers[0] lyrl = mapnik.Layer('lines', spherical_mercator) lyrl.datasource = mapnik.Shapefile( file=external_file_location( layer.presentationshape.geo_source.file_location)) lyrl.styles.append('Line Style') m.layers.append(lyrl) #################### read data ################################### #read source and attach values lyr = mapnik.Layer('points', spherical_mercator) lyr.datasource = mapnik.PointDatasource() log.debug('ready setting up map ' + str(datetime.datetime.now())) log.debug('start reading point cache ' + str(datetime.datetime.now())) points = cache.get('model_nodes_' + str(presentationlayer_id) + '_' + str(timestep) + '_' + str(legend_id)) log.debug('ready reading point cache ' + str(datetime.datetime.now())) if points is None: log.debug('start reading points from shape and his file ' + str(datetime.datetime.now())) points = [] drv = ogr.GetDriverByName('ESRI Shapefile') shapefile_name = external_file_location( pl.presentationshape.geo_source.file_location) ds = drv.Open(shapefile_name) layer = ds.GetLayer() has_his_file_field = False has_geo_file_field = False geo_fields = [] log.debug('fields are: ' + str(fields)) for nr in fields: field = fields[nr] if field.source_type == Field.SOURCE_TYPE_VALUE_SOURCE_PARAM: has_his_file_field = True his_field = field #only his files yet supported. read needed data log.debug('start reading hiscache' + str(datetime.datetime.now())) his = cache.get('his_' + str(presentationlayer_id)) log.debug('ready reading hiscache' + str(datetime.datetime.now())) if his == None: log.debug('read hisfile' + str(datetime.datetime.now())) zip_name = external_file_location( pl.presentationshape.value_source.file_location) input_file = ZipFile(zip_name, "r") if pl.presentationtype.geo_source_filter: filename = pl.presentationtype.geo_source_filter else: filename = input_file.filelist[0].filename his = HISFile(Stream(input_file.read(filename))) input_file.close() log.debug('ready reading hisfile' + str(datetime.datetime.now())) cache.set('his_' + str(presentationlayer_id), his, 3000) values = his.get_values_timestep_by_index( his.get_parameter_index(his_field.name_in_source), timestep) elif field.source_type == Field.SOURCE_TYPE_GEO_SOURCE_COL: has_geo_file_field = True geo_fields.append(field) else: log.debug('field source type ' + field.source_type + ' not yet supported') if (layer.GetFeatureCount() > 0): feature = layer.next() id_index = feature.GetFieldIndex('id') for field in geo_fields: field.index_nr = feature.GetFieldIndex( str(field.name_in_source)) layer.ResetReading() ############################# place features in the right 'legend box' for feature in layer: point = feature.geometry() input_dict = {} if has_his_file_field: #get id form geosource, needed for reading hisfile id = (pl.presentationtype.value_source_id_prefix + str(feature.GetField(id_index).strip())) if pl.presentationtype.absolute: try: input_dict[his_field.name_in_source] = abs( values.get(id, None)) if values.get(id, None) > 1: pass except TypeError: input_dict[his_field.name_in_source] = None else: input_dict[his_field.name_in_source] = values.get(id, None) if has_geo_file_field: for field in geo_fields: input_dict[field.name_in_source] = feature.GetField( field.index_nr) rule_name = mpl.get_rule_name(input_dict) if pl.presentationtype.geo_type == 3: x = (point.GetX(0) + point.GetX(1)) / 2 y = (point.GetY(0) + point.GetY(1)) / 2 else: x = point.GetX() y = point.GetY() #rule_name = str('1_neerslag_64_0010000100_24x24_0_0') points.append((x, y, "NAME", rule_name)) # Clean up ds.Destroy() log.debug('ready reading points form shape en his file ' + str(datetime.datetime.now())) cache.set('model_nodes_' + str(presentationlayer_id) + '_' + str(timestep) + '_' + str(legend_id), points, 300) log.debug('start making memory datasource ' + str(datetime.datetime.now())) for x, y, name, rule_name in points: lyr.datasource.add_point(x, y, name, rule_name) log.debug('finish making memory datasource ' + str(datetime.datetime.now())) lyr.styles.append('1') m.layers.append(lyr) if presentationlayer_id in [62007, 62008]: m = mapnik.Map(width, height) spherical_mercator = mapnik.Projection( '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 ' '+y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') m.srs = spherical_mercator m.background = mapnik.Color('transparent') sl = mapnik.Style() rule_l = mapnik.Rule() rule_stk = mapnik.Stroke() rule_stk.color = mapnik.Color(3, 158, 137) rule_stk.line_cap = mapnik.line_cap.ROUND_CAP rule_stk.width = 2.0 rule_l.symbols.append(mapnik.LineSymbolizer(rule_stk)) sl.rules.append(rule_l) m.append_style('Line Style2', sl) scenario = pl.scenario_set.get() layers = PresentationLayer.objects.filter( presentationtype=spt, scenario=scenario) log.debug('supportive layer found for this presentationlayer') rds = mapnik.Projection( "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 " "+k=0.999908 +x_0=155000 +y_0=463000 +ellps=bessel " "+towgs84=565.237,50.0087,465.658,-0.406857,0.350733," "-1.87035,4.0812 +units=m +no_defs") lyrl = mapnik.Layer('lines', rds) # Commented out, variable 'presentation_dir' doesn't exist so this # cannot work. -- RG20120621 # lyrl.datasource = mapnik.Shapefile( # file=external_file_location( # str(presentation_dir + '\\' + # pl.presentationshape.geo_source.file_location))) lyrl.styles.append('Line Style2') m.layers.append(lyrl) ##################### render map ############################# m.zoom_to_box(mapnik.Envelope(*bbox)) # lyrl.envelope log.debug('start render map ' + str(datetime.datetime.now())) img = mapnik.Image(width, height) mapnik.render(m, img) log.debug('ready render map ' + str(datetime.datetime.now())) log.debug('start PIL ' + str(datetime.datetime.now())) # you can use this if you want te modify image with PIL imgPIL = Image.fromstring('RGBA', (width, height), img.tostring()) #imgPIL = imgPIL.convert('RGB') buffer = StringIO.StringIO() imgPIL.save(buffer, 'png') # ,transparency = 10 buffer.seek(0) response = HttpResponse(buffer.read()) log.debug('end PIL ' + str(datetime.datetime.now())) response['Content-type'] = 'image/png' log.debug('ready sending map ' + str(datetime.datetime.now())) return response
def get_or_create_value_presentation_source( scenario, pt, get_animation_info, check_timestamp_original_sourcefile): ''' create PresentationSource objects for given Flooding.Scenario and PresentationType input: scenario: Flooding.Scenario object pt: PresentationType object check_timestamp_original_sourcefile output: source object if there are no Results to fill the presentationLayer, this function throws an exception In case of an error, an exeption is thrown. ''' log.debug('!!!get_or_create_value_presentation_source') dest_dir = Setting.objects.get(key='DESTINATION_DIR').value presentation_dir = Setting.objects.get(key='PRESENTATION_DIR').value #source_dir = Setting.objects.get(key='SOURCE_DIR').value try: animation = {} result = scenario.result_set.get(resulttype__presentationtype=pt) link_type, _ = SourceLinkType.objects.get_or_create( name='flooding_scenario_value') source_link, new = SourceLink.objects.get_or_create( link_id=str(scenario.id), sourcelinktype=link_type, type='fl_rlt_' + str(result.id)) source, new = source_link.presentationsource.get_or_create( type=PresentationSource.SOURCE_TYPE_ZIPPED_HISFILE) if new or source.file_location is None: new = True elif not os.path.isfile( os.path.join(presentation_dir, rel_path(source.file_location))): new = True #source resultloc = result.resultloc.replace('\\', '/') source_file_name = os.path.join(dest_dir, resultloc) if check_timestamp_original_sourcefile and not new: log.debug('check timestamps') if os.path.isfile(source_file_name): source_file_last_modified = datetime.datetime.fromtimestamp( os.stat(str(source_file_name))[8]) #adding timestamps where this was not filled before if source.t_origin == None or source.t_source == None: source.t_origin = source_file_last_modified source.t_source = datetime.datetime.now() source.save() elif (source_file_last_modified > source.t_origin): log.info('source file has changed, recreate') new = True output_dir_name = os.path.join( 'flooding', 'scenario', str(scenario.id)) filename = source_file_name.replace('\\', '/').split('/')[-1] output_file_name = os.path.join(output_dir_name, filename) if new: log.debug('copy file') #output output_path = os.path.join(presentation_dir, output_dir_name) if not os.path.isdir(output_path): os.makedirs(output_path) destination_file_name = os.path.join( presentation_dir, output_file_name) log.debug('source file is ' + str(source_file_name)) log.debug('destination is ' + str(destination_file_name)) try: if filename[-3:].lower() == 'zip': copyfile(source_file_name, destination_file_name) else: dest = ZipFile(str(destination_file_name)[:-3] + '.zip', mode="w", compression=ZIP_DEFLATED) dest.writestr( filename, file(source_file_name, 'rb').read()) dest.close() source.file_location = output_file_name source.t_original = datetime.datetime.fromtimestamp( os.stat(str(source_file_name))[8]) source.t_source = datetime.datetime.now() source.save() get_animation_info = True except IOError as e: source.delete() raise IOError(e) if get_animation_info: log.debug('get animation information') zip_name = os.path.join(presentation_dir, output_file_name) input_file = ZipFile(zip_name, "r") his = HISFile( Stream(input_file.read(input_file.filelist[0].filename))) input_file.close() delta = (his.get_datetime_of_timestep(1) - his.get_datetime_of_timestep(0)) animation['delta_timestep'] = ( delta.days + float(delta.seconds) / (24 * 60 * 60)) animation['firstnr'] = 0 animation['lastnr'] = his.size() - 1 animation['startnr'] = 0 except Exception as e: log.error('error generation value source') log.error(','.join(map(str, e.args))) return False, None, None, None return True, source, animation, new
def service_get_wms_of_shape(request, width, height, bbox, presentationlayer_id, legend_id, timestep): """ width = int height = int bbox = tuple """ pl = get_object_or_404(PresentationLayer, pk=presentationlayer_id) if legend_id == -1: legend_id = pl.presentationtype.default_legend_id #presentation_dir = Setting.objects.get( key = 'presentation_dir' ).value #################### set up map ################################### log.debug('start setting up map ' + str(datetime.datetime.now())) m = mapnik.Map(width, height) spherical_mercator = ( '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 ' '+y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') m.srs = spherical_mercator m.background = mapnik.Color('transparent') #p = mapnik.Projection(spherical_mercator) log.debug('start setting up legend ' + str(datetime.datetime.now())) #################### set up legend ################################### mpl = cache.get('legend_' + str(legend_id)) if mpl == None: sdl = get_object_or_404(ShapeDataLegend, pk=legend_id) if pl.presentationtype.geo_type in [ PresentationType.GEO_TYPE_POLYGON, PresentationType.GEO_TYPE_LINE, PresentationType.GEO_TYPE_POINT ]: sm = SymbolManager('media/flooding_presentation/symbols/') mpl = MapnikPointLegend(sdl, sm) cache.set('legend_' + str(legend_id), mpl, 300) fields = mpl.get_presentationtype_fields() log.debug('start setting up lijntje ' + str(datetime.datetime.now())) #################### supportive layers ################################### if SupportLayers.objects.filter( presentationtype=pl.presentationtype).count() > 0: supportive_layers = (pl.presentationtype.supported_presentationtype. supportive_presentationtype.all()) sl = mapnik.Style() rule_l = mapnik.Rule() rule_stk = mapnik.Stroke() rule_stk.color = mapnik.Color(3, 158, 137) rule_stk.line_cap = mapnik.line_cap.ROUND_CAP rule_stk.width = 2.0 rule_l.symbols.append(mapnik.LineSymbolizer(rule_stk)) sl.rules.append(rule_l) m.append_style('Line Style', sl) for spt in supportive_layers: log.debug('supportive layer with id: ' + str(spt.id)) # warning! works only for flooding scenarios if pl.scenario_set.count() > 0: scenario = pl.scenario_set.get() layers = PresentationLayer.objects.filter(presentationtype=spt, scenario=scenario) if len(layers) > 0: log.debug( 'supportive layer found for this presentationlayer') layer = layers[0] lyrl = mapnik.Layer('lines', spherical_mercator) lyrl.datasource = mapnik.Shapefile( file=external_file_location( layer.presentationshape.geo_source.file_location)) lyrl.styles.append('Line Style') m.layers.append(lyrl) #################### read data ################################### #read source and attach values log.debug('ready setting up map ' + str(datetime.datetime.now())) log.debug('start reading point cache ' + str(datetime.datetime.now())) points = cache.get('model_nodes_' + str(presentationlayer_id) + '_' + str(timestep) + '_' + str(legend_id)) log.debug('ready reading point cache ' + str(datetime.datetime.now())) if points is None: log.debug('start reading points from shape and his file ' + str(datetime.datetime.now())) points = [] drv = ogr.GetDriverByName('ESRI Shapefile') shapefile_name = external_file_location( pl.presentationshape.geo_source.file_location) ds = drv.Open(shapefile_name) layer = ds.GetLayer() has_his_file_field = False has_geo_file_field = False geo_fields = [] log.debug('fields are: ' + str(fields)) for nr in fields: field = fields[nr] if field.source_type == Field.SOURCE_TYPE_VALUE_SOURCE_PARAM: has_his_file_field = True his_field = field #only his files yet supported. read needed data log.debug('start reading hiscache' + str(datetime.datetime.now())) his = cache.get('his_' + str(presentationlayer_id)) log.debug('ready reading hiscache' + str(datetime.datetime.now())) if his == None: log.debug('read hisfile' + str(datetime.datetime.now())) zip_name = external_file_location( pl.presentationshape.value_source.file_location) input_file = ZipFile(zip_name, "r") if pl.presentationtype.geo_source_filter: filename = pl.presentationtype.geo_source_filter else: filename = input_file.filelist[0].filename his = HISFile(Stream(input_file.read(filename))) input_file.close() log.debug('ready reading hisfile' + str(datetime.datetime.now())) cache.set('his_' + str(presentationlayer_id), his, 3000) values = his.get_values_timestep_by_index( his.get_parameter_index(his_field.name_in_source), timestep) elif field.source_type == Field.SOURCE_TYPE_GEO_SOURCE_COL: has_geo_file_field = True geo_fields.append(field) else: log.debug('field source type ' + field.source_type + ' not yet supported') if (layer.GetFeatureCount() > 0): feature = layer.next() id_index = feature.GetFieldIndex('id') for field in geo_fields: field.index_nr = feature.GetFieldIndex( str(field.name_in_source)) layer.ResetReading() ############################# place features in the right 'legend box' for feature in layer: point = feature.geometry() input_dict = {} if has_his_file_field: #get id form geosource, needed for reading hisfile id = (pl.presentationtype.value_source_id_prefix + str(feature.GetField(id_index).strip())) if pl.presentationtype.absolute: try: input_dict[his_field.name_in_source] = abs( values.get(id, None)) if values.get(id, None) > 1: pass except TypeError: input_dict[his_field.name_in_source] = None else: input_dict[his_field.name_in_source] = values.get(id, None) if has_geo_file_field: for field in geo_fields: input_dict[field.name_in_source] = feature.GetField( field.index_nr) rule_name = mpl.get_rule_name(input_dict) if pl.presentationtype.geo_type == 3: x = (point.GetX(0) + point.GetX(1)) / 2 y = (point.GetY(0) + point.GetY(1)) / 2 else: x = point.GetX() y = point.GetY() #rule_name = str('1_neerslag_64_0010000100_24x24_0_0') points.append((x, y, "NAME", rule_name)) # Clean up ds.Destroy() log.debug('ready reading points form shape en his file ' + str(datetime.datetime.now())) cache.set( 'model_nodes_' + str(presentationlayer_id) + '_' + str(timestep) + '_' + str(legend_id), points, 300) log.debug('start making memory datasource ' + str(datetime.datetime.now())) lyr = mapnik.Layer('Points', spherical_mercator) m.append_style('Points legend', mpl.get_style()) memory_ds = mapnik.MemoryDatasource() #lyr.datasource context = mapnik.Context() context.push("name") next_id = 1 for x, y, name, rule_name in points: wkt = "POINT(%0.1f %0.1f)" % (x, y) feature = mapnik.Feature(context, next_id) feature[name] = rule_name feature.add_geometries_from_wkt(wkt) memory_ds.add_feature(feature) next_id += 1 lyr.datasource = memory_ds log.debug('finish making memory datasource ' + str(datetime.datetime.now())) lyr.styles.append('Points legend') m.layers.append(lyr) if presentationlayer_id in [62007, 62008]: m = mapnik.Map(width, height) spherical_mercator = mapnik.Projection( '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 ' '+y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') m.srs = spherical_mercator m.background = mapnik.Color('transparent') sl = mapnik.Style() rule_l = mapnik.Rule() rule_stk = mapnik.Stroke() rule_stk.color = mapnik.Color(3, 158, 137) rule_stk.line_cap = mapnik.line_cap.ROUND_CAP rule_stk.width = 2.0 rule_l.symbols.append(mapnik.LineSymbolizer(rule_stk)) sl.rules.append(rule_l) m.append_style('Line Style2', sl) scenario = pl.scenario_set.get() layers = PresentationLayer.objects.filter(presentationtype=spt, scenario=scenario) log.debug('supportive layer found for this presentationlayer') rds = mapnik.Projection( "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 " "+k=0.999908 +x_0=155000 +y_0=463000 +ellps=bessel " "+towgs84=565.237,50.0087,465.658,-0.406857,0.350733," "-1.87035,4.0812 +units=m +no_defs") lyrl = mapnik.Layer('lines', rds) # Commented out, variable 'presentation_dir' doesn't exist so this # cannot work. -- RG20120621 # lyrl.datasource = mapnik.Shapefile( # file=external_file_location( # str(presentation_dir + '\\' + # pl.presentationshape.geo_source.file_location))) lyrl.styles.append('Line Style2') m.layers.append(lyrl) ##################### render map ############################# m.zoom_to_box(mapnik.Envelope(*bbox)) # lyrl.envelope log.debug('start render map ' + str(datetime.datetime.now())) img = mapnik.Image(width, height) mapnik.render(m, img) log.debug('ready render map ' + str(datetime.datetime.now())) log.debug('start PIL ' + str(datetime.datetime.now())) # you can use this if you want te modify image with PIL imgPIL = Image.fromstring('RGBA', (width, height), img.tostring()) #imgPIL = imgPIL.convert('RGB') buffer = StringIO.StringIO() imgPIL.save(buffer, 'png') # ,transparency = 10 buffer.seek(0) response = HttpResponse(buffer.read()) log.debug('end PIL ' + str(datetime.datetime.now())) response['Content-type'] = 'image/png' log.debug('ready sending map ' + str(datetime.datetime.now())) return response
def calc_damage(scenario_id): log.debug('read django and lizard settings') source_dir = Setting.objects.get(key='source_dir').value result_dir = Setting.objects.get(key='destination_dir').value presentation_dir = Setting.objects.get(key='presentation_dir').value log.debug('read scenario and file settings') dest_shape = os.path.join('flooding', 'scenario', str(scenario_id), 'embank_damage.shp') scenario = Scenario.objects.get(pk=scenario_id) pt = PresentationType.objects.get(code='damage_embankments') try: #damage settings extwater_models = SobekModel.objects.filter( scenariobreach__scenario=scenario, embankment_damage_shape__isnull=False) if extwater_models.count() == 0: log.info( 'no externalwatermodel or embankment_damage_shape for this scenario. Scenario skipped' ) return True, 'skipped 1 ' else: model = extwater_models[0] result_waterlevel = Result.objects.filter(scenario=scenario, resulttype=13) if result_waterlevel.count() > 0: his_zip_name = os.path.join(result_dir, result_waterlevel[0].resultloc) else: log.warning( 'Not enough results for calculation of embankment damage. Scenario skipped' ) return True, 'skipped 2 ' except Result.DoesNotExist: log.warning( 'Not enough results for calculation of embankment damage. Scenario skipped' ) return True, 'skipped 3 ' shapefile_name = os.path.join(source_dir, model.embankment_damage_shape) log.debug('read damage settings') damage = { 'onvhard': 500, # for each m 'verhard': 1000, # for each m 'huizen': 3000, # for each m2 'woonboot': 50000 } # each start_damage = { 'zonder verdediging': 0.5, 'met beschoeiing': 0.5, 'stalen damwand': 0.5, 'stortsteen': 0.5, 'woonboot': 1.0, 'kleidijk': 3.0, 'veendijk': 2.5, 'zanddijk': 2.0, 'Klei': 3.0, 'Veen': 2.5, 'Zand': 2.0, None: 0.5 } full_damage = { 'zonder verdediging': 1.0, 'met beschoeiing': 1.0, 'stalen damwand': 1.0, 'stortsteen': 2.0, 'woonboot': 1.0, 'kleidijk': 3.0, 'veendijk': 2.5, 'zanddijk': 2.0, 'Klei': 3.0, 'Veen': 2.5, 'Zand': 2.0, None: 1.0 } log.debug('open source shapefile') drv = ogr.GetDriverByName('ESRI Shapefile') source = drv.Open(str(shapefile_name)) source_layer = source.GetLayer() log.debug('create dest shapefile') output_dir = os.path.dirname(os.path.join(presentation_dir, dest_shape)) if not os.path.isdir(output_dir): log.debug('create output dir' + output_dir) os.makedirs(output_dir) dest_shape_full = os.path.join(presentation_dir, dest_shape) if os.path.isfile(dest_shape_full): log.debug('shapefile already exist, delete shapefile.') os.remove(dest_shape_full) os.remove(dest_shape_full.replace('.shp', '.shx')) os.remove(dest_shape_full.replace('.shp', '.dbf')) os.remove(dest_shape_full.replace('.shp', '.prj')) dest = drv.CreateDataSource(str(dest_shape_full)) dest_srs = osr.SpatialReference() dest_srs.ImportFromProj4( '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over' ) dest_layer = dest.CreateLayer(dest.GetName(), geom_type=ogr.wkbLineString, srs=dest_srs) log.debug('create geo-transformation') #source_srs = source_layer.GetSpatialRef() #transformation = ogr.osr.CoordinateTransformation(source_srs,dest_srs) log.debug('get field ids') if (source_layer.GetFeatureCount() > 0): feature = source_layer.next() id_index = feature.GetFieldIndex('ID_1') length_index = feature.GetFieldIndex('LENGTE') onverhard_index = feature.GetFieldIndex('ONVERH') verhard_index = feature.GetFieldIndex('VERHARD') huizen_index = feature.GetFieldIndex('HUIZEN') woonboot_index = feature.GetFieldIndex('WOONBOOT') kade_type_index = feature.GetFieldIndex('BSOORT') source_layer.ResetReading() log.debug("create 'source' fields in new shapefile") dest_layer.CreateField(ogr.FieldDefn('id', ogr.OFTString)) dest_layer.CreateField(ogr.FieldDefn('length', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('onverh', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('verhard', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('huizen', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('woonboot', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('kade_type', ogr.OFTString)) log.debug("create 'result' fields in new shapefile") dest_layer.CreateField(ogr.FieldDefn('daling', ogr.OFTReal)) dest_layer.CreateField(ogr.FieldDefn('perc_bezw', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_kade', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_bebouw', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_woonboot', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_totaal', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_max', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_perc', ogr.OFTInteger)) dest_layer.CreateField(ogr.FieldDefn('sch_per_m', ogr.OFTReal)) #openen hisfile log.debug('read hisfile') input_file = ZipFile(his_zip_name, "r") for filename in input_file.namelist(): if filename.lower() == 'calcpnt.his': his = HISFile(StringIO.StringIO(input_file.read(filename))) break input_file.close() def get_perc_bezw(daling, start_damage, full_damage): if daling <= start_damage: return 0.0 elif daling >= full_damage: return 100.0 else: return 100 * (daling - start_damage) / (full_damage - start_damage) sum_damage = 0 for layer in source_layer: #opzetten feature feat = ogr.Feature(feature_def=dest_layer.GetLayerDefn()) feat.SetFID(layer.GetFID()) #omzetten geometry geom = layer.geometry() #geom.TransformTo(dest_srs) feat.SetGeometry(geom) #omzetten basisgegevens feat.SetField('id', layer.GetField(id_index)) feat.SetField('length', layer.GetField(length_index)) feat.SetField('onverh', layer.GetField(onverhard_index)) feat.SetField('verhard', layer.GetField(verhard_index)) feat.SetField('huizen', layer.GetField(huizen_index)) feat.SetField('woonboot', layer.GetField(woonboot_index)) feat.SetField('kade_type', layer.GetField(kade_type_index)) #uitlezen peildaling uit hisfile try: daling = his.get_timeseries( 'c_' + layer.GetField(id_index), 'Waterlevel', None, None, list)[0][1] - min([ value for date, value in his.get_timeseries( 'c_' + layer.GetField(id_index), 'Waterlevel', None, None, list) ]) except KeyError: log.debug('no value for %s' % layer.GetField(id_index)) daling = 0. feat.SetField('daling', float(daling)) #uitvoeren berekeningen type_besch = layer.GetField(kade_type_index) perc_bezw = get_perc_bezw(daling, start_damage[type_besch], full_damage[type_besch]) feat.SetField('perc_bezw', int(perc_bezw)) max_sch_kade = layer.GetField(onverhard_index) * damage[ 'onvhard'] + layer.GetField(verhard_index) * damage['verhard'] feat.SetField('sch_kade', int(max_sch_kade * perc_bezw / 100)) max_sch_bebouwing = layer.GetField(huizen_index) * damage['huizen'] feat.SetField('sch_bebouw', int(max_sch_bebouwing * perc_bezw / 100)) max_sch_woonboot = layer.GetField(woonboot_index) * damage['woonboot'] sch_woonboot = max_sch_woonboot * get_perc_bezw( daling, start_damage['woonboot'], full_damage['woonboot']) / 100 feat.SetField('sch_woonboot', int(sch_woonboot)) sch_totaal = sch_woonboot + (max_sch_bebouwing + max_sch_kade) * perc_bezw / 100 feat.SetField('sch_totaal', int(sch_totaal)) sum_damage += sch_totaal max_schade = max_sch_woonboot + max_sch_bebouwing + max_sch_kade feat.SetField('sch_max', int(max_schade)) if max_schade > 0: sch_perc = 100 * sch_totaal / max_schade else: sch_perc = -1 feat.SetField('sch_perc', int(sch_perc)) length = layer.GetField(length_index) if length > 0: sch_per_m = sch_totaal / length else: sch_per_m = -1 feat.SetField('sch_per_m', int(sch_per_m)) #wegschrijven dest_layer.CreateFeature(feat) dest_layer.SyncToDisk() # Clean up dest.Destroy() source.Destroy() pl, pl_new = PresentationLayer.objects.get_or_create( presentationtype=pt, scenario_presentationlayer__scenario=scenario, defaults={"value": sum_damage}) if pl_new: Scenario_PresentationLayer.objects.create(scenario=scenario, presentationlayer=pl) else: pl.value = sum_damage pl.save() link_type, _ = SourceLinkType.objects.get_or_create( name='flooding_scenario_embankment_damage') source_link, new = SourceLink.objects.get_or_create( link_id=str(scenario.id), sourcelinktype=link_type, type='fl_emb_dam_' + str(scenario.id)) source, new = source_link.presentationsource.get_or_create( type=PresentationSource.SOURCE_TYPE_SHAPEFILE) source.file_location = dest_shape source.t_original = datetime.datetime.fromtimestamp( os.stat(str(his_zip_name))[8]) source.t_source = datetime.datetime.now() source.save() ps, ps_new = PresentationShape.objects.get_or_create(presentationlayer=pl) ps.geo_source = source ps.save() log.debug("Finish task.") log.debug("close db connection to avoid an idle process.") db.close_connection() return True, "ok "