def _get_multipolygon_geometry_from_row(self, row):
if row.geom_type.django == 'PolygonField':
geom = OGRGeometry(OGRGeomType('MultiPolygon'))
geom.add(row.geom)
geom.coord_dim = 2
return geom
elif row.geom_type.django == 'MultiPolygonField':
return geom
def process(key, metadata):
defaults = {}
if 'geometry' in metadata:
geom = OGRGeometry(json.dumps(metadata['geometry']))
geom.coord_dim = 2
defaults['geom'] = geom.geos
# del metadata['geometry']
if 'acquired' in metadata:
defaults['acquired'] = parse(metadata['acquired'])
if 'archived' in metadata:
defaults['published'] = parse(metadata['archived'])
if 'published' in metadata:
defaults['published'] = parse(metadata['published'])
if 'processed' in metadata:
defaults['processed'] = datetime.utcfromtimestamp(metadata['processed'])
if 'file_sizes' in metadata:
defaults['bytes'] = functools.reduce(operator.add, metadata['file_sizes'])
if 'raster_size' in metadata:
defaults['pixels'] = functools.reduce(operator.mul, metadata['raster_size'])
if 'cloud_fraction' in metadata:
defaults['cloud_fraction'] = float(metadata['cloud_fraction'])
if 'fill_fraction' in metadata:
defaults['fill_fraction'] = float(metadata['fill_fraction'])
defaults['data'] = metadata
product_id = metadata.get('product','N/A')
product, created = Product.objects.get_or_create(name=product_id, defaults={'groups':['public']})
defaults['groups'] = product.groups
instance, created = Metadata.objects.update_or_create(key=key, product=product, defaults=defaults)
return instance, created
def convert_shapefile(shapefilename, srid=4674):
"""
shapefilename: considera nomenclatura de shapefile do IBGE para determinar se é UF
ou Municípios.
ex. 55UF2500GC_SIR.shp para UF e 55MU2500GC_SIR.shp para Municípios
srid: 4674 (Projeção SIRGAS 2000)
"""
# /home/nando/Desktop/IBGE/2010/55MU2500GC_SIR.shp
ds = DataSource(shapefilename)
is_uf = shapefilename.upper().find('UF') != -1
transform_coord = None
if srid != SRID:
transform_coord = CoordTransform(SpatialReference(srid), SpatialReference(SRID))
if is_uf:
model = UF
else:
model = Municipio
ct = 0
for f in ds[0]:
# 3D para 2D se necessário
if f.geom.coord_dim != 2:
f.geom.coord_dim = 2
# converte para MultiPolygon se necessário
if isinstance(f.geom, Polygon):
g = OGRGeometry(OGRGeomType('MultiPolygon'))
g.add(f.geom)
else:
g = f.geom
# transforma coordenadas se necessário
if transform_coord:
g.transform(transform_coord)
# força 2D
g.coord_dim = 2
kwargs = {}
if is_uf:
kwargs['nome'] = capitalize_name(unicode(f.get(CAMPO_NOME_UF),'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_UF)
kwargs['regiao'] = capitalize_name(unicode(f.get(CAMPO_REGIAO_UF), 'latin1'))
kwargs['uf'] = UF_SIGLAS_DICT.get(kwargs['id_ibge'])
else:
kwargs['nome'] = capitalize_name(unicode(f.get(CAMPO_NOME_MU),'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_MU)
kwargs['uf'] = UF.objects.get(pk=f.get(CAMPO_GEOCODIGO_MU)[:2])
kwargs['uf_sigla'] = kwargs['uf'].uf
kwargs['nome_abreviado'] = slugify(kwargs['nome'])
# tenta corrigir nomes duplicados, são em torno de 242 nomes repetidos
# adicionando a sigla do estado no final
if Municipio.objects.filter(nome_abreviado=kwargs['nome_abreviado']).count() > 0:
kwargs['nome_abreviado'] = u'%s-%s' % (kwargs['nome_abreviado'], kwargs['uf_sigla'].lower())
instance = model(**kwargs)
instance.save()
ct += 1
print(ct, (is_uf and "Unidades Federativas criadas" or "Municipios criados"))
def clean_geom(self):
""" Tricks to add default Z dimension into geometry """
geom = self.cleaned_data["geom"]
ogr = OGRGeometry(geom.wkt)
ogr.coord_dim = 3
return ogr.geos
print(f'\nReading input data from the following file:\n {geojson_path}\n')
data_source = DataSource(geojson_path)
layer = data_source[0]
feature_count = len(layer)
i = 1
for feature in layer:
try:
feature_data = {
key: feature.get(value) for (key, value) in model_layer_mapping.items()
}
# Use OGRGeometry to strip z-values from polyline vertices
geometry = OGRGeometry(feature.geom.ewkt)
geometry.coord_dim = 2
feature_data['the_geom'] = geometry.ewkt
feature_data['diff_id'] = diff_id
if feature_data['type'] == 'way':
feature_data['type'] = OsmDiff.WAY
if feature_data['type'] == 'node':
feature_data['type'] = OsmDiff.NODE
if feature_data['type'] == 'relation':
feature_data['type'] = OsmDiff.RELATION
if isinstance(feature_data['tags'], str):
feature_data['tags'] = json.loads(feature_data['tags'])
if isinstance(feature_data['meta'], str):
feature_data['meta'] = json.loads(feature_data['meta'])
def load_osm_diff(diff_id, logger=None):
start_time = datetime.datetime.now()
if not logger:
logger = initialize_logger()
logger.info(f'Executing script: {os.path.abspath(__file__)}')
logger.info(f'Execution start at: {start_time}')
model_layer_mapping = {
'osm_id': 'id',
'tags': 'tags',
'meta': 'meta',
'tainted': 'tainted',
'type': 'type',
}
geojson_path = r'D:\OpenStreetMap\diff_data\{diff_id}\{diff_id}.geojson'.format(
diff_id=diff_id)
logger.debug(f'Input geoJSON file: {geojson_path}')
try:
data_source = DataSource(geojson_path)
layer = data_source[0]
feature_count = len(layer)
i = 1
for feature in layer:
try:
feature_data = {
key: feature.get(value)
for (key, value) in model_layer_mapping.items()
}
# Use OGRGeometry to strip z-values from polyline vertices
geometry = OGRGeometry(feature.geom.ewkt)
geometry.coord_dim = 2
feature_data['the_geom'] = geometry.ewkt
feature_data['diff_id'] = diff_id
if feature_data['type'] == 'way':
feature_data['type'] = OsmDiff.WAY
if feature_data['type'] == 'node':
feature_data['type'] = OsmDiff.NODE
if feature_data['type'] == 'relation':
feature_data['type'] = OsmDiff.RELATION
if isinstance(feature_data['tags'], str):
feature_data['tags'] = json.loads(feature_data['tags'])
if isinstance(feature_data['meta'], str):
feature_data['meta'] = json.loads(feature_data['meta'])
route = OsmDiff(**feature_data)
route.save()
if i % 10 == 0:
logger.debug(f'Processed {i} of {feature_count} features')
if i == feature_count:
logger.debug(f'Processed {i} of {feature_count} features')
i += 1
except Exception as exc:
logger.error(f'\n\nFailed on feautre {i} of {feature_count}')
logger.error(feature_data)
logger.exception(exc)
i += 1
logger.debug(
'\nCreating buffer polygons of the newly generated ways and saving to database'
)
ways = OsmDiff.objects.filter(type=OsmDiff.WAY).filter(diff_id=diff_id)
polys = [LineString(way.the_geom).buffer(0.01) for way in ways]
multi_polys = cascaded_union(polys)
logger.debug('Saving polygons to database')
feature_count = len(multi_polys)
i = 0
for poly in multi_polys:
osm_diff_buffer = OsmDiffBuffer(
diff_id=diff_id,
created_date=datetime.datetime.today(),
the_geom=poly.wkt)
osm_diff_buffer.save()
if i % 10 == 0:
logger.debug(f'Processed {i} of {feature_count} features')
if i == feature_count:
logger.debug(f'Processed {i} of {feature_count} features')
i += 1
end_time = datetime.datetime.now()
logger.info(f'Execution complete at: {end_time}')
logger.info(f'Execution run time: {end_time-start_time}')
return True
except Exception as exc:
logger.error(f'An error occurred: load_osm_diff: {diff_id}')
logger.exception(exc)
return False
