def locate(self, tw, conf=0.95):
poly = tw.geom.buffer(5000) # FIXME: change to meters?
multiregion = geos.MultiPolygon((poly), srid=poly.srid)
return self.LE(point=tw.geom, cregion=multiregion, confidence=conf)
def import_events(data_from_feed, tolerance=0.0001):
"""
For every shapefile in _merged folder, creates or updates event in Django DB and Geoserver DB.
The geometry inserted in Geoserver DB is the union of all features found in layer.
"""
db = DbUtils()
conn = db.get_db_conn()
curs = conn.cursor()
allowed_extensions = [".shp"]
try:
for root, dirs, files in os.walk(SHAPEFILES_BASE_DIR):
print('shapefiles base dir: {}'.format(SHAPEFILES_BASE_DIR))
for d in dirs:
if d.endswith('_merged'):
print('found merged folder')
for f in os.listdir(os.path.join(SHAPEFILES_BASE_DIR, d)):
if f.endswith(tuple(allowed_extensions)):
print('found shapefiles: {}'.format(f))
'''try:
filepath = os.path.join(SHAPEFILES_BASE_DIR, d, f)
ds = DataSource(filepath)
except GDALException, e:
traceback.print_exc()
break
for layer in ds:
count = 0
polygon_union = None
for feat in layer:
print('processing feature {}'.format(count))
# Simplify the Geometry
geom = geos.fromstr(feat.geom.wkt, srid=4326)
if tolerance > 0:
geom = geom.simplify(tolerance, preserve_topology=True)
# Generalize to 'Multiploygon'
if isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
if count == 0:
polygon_union = geom.buffer(0)
else:
polygon_union = polygon_union.union(geom.buffer(0))
count += 1
try:
print('re-extracting geom from union polygon')
polygon_union = geos.fromstr(polygon_union.wkt, srid=4326)
except:
break'''
ems = d.split('_')[0]
select_template = "SELECT ST_AsText(ST_Union(ARRAY(SELECT ST_Buffer(the_geom, 1e-5) FROM {})))".format(
ems)
row = None
try:
curs.execute(select_template)
row = curs.fetchone()
except:
pass
#set default multipolygon
ext_coords = ((0, 0), (0, 1), (1, 1), (1, 0), (0,
0))
int_coords = ((0.4, 0.4), (0.4, 0.6), (0.6, 0.6),
(0.6, 0.4), (0.4, 0.4))
polygon_union = geos.MultiPolygon(geos.Polygon(
ext_coords, int_coords),
srid=4326)
if row:
polygon_union = geos.fromstr(row[0], srid=4326)
# Update event in Django
event_from_feed = get_event_from_feed(
data_from_feed, ems)
new_event = generate_event_from_feed(
event_from_feed, polygon_union)
if new_event:
params = {
'geom': polygon_union,
'event_id': new_event.event_id,
'begin_date': new_event.begin_date,
'end_date': new_event.end_date
}
update_template = """INSERT INTO events (the_geom, event_id, begin_date, end_date)
SELECT
'{geom}', '{event_id}', '{begin_date}', '{end_date}'
ON CONFLICT (event_id)
DO UPDATE SET the_geom = excluded.the_geom;"""
print update_template.format(**params)
curs.execute(update_template.format(**params))
archive_path = os.path.join(SHAPEFILES_BASE_DIR, d,
'archive')
if not os.path.exists(archive_path):
os.makedirs(archive_path)
for f2 in os.listdir(os.path.join(SHAPEFILES_BASE_DIR, d)):
filepath = os.path.join(SHAPEFILES_BASE_DIR, d, f2)
if os.path.isfile(filepath):
print('moving file {} to {}'.format(
f2, archive_path))
os.rename(filepath, os.path.join(archive_path, f2))
conn.commit()
except Exception:
try:
conn.rollback()
except:
pass
traceback.print_exc()
finally:
conn.close()
def clean_poly(self, poly):
if isinstance(poly, geos.Polygon):
poly = geos.MultiPolygon(poly, srid=self.get_srid('districts'))
return poly
return poly
def handle(self, **options):
adm_level = options.get('adm_level')
region = options.get('region')
region_level = options.get('region_level')
shape_file = options.get('shape_file')
tolerance = options.get('tolerance')
if adm_level is None:
raise CommandError("Input Administrative Unit Level '--adm-level' \
is mandatory")
if region is None:
raise CommandError("Input Destination Region '--region' \
is mandatory")
if region_level is None:
raise CommandError("Input Region Level '--region-level' \
is mandatory")
if not shape_file or len(shape_file) == 0:
raise CommandError("Input Administrative Unit Shapefile \
'--shape-file' is mandatory")
ds = DataSource(shape_file)
print('Opening Data Source "%s"' % ds.name)
#print('rebuilding tree')
#AdministrativeDivision.objects.rebuild()
#print('rebuilding complete!')
for layer in ds:
print('Layer "%s": %i %ss' %
(layer.name, len(layer), layer.geom_type.name))
(region_obj, is_new_region) = Region.objects.get_or_create(
name=region,
defaults=dict(
level=region_level
)
)
adm_rows = []
for feat in layer:
# Simplify the Geometry
geom = geos.fromstr(feat.geom.wkt, srid=4326)
if tolerance > 0:
geom = geom.simplify(tolerance, preserve_topology=True)
# Generalize to 'Multiploygon'
if isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
if adm_level == 0:
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
code=feat.get('HRPcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.save()
else:
#region_obj.administrative_divisions.add(adm_division)
RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division)
if adm_level == 1:
print 'parent {}'.format(feat.get('HRparent')[:2])
adm_division_0 = \
AdministrativeDivision.objects.get(
code=feat.get('HRparent')[:2])
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
code=feat.get('HRpcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt,
parent=adm_division_0
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.parent = adm_division_0
adm_division.save()
else:
#region_obj.administrative_divisions.add(adm_division)
RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division)
if adm_level == 2:
print('region = {}'.format(feat.get('HRpcode')))
adm_division_1 = AdministrativeDivision.objects.get(code=feat.get('HRparent'))
(adm_division, is_new_amdiv) = AdministrativeDivision.objects.get_or_create(
code=feat.get('HRpcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt,
parent=adm_division_1
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.parent = adm_division_1
adm_division.save()
else:
#region_obj.administrative_divisions.add(adm_division)
RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division)
if adm_level == 3:
print('adm = {}'.format(feat.get('HRpcode')))
print('parent = {}'.format(feat.get('HRparent')))
adm_division_2 = AdministrativeDivision.objects.get(code=feat.get('HRparent'))
'''(adm_division, is_new_amdiv) = AdministrativeDivision.objects.get_or_create(
code=feat.get('HRpcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt,
region=region_obj,
parent=adm_division_2
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.region = region_obj
adm_division.parent = adm_division_2
adm_division.save()
else:
region_obj.administrative_divisions.add(adm_division)'''
lookup_obj = AdministrativeDivision.objects.filter(code=feat.get('HRpcode'))
if lookup_obj.exists():
adm_division = AdministrativeDivision.objects.get(code=feat.get('HRpcode'))
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.parent = adm_division_2
adm_division.save()
association, created = RegionAdministrativeDivisionAssociation.objects.get_or_create(region=region_obj, administrativedivision=adm_division)
print('updated object {}'.format(feat.get('HRpcode')))
else:
adm_division = AdministrativeDivision(
code=feat.get('HRpcode'),
name=feat.get('HRname'),
geom=geom.wkt,
level=adm_level,
parent=adm_division_2,
lft=1,
rght=1,
tree_id=1,
)
adm_rows.append(adm_division)
if(len(adm_rows) > 9999):
print('bulk insert starting')
AdministrativeDivision.objects.bulk_create(adm_rows)
for adm_code in adm_rows:
adm_div = AdministrativeDivision.objects.get(code=adm_code)
RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_div)
print('bulk insert complete')
adm_rows[:] = []
#print(adm_rows.count())
if(len(adm_rows) > 0):
print('bulk insert starting')
AdministrativeDivision.objects.bulk_create(adm_rows)
for adm_code in adm_rows:
adm_div = AdministrativeDivision.objects.get(code=adm_code)
RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_div)
print('bulk insert complete')
def handle(self, **options):
adm_level = options.get('adm_level')
shape_file = options.get('shape_file')
tolerance = options.get('tolerance')
if adm_level is None:
raise CommandError("Input Administrative Unit Level '--adm-level' \
is mandatory")
if not shape_file or len(shape_file) == 0:
raise CommandError("Input Administrative Unit Shapefile \
'--shape-file' is mandatory")
ds = DataSource(shape_file)
print ('Opening Data Source "%s"' % ds.name)
for layer in ds:
print('Layer "%s": %i %ss' %
(layer.name, len(layer), layer.geom_type.name))
counter = 1
for feat in layer:
# Simplify the Geometry
geom = geos.fromstr(feat.geom.wkt, srid=4326)
if tolerance > 0:
geom = geom.simplify(tolerance, preserve_topology=True)
if geom:
# Generalize to 'Multiploygon'
if isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
region_name = feat.get("NAME_{adm_level}".format(adm_level=adm_level - 1)) if adm_level > 0 else \
(feat.get("UNREGION2") if feat.get("UNREGION2") else feat.get("UNREGION1"))
(region_obj, is_new_region) = Region.objects.get_or_create(
name=region_name,
defaults=dict(
level=(adm_level - 1 if adm_level > 0 else adm_level)
)
)
level = adm_level
objectid = feat.get("OBJECTID")
iso_id = feat.get("ID_{adm_level}".format(adm_level=adm_level))
name_iso = feat.get("NAME_ISO") if adm_level == 0 else feat.get("NAME_{adm_level}".format(adm_level=adm_level))
iso_parent = feat.get("ISO")
iso_parent_id = iso_id if adm_level == 0 else feat.get("ID_{adm_level}".format(adm_level=adm_level - 1))
shape_leng = Decimal(feat.get("Shape_Leng"))
shape_area = Decimal(feat.get("Shape_Area"))
parent = None
if name_iso:
if level > 0:
type = feat.get("TYPE_{adm_level}".format(adm_level=adm_level))
engtype = feat.get("ENGTYPE_{adm_level}".format(adm_level=adm_level))
try:
parent = AdministrativeDivision.objects.get(Q(iso=iso_parent) &
Q(iso_id=iso_parent_id) &
Q(level=(adm_level - 1)))
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
name=name_iso,
defaults=dict(
objectid=objectid,
level=level,
iso=iso_parent,
iso_id=iso_id,
type=type,
engtype=engtype,
shape_leng=shape_leng,
shape_area=shape_area,
geom=geom,
region=region_obj,
parent=parent
)
)
if adm_division and is_new_amdiv:
region_obj.administrative_divisions.add(adm_division)
except:
pass
else:
parent = region_obj
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
name=name_iso,
defaults=dict(
objectid=objectid,
level=level,
iso=iso_parent,
iso_id=iso_id,
name_eng=feat.get("NAME_ENGLI"),
name_fao=feat.get("NAME_FAO"),
name_local=feat.get("NAME_LOCAL"),
contains=feat.get("CONTAINS"),
sovereign=feat.get("SOVEREIGN"),
fips=feat.get("FIPS"),
unregion=feat.get("UNREGION1"),
ison=feat.get("ISON"),
valid_from=feat.get("VALIDFR"),
valid_to=feat.get("VALIDTO"),
population=Decimal(feat.get("POP2000")),
sqkm=Decimal(feat.get("SQKM")),
pop_sqkm=Decimal(feat.get("POPSQKM")),
shape_leng=shape_leng,
shape_area=shape_area,
geom=geom,
region=region_obj
)
)
if adm_division and is_new_amdiv:
region_obj.administrative_divisions.add(adm_division)
if parent:
print(" %s ************* Region : %s - Parent: %s(%s) - Adm Unit : %s(%s) " % (counter,
region_name,
parent,
iso_parent,
name_iso,
iso_id))
counter = counter + 1
def handle(self, **options):
adm_level = options.get('adm_level')
region = options.get('region')
region_level = options.get('region_level')
shape_file = options.get('shape_file')
tolerance = options.get('tolerance')
if adm_level is None:
raise CommandError("Input Administrative Unit Level '--adm-level' \
is mandatory")
if region is None:
raise CommandError("Input Destination Region '--region' \
is mandatory")
if region_level is None:
raise CommandError("Input Region Level '--region-level' \
is mandatory")
if not shape_file or len(shape_file) == 0:
raise CommandError("Input Administrative Unit Shapefile \
'--shape-file' is mandatory")
ds = DataSource(shape_file)
print('Opening Data Source "%s"' % ds.name)
for layer in ds:
print('Layer "%s": %i %ss' %
(layer.name, len(layer), layer.geom_type.name))
(region_obj, is_new_region) = Region.objects.get_or_create(
name=region, defaults=dict(level=region_level))
for feat in layer:
# Simplify the Geometry
geom = geos.fromstr(feat.geom.wkt, srid=4326)
if tolerance > 0:
geom = geom.simplify(tolerance, preserve_topology=True)
# Generalize to 'Multiploygon'
geom = geos.MultiPolygon(geom)
if adm_level == 0:
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
code=feat.get('HRPcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt,
region=region_obj
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.region = region_obj
adm_division.save()
else:
region_obj.administrative_divisions.add(adm_division)
if adm_level == 1:
adm_division_0 = \
AdministrativeDivision.objects.get(
code=feat.get('HRparent')[:-2])
(adm_division, is_new_amdiv) = \
AdministrativeDivision.objects.get_or_create(
code=feat.get('HRpcode'),
defaults=dict(
name=feat.get('HRname'),
geom=geom.wkt,
region=region_obj,
parent=adm_division_0
)
)
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.region = region_obj
adm_division.parent = adm_division_0
adm_division.save()
else:
region_obj.administrative_divisions.add(adm_division)
if adm_level == 2:
adm_division_1 = AdministrativeDivision.objects.get(
code=feat.get('HRparent'))
(adm_division, is_new_amdiv
) = AdministrativeDivision.objects.get_or_create(
code=feat.get('HRpcode'),
defaults=dict(name=feat.get('HRname'),
geom=geom.wkt,
region=region_obj,
parent=adm_division_1))
if not is_new_amdiv:
adm_division.name = feat.get('HRname')
adm_division.geom = geom.wkt
adm_division.region = region_obj
adm_division.parent = adm_division_1
adm_division.save()
else:
region_obj.administrative_divisions.add(adm_division)
def execute(self): # NOQA C901
func_start = perf_counter()
if not self.area_types:
self.area_types = AREA_IMPORT_TYPES.keys()
errors = []
for area_import_type in self.area_types:
type_start = perf_counter()
self.stdout.write(
'Starting to import the area type "{}"...\n'.format(
area_import_type, ))
area_import = AREA_IMPORT_TYPES[area_import_type]
try:
conn = psycopg2.connect(
getattr(settings, area_import["source_dsn_setting_name"]),
cursor_factory=psycopg2.extras.NamedTupleCursor,
)
except (psycopg2.ProgrammingError, psycopg2.OperationalError) as e:
self.stderr.write(str(e))
self.stderr.write(
'Could not connect to the database when importing area type "{}". DSN setting name "{}"'
.format(area_import_type,
area_import["source_dsn_setting_name"]))
continue
cursor = conn.cursor()
self.stdout.write(area_import["source_name"])
(source, source_created) = AreaSource.objects.get_or_create(
identifier=area_import["source_identifier"],
defaults={"name": area_import["source_name"]},
)
cursor.execute(area_import["query"])
imported_identifiers = []
count = 0
sum_row_time, avg_row_time, min_row_time, max_row_time = (0, ) * 4
self.stdout.write("Starting to update areas...\n")
for row in cursor:
row_start = perf_counter()
try:
metadata = {
METADATA_COLUMN_NAME_MAP[column_name]:
getattr(row, column_name)
for column_name in area_import["metadata_columns"]
}
except AttributeError as e:
errors.append(
"id #{}, metadata field missing. Error: {}\n".format(
row.id, str(e)))
count += 1
self.stdout.write("E", ending="")
if count % 1000 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
continue
match_data = {
"type":
area_import["area_type"],
"external_id":
row.id,
"identifier":
getattr(row, area_import["identifier_field_name"]),
"source":
source,
}
try:
geom = geos.GEOSGeometry(row.geom_text)
except geos.error.GEOSException as e:
errors.append("id #{} error: {}\n".format(row.id, str(e)))
count += 1
self.stdout.write("E", ending="")
if count % 1000 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
continue
if geom and isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
if geom and not isinstance(geom, geos.MultiPolygon):
errors.append(
'id #{} Error! Geometry is not a Multipolygon but "{}"\n'
.format(row.id, geom))
count += 1
self.stdout.write("E", ending="")
if count % 1000 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
continue
other_data = {
"geometry": geom,
"metadata": metadata,
}
try:
Area.objects.update_or_create(defaults=other_data,
**match_data)
except MultipleObjectsReturned: # There should only be one object per identifier...
ext_id = other_data.pop("external_id")
# ...so we delete them all but spare the one with the correct external_id (if it happens to exist)
Area.objects.filter(**match_data).exclude(
external_id=ext_id).delete()
match_data["external_id"] = ext_id
Area.objects.update_or_create(defaults=other_data,
**match_data)
imported_identifiers.append(match_data["identifier"])
count += 1
if count % 100 == 0:
self.stdout.write(".", ending="")
if count % 1000 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
row_end = perf_counter()
row_time = row_end - row_start
sum_row_time += row_time
min_row_time = (row_time if min_row_time == 0
or row_time < min_row_time else min_row_time)
max_row_time = row_time if row_time > max_row_time else max_row_time
if count > 0:
avg_row_time = sum_row_time / count
self.stdout.write(
"Updated area count {}. Execution time: {:.2f}s "
"(Row time avg: {:.2f}s, min: {:.2f}s, max: {:.2f}s)\n".format(
count, sum_row_time, avg_row_time, min_row_time,
max_row_time))
self.stdout.write("Starting to remove stales...\n")
stale_time_start = perf_counter()
stale = Area.objects.filter(
type=area_import["area_type"],
source=source).exclude(identifier__in=imported_identifiers)
stale_count = stale.count()
stale.delete()
stale_time_end = perf_counter()
self.stdout.write(
"Removed stale count {}. Execution time: {:.2f}s\n".format(
stale_count, stale_time_end - stale_time_start))
if errors:
self.stdout.write(" {} errors:\n".format(len(errors)))
for error in errors:
self.stdout.write(error)
type_end = perf_counter()
self.stdout.write(
'The area import of type "{}" is completed. Execution time: {:.2f}s\n'
.format(area_import_type, (type_end - type_start)))
func_end = perf_counter()
self.stdout.write(
"The area import is completed. Execution time: {0:.2f}s\n".format(
func_end - func_start))
def import_shapes(shapefiles, source):
adm_scheme, _ = SkosConceptScheme.objects.get_or_create(
dc_title="administrative_unit")
temp_spatial_ids = []
for x in shapefiles:
df = gp.read_file(x).to_crs({
'proj': 'longlat',
'ellps': 'WGS84',
'datum': 'WGS84'
})
for i, row in df.iterrows():
add_data = {}
if row['geometry'].geom_type == 'MultiPolygon':
mp = row['geometry'].wkt
else:
mp = geos.MultiPolygon(fromstr(row['geometry'].wkt))
spat, _ = TempSpatial.objects.get_or_create(
start_date=row['start_date'],
end_date=row['end_date'],
date_accuracy=row['date_acc'],
geom=mp)
try:
try:
adm, _ = SkosConcept.objects.get_or_create(
pref_label=row['adm_type'])
except IntegrityError:
adm, _ = SkosConcept.objects.get_or_create(
pref_label="unknown adm")
except KeyError:
adm, _ = SkosConcept.objects.get_or_create(
pref_label="unknown adm")
if adm is not None:
adm.scheme.add(adm_scheme)
spat.administrative_unit = adm
try:
if row['name']:
spat.name = row['name']
except KeyError:
pass
try:
if row['name_alt']:
spat.alt_name = row['name_alt']
except KeyError:
pass
try:
if row['id']:
spat.orig_id = row['id']
except KeyError:
pass
try:
if row['wiki_id']:
spat.wikidata_id = row['wiki_id']
except KeyError:
pass
for x in list(df.keys()):
if x not in mandatory_keys:
add_data[x] = row[x]
spat.additional_data = json.dumps(add_data)
spat.source = source
try:
spat.save()
temp_spatial_ids.append(spat.id)
except Exception as e:
print(e)
return temp_spatial_ids
def execute(self): # NOQA C901
cursor = self.cursor
django_cursor = django_connection.cursor()
errors = []
area_import = AREA_IMPORT_TYPES["lease_area"]
self.stdout.write(area_import["source_name"])
(source, source_created) = AreaSource.objects.get_or_create(
identifier=area_import["source_identifier"],
defaults={"name": area_import["source_name"]},
)
cursor.execute(area_import["query"])
lease_area_rows = rows_to_dict_list(cursor)
count = 0
for row in lease_area_rows:
metadata = {
METADATA_COLUMN_NAME_MAP[column_name]:
row.get(column_name.upper())
for column_name in area_import["metadata_columns"]
}
match_data = {
"type": area_import["area_type"],
"identifier": row.get(area_import["identifier_field_name"]),
"external_id": row.get("ID"),
"source": source,
}
# There is no SRID info in the data. Assume it's 3879.
wkt_geom_with_srid = "SRID=3879;{}".format(
row.get("WKT_GEOM").read())
# Convert possible curves to lines in the geometry and transform the geometry to SRID 4326
django_cursor.execute(
"SELECT ST_AsText(ST_Transform(ST_CurveToLine(ST_GeomFromEWKT(%s)), 4326));",
[wkt_geom_with_srid],
)
geom_text = django_cursor.fetchone()[0]
try:
geom = geos.GEOSGeometry(geom_text)
except geos.error.GEOSException as e:
errors.append("id #{} error: " + str(e))
count += 1
self.stdout.write("E", ending="")
if count % 100 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
# self.stdout.write(str(e))
continue
if geom and isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
if geom and not isinstance(geom, geos.MultiPolygon):
errors.append(
"id #{} error: " +
' Error! Geometry is not a Multipolygon but "{}"\n'.format(
geom))
count += 1
self.stdout.write("E", ending="")
if count % 100 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
# self.stdout.write(' Error! Geometry is not a Multipolygon but "{}"\n'.format(geom))
continue
other_data = {"geometry": geom, "metadata": metadata}
Area.objects.update_or_create(defaults=other_data, **match_data)
count += 1
self.stdout.write(".", ending="")
if count % 100 == 0:
self.stdout.write(" {}".format(count))
self.stdout.flush()
self.stdout.write(" Count {}\n".format(count))
if errors:
self.stdout.write(" {} errors:\n".format(len(errors)))
for error in errors:
self.stdout.write(error)
def save(self, **kwargs):
from regionfestival.snippets import unique_slugify
unique_slugify(self, self.name)
if self.boundaries and isinstance(self.boundaries, geos.Polygon):
self.boundaries = geos.MultiPolygon(self.boundaries)
super(Region, self).save(**kwargs)
def execute(self): # NOQA C901
if not self.area_types:
self.area_types = AREA_IMPORT_TYPES.keys()
errors = []
for area_import_type in self.area_types:
area_import = AREA_IMPORT_TYPES[area_import_type]
try:
conn = psycopg2.connect(
getattr(settings, area_import['source_dsn_setting_name']),
cursor_factory=psycopg2.extras.NamedTupleCursor)
except (psycopg2.ProgrammingError, psycopg2.OperationalError) as e:
self.stderr.write(str(e))
self.stderr.write(
'Could not connect to the database when importing area type "{}". DSN setting name "{}"'
.format(area_import_type,
area_import['source_dsn_setting_name']))
continue
cursor = conn.cursor()
self.stdout.write(area_import['source_name'])
(source, source_created) = AreaSource.objects.get_or_create(
identifier=area_import['source_identifier'],
defaults={'name': area_import['source_name']})
cursor.execute(area_import['query'])
count = 0
for row in cursor:
metadata = {
METADATA_COLUMN_NAME_MAP[column_name]:
getattr(row, column_name)
for column_name in area_import['metadata_columns']
}
match_data = {
'type':
area_import['area_type'],
'identifier':
getattr(row, area_import['identifier_field_name']),
'external_id':
row.id,
'source':
source,
}
try:
geom = geos.GEOSGeometry(row.geom_text)
except geos.error.GEOSException as e:
errors.append('id #{} error: ' + str(e))
count += 1
self.stdout.write('E', ending='')
if count % 100 == 0:
self.stdout.write(' {}'.format(count))
self.stdout.flush()
# self.stdout.write(str(e))
continue
if geom and isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
if geom and not isinstance(geom, geos.MultiPolygon):
errors.append(
'id #{} error: ' +
' Error! Geometry is not a Multipolygon but "{}"\n'.
format(geom))
count += 1
self.stdout.write('E', ending='')
if count % 100 == 0:
self.stdout.write(' {}'.format(count))
self.stdout.flush()
# self.stdout.write(' Error! Geometry is not a Multipolygon but "{}"\n'.format(geom))
continue
other_data = {
'geometry': geom,
'metadata': metadata,
}
Area.objects.update_or_create(defaults=other_data,
**match_data)
count += 1
self.stdout.write('.', ending='')
if count % 100 == 0:
self.stdout.write(' {}'.format(count))
self.stdout.flush()
self.stdout.write(' Count {}\n'.format(count))
if errors:
self.stdout.write(' {} errors:\n'.format(len(errors)))
for error in errors:
self.stdout.write(error)
def handle(self, *args, **options):
feature_index = 'q'
features = []
locations = []
while feature_index.lower() == 'q':
original_url = 'https://s3.amazonaws.com/osm-polygons.mapzen.com/'
countries = requests.get(original_url).text
countries = xmltodict.parse(countries)
for i, c in enumerate(
countries['ListBucketResult']['Contents'][1:], start=1):
print("{}: {}".format(i, c['Key']))
print("Press <return> to exit\n")
country_index = input('Enter selected country: ')
if not country_index:
return
selected = countries['ListBucketResult']['Contents'][int(
country_index)]
response = requests.get(original_url + selected['Key'])
tar_file = tarfile.open(mode='r:gz',
fileobj=BytesIO(response.content))
tmpdir = tempfile.mkdtemp()
tar_file.extractall(tmpdir)
json_file_path = os.path.join(tmpdir, os.listdir(tmpdir)[0])
all_files = sorted(os.listdir(json_file_path))
for i, f in enumerate(all_files, start=1):
file_size = os.path.getsize(os.path.join(json_file_path, f))
print("{}: {} ({})".format(i, f, file_size))
print("Press Q to restart and <return> to exit\n")
level_index = input('Enter selected file: ')
if level_index.lower() == 'q':
continue
if not level_index:
return
selected_file = all_files[int(level_index) - 1]
with open(os.path.join(json_file_path, selected_file)) as f:
current_json = f.read()
content = json.loads(current_json)
features = content['features']
shutil.rmtree(tmpdir)
locations = sorted(
[a for a in features if 'name' in a['properties']],
key=lambda x: x['properties']['name'])
for i, l in enumerate(locations):
print("{}: {} ({})".format(
i + 1, l['properties']['name:en'] if 'name:en'
in l['properties'] else '', l['properties']['name']))
print("Press Q to restart and <return> to exit\n")
feature_index = input('Enter selected feature: ')
if not feature_index:
return
if not features or not locations:
return
if feature_index.lower() == 'all':
to_be_merged = [
geos.GEOSGeometry(json.dumps(a['geometry'])) for a in features
]
polys = []
for p in to_be_merged:
if type(p) == geos.Polygon:
polys.append(p)
elif type(p) == geos.MultiPolygon:
polys = polys + list(p)
feature = geos.MultiPolygon(polys)
elif ',' in feature_index:
indices = [int(a) for a in feature_index.split(',')]
to_be_merged = [
geos.GEOSGeometry(json.dumps(locations[a - 1]['geometry']))
for a in indices
]
polys = []
for p in to_be_merged:
if type(p) == geos.Polygon:
polys.append(p)
elif type(p) == geos.MultiPolygon:
polys = polys + list(p)
feature = geos.MultiPolygon(polys)
else:
feature_index = int(feature_index)
print(locations[feature_index - 1]['properties']['name'])
feature_json = json.dumps(locations[feature_index - 1]['geometry'])
feature = geos.GEOSGeometry(feature_json)
is_new_location = input('Is this a new location? (y/N) ')
is_new_location = True if 'Y' in is_new_location.upper() else False
# merged = feature.convex_hull if len(feature) > 3 else feature[0]
# merged = merged.simplify(tolerance=0.01, preserve_topology=True)
if is_new_location:
location_name = input('Name of location: ')
area_type = input(
'Area Type (1: Country; 2: Region): [1/2] ') or '1'
area_type = int(area_type)
location_slug = slugify(location_name)
parent_id = input('Parent Id: ')
location = models.GeographicRegion.objects.create(
name=location_name,
parent_id=int(parent_id) if parent_id else None,
geom=feature,
level=area_type,
slug=location_slug,
)
else:
location_id = input('Location Id: ')
location = models.GeographicRegion.objects.get(id=int(location_id))
location.geom = feature
location.save()
def convert2multi(validated_data, geo_field):
geom = validated_data.get(geo_field, None)
if geom and isinstance(geom, geos.Polygon):
geom = geos.MultiPolygon(geom)
validated_data[geo_field] = geom
def test_point_within_mpolygon_double(self):
create_points()
poly1, poly2 = create_polygons()
mpoly = geos.MultiPolygon(poly1, poly2)
features = Feature.objects.filter(geom_point__within=mpoly)
self.assertEqual(len(features), 2)