def main(settings):
problems_datasource = create_GEOJSON('', 'problems.geojson')
admin_level_data_path = os.path.join(
settings.get('sources').get('data_directory'),
'{}.pbf'.format(settings.get('sources').get('admin_levels_file'))
)
if not(is_file_readable(admin_level_data_path)):
sys.exit(99)
lyr_read = AdminLevelReader(admin_level_data_path)
LOG.info('Detecting problems in the dataset...')
for layer, feature in lyr_read.readData():
# get data
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
continue
geom_raw = feature.GetGeometryRef()
bad_geom = check_bad_geom(geom_raw, osm_id)
# BONKERS features usually crash QGIS, we need to skip those
if bad_geom:
writeProblem(problems_datasource, osm_id, bad_geom)
continue
# write geojson problems file
problems_datasource = None
def main(settings, directory, bufferDistance, simplifyDistance, geojson):
abspath = os.path.abspath(directory)
if os.path.isdir(abspath):
if not(os.access(abspath, os.W_OK)):
LOG.error('Missing write permission: %s', abspath)
sys.exit(99)
else:
LOG.error('Directory not found: %s', abspath)
sys.exit(99)
admin_level_data_path = os.path.join(
settings.get('sources').get('data_directory'),
'{}.pbf'.format(settings.get('sources').get('admin_levels_file'))
)
if not(is_file_readable(admin_level_data_path)):
sys.exit(99)
lyr_read = AdminLevelReader(admin_level_data_path)
for layer, feature in lyr_read.readData():
# get data
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
continue
admin_level = feature.GetField('admin_level')
geom_raw = feature.GetGeometryRef()
bad_geom = check_bad_geom(geom_raw, osm_id)
# BONKERS features usually crash QGIS, we need to skip those
if bad_geom:
# skip further processing
continue
if feature.GetField('boundary') != 'administrative':
# skip further processing
continue
# process national level boundary
if admin_level == '2':
createPolys(
feature, abspath, bufferDistance, simplifyDistance, geojson,
osm_id
)
def main(settings, problems_geojson):
if problems_geojson:
problems_datasource = create_GEOJSON('', 'problems.geojson')
adm_0_temp = set()
adm_1_temp = set()
adm_2_temp = set()
unusable_features = set()
# setup index
spat_index_0 = rtree.index.Index()
# extract countries
admin_level_0 = {}
lyr_save = AdminLevelWriter.create_postgis('admin_level_0', settings)
admin_files = extractGADMArchive(settings.get('gadm_source').get('shp_package'))
LOG.debug('Admin files: {}'.format(admin_files))
if not(all(is_file_readable(admin_path) for admin_path in admin_files)):
sys.exit(99)
lyr_read = GADMAdminLevelReader(admin_files[0])
feature_id = 0
LOG.info('Started exporting admin_level_0 boundaries!')
for layer, feature in lyr_read.readData():
osm_id = '{}0'.format(feature.GetField('GID_0')) # append 0 to the osm_id to satisfy import_geojson checks
name = feature.GetField('name_0')
name_en = feature.GetField('name_0')
# iso3166 = 'NG'
geom_raw = ogr.ForceToMultiPolygon(feature.GetGeometryRef())
feature_data = (
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', 0),
('iso3166', osm_id),
('is_in', None)
)
bad_geom = check_bad_geom(geom_raw, osm_id)
# BONKERS features usually crash QGIS, we need to skip those
if bad_geom:
# add bad geom to the list
unusable_features.add(osm_id)
# skip further processing
if problems_geojson:
writeProblem(problems_datasource, osm_id, bad_geom)
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
lyr_save.saveFeature(feature_data, geom_raw)
admin_level_0.update({osm_id: prep(geom)})
spat_index_0.insert(
feature_id, geom.envelope.bounds, obj=osm_id
)
LOG.debug('Index %s, record %s', feature_id, osm_id)
feature_id += 1
lyr_read.datasource = None
lyr_save.datasource = None
# write geojson problems file
if problems_geojson:
problems_datasource = None
# extract states
admin_level_1 = {}
# create index
spat_index_1 = rtree.index.Index()
feature_id = 0
lyr_save = AdminLevelWriter.create_postgis('admin_level_1', settings)
if len(admin_files) < 2:
lyr_save = None
else:
lyr_read = GADMAdminLevelReader(admin_files[1])
for layer, feature in lyr_read.readData():
osm_id = feature.GetField('GID_1')
name = feature.GetField('name_1')
name_en = feature.GetField('name_1')
geom_raw = ogr.ForceToMultiPolygon(feature.GetGeometryRef())
if osm_id in unusable_features:
# skip this feature
LOG.debug(
'Feature previously marked as unusable: %s, skipping', osm_id
)
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
# check spatial relationship
# representative point is guaranteed within polygon
geom_repr = geom.representative_point()
# check index intersection
is_in = intersect_geom(geom_repr, spat_index_0, admin_level_0)
if not(is_in):
# if we can't determine relationship, skip this feature
LOG.info('Missing country information ... %s', osm_id)
continue
feature_data = (
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', 1),
('iso3166', None),
('is_in', is_in)
)
lyr_save.saveFeature(feature_data, geom_raw)
admin_level_1.update({osm_id: prep(geom)})
spat_index_1.insert(
feature_id, geom.envelope.bounds, obj=osm_id
)
LOG.debug('Index %s, record %s', feature_id, osm_id)
feature_id += 1
adm_1_temp.add(osm_id)
lyr_read.datasource = None
lyr_save.datasource = None
# extract counties
lyr_save = AdminLevelWriter.create_postgis('admin_level_2', settings)
if len(admin_files) < 3:
lyr_save = None
else:
lyr_read = GADMAdminLevelReader(admin_files[2])
for layer, feature in lyr_read.readData():
osm_id = feature.GetField('GID_2')
name = feature.GetField('name_2')
name_en = feature.GetField('name_2')
geom_raw = ogr.ForceToMultiPolygon(feature.GetGeometryRef())
if osm_id in unusable_features:
# skip this feature
LOG.debug(
'Feature previously marked as unusable: %s, skipping', osm_id
)
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
# representative point is guaranteed within polygon
geom_repr = geom.representative_point()
is_in = intersect_geom(geom_repr, spat_index_0, admin_level_0)
is_in_state = intersect_geom(geom_repr, spat_index_1, admin_level_1)
if not(is_in_state):
# if we can't determine relationship, skip this feature
LOG.info('Missing state information ... %s', osm_id)
continue
feature_data = (
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', 2),
('iso3166', None),
('is_in', is_in_state)
)
lyr_save.saveFeature(feature_data, geom_raw)
adm_2_temp.add(osm_id)
lyr_read.datasource = None
lyr_save.datasource = None
def main(settings, admin_levels, problems_geojson):
if problems_geojson:
problems_datasource = create_GEOJSON('', 'problems.geojson')
adm_0_temp = set()
adm_1_temp = set()
adm_2_temp = set()
unusable_features = set()
# setup index
spat_index_0 = rtree.index.Index()
# extract countries
admin_level_0 = {}
lyr_save = AdminLevelWriter.create_postgis('admin_level_0', settings)
admin_level_data_path = os.path.join(
settings.get('sources').get('data_directory'),
'{}.pbf'.format(settings.get('sources').get('admin_levels_file'))
)
if not(is_file_readable(admin_level_data_path)):
sys.exit(99)
lyr_read = AdminLevelReader(admin_level_data_path)
feature_id = 0
LOG.info('Started exporting admin_level_0 boundaries!')
for layer, feature in lyr_read.readData():
# get data
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
continue
admin_level = feature.GetField('admin_level')
name = feature.GetField('name')
name_en = feature.GetField('name:en')
iso3166 = feature.GetField('ISO3166-1')
if iso3166:
iso3166 = iso3166.upper()
geom_raw = feature.GetGeometryRef()
feature_data = []
bad_geom = check_bad_geom(geom_raw, osm_id)
# BONKERS features usually crash QGIS, we need to skip those
if bad_geom:
# add bad geom to the list
unusable_features.add(osm_id)
# skip further processing
if problems_geojson:
writeProblem(problems_datasource, osm_id, bad_geom)
continue
if feature.GetField('boundary') != 'administrative':
LOG.debug(
'Feature %s, boundary tag value: %s',
osm_id,
feature.GetField('boundary')
)
# add bad feature to unusable features
unusable_features.add(osm_id)
# skip further processing
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
# process national level boundary
if admin_level == '2':
feature_data = [
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', admin_level),
('iso3166', iso3166),
('is_in', None)
]
lyr_save.saveFeature(feature_data, geom_raw)
admin_level_0.update({osm_id: prep(geom)})
spat_index_0.insert(
feature_id, geom.envelope.bounds, obj=osm_id
)
LOG.debug('Index %s, record %s', feature_id, osm_id)
feature_id += 1
# add feature to temporary set
adm_0_temp.add(osm_id)
lyr_read.datasource = None
lyr_save.datasource = None
# write geojson problems file
if problems_geojson:
problems_datasource = None
# extract states
admin_level_1 = {}
# create index
spat_index_1 = rtree.index.Index()
feature_id = 0
lyr_save = AdminLevelWriter.create_postgis('admin_level_1', settings)
lyr_read = AdminLevelReader(admin_level_data_path)
for layer, feature in lyr_read.readData():
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
continue
admin_level = feature.GetField('admin_level')
name = feature.GetField('name')
name_en = feature.GetField('name:en')
geom_raw = feature.GetGeometryRef()
feature_data = []
if osm_id in unusable_features:
# skip this feature
LOG.debug(
'Feature previously marked as unusable: %s, skipping', osm_id
)
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
# check spatial relationship
# representative point is guaranteed within polygon
geom_repr = geom.representative_point()
# check index intersection
is_in = intersect_geom(geom_repr, spat_index_0, admin_level_0)
# check for specific admin level mapping
if is_in in admin_levels.get('per_country'):
search_admin_level = (
admin_levels['per_country'][is_in]['admin_level_1']
)
if search_admin_level:
LOG.info(
'Using custom admin_level for %s (%s)',
admin_levels.get('per_country')
.get(is_in).get('meta').get('name'), is_in
)
else:
# use the default admin_level
search_admin_level = (
admin_levels['default']['admin_level_1']
)
elif is_in:
search_admin_level = (
admin_levels['default']['admin_level_1']
)
else:
# if we can't determine relationship, skip this feature
continue
if not(is_in):
# if we can't determine relationship, skip this feature
LOG.info('Missing country information ... %s', osm_id)
continue
# check current feature admin level
if admin_level == str(search_admin_level):
feature_data = [
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', admin_level),
('is_in', is_in)
]
lyr_save.saveFeature(feature_data, geom_raw)
admin_level_1.update({osm_id: prep(geom)})
spat_index_1.insert(
feature_id, geom.envelope.bounds, obj=osm_id
)
LOG.debug('Index %s, record %s', feature_id, osm_id)
feature_id += 1
adm_1_temp.add(osm_id)
lyr_read.datasource = None
lyr_save.datasource = None
# extract counties
lyr_save = AdminLevelWriter.create_postgis('admin_level_2', settings)
lyr_read = AdminLevelReader(admin_level_data_path)
for layer, feature in lyr_read.readData():
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
continue
admin_level = feature.GetField('admin_level')
name = feature.GetField('name')
name_en = feature.GetField('name:en')
geom_raw = feature.GetGeometryRef()
feature_data = []
if osm_id in unusable_features:
# skip this feature
LOG.debug(
'Feature previously marked as unusable: %s, skipping', osm_id
)
continue
geom = shapely.wkb.loads(geom_raw.ExportToWkb())
# representative point is guaranteed within polygon
geom_repr = geom.representative_point()
is_in = intersect_geom(geom_repr, spat_index_0, admin_level_0)
is_in_state = intersect_geom(geom_repr, spat_index_1, admin_level_1)
# check for specific admin level mapping
if is_in in admin_levels.get('per_country'):
search_admin_level = (
admin_levels.get('per_country')
.get(is_in)
.get('admin_level_2')
)
if search_admin_level:
LOG.info(
'Using custom admin_level for %s (%s)',
admin_levels.get('per_country')
.get(is_in).get('meta').get('name'), is_in
)
else:
# use the default admin_level
search_admin_level = (
admin_levels.get('default').get('admin_level_2')
)
elif is_in:
search_admin_level = (
admin_levels.get('default').get('admin_level_2')
)
else:
# if we can't determine relationship, skip this feature
continue
if not(is_in_state):
# if we can't determine relationship, skip this feature
LOG.info('Missing state information ... %s', osm_id)
continue
# check current feature admin level
if admin_level == str(search_admin_level):
feature_data = [
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', admin_level),
('is_in', is_in_state)
]
lyr_save.saveFeature(feature_data, geom_raw)
adm_2_temp.add(osm_id)
lyr_read.datasource = None
lyr_save.datasource = None
write_admin_check_files(adm_0_temp, adm_1_temp, adm_2_temp)
def main(settings):
lyr_discard = DiscardFeatureWriter.create_shp('discarded', settings)
lyr_save1 = AdminLevelWriter.create_shp('admin_level_1', settings)
lyr_save2 = AdminLevelWriter.create_shp('admin_level_2', settings)
lyr_save3 = AdminLevelWriter.create_shp('admin_level_3', settings)
lyr_save4 = AdminLevelWriter.create_shp('admin_level_4', settings)
lyr_save5 = AdminLevelWriter.create_shp('admin_level_5', settings)
lyr_save6 = AdminLevelWriter.create_shp('admin_level_6', settings)
lyr_save7 = AdminLevelWriter.create_shp('admin_level_7', settings)
lyr_save8 = AdminLevelWriter.create_shp('admin_level_8', settings)
lyr_save9 = AdminLevelWriter.create_shp('admin_level_9', settings)
lyr_save10 = AdminLevelWriter.create_shp('admin_level_10', settings)
admin_level_data_path = os.path.join(
settings.get('sources').get('data_directory'),
'{}.pbf'.format(settings.get('sources').get('admin_levels_file'))
)
if not(is_file_readable(admin_level_data_path)):
sys.exit(99)
lyr_read = AdminLevelReader(admin_level_data_path)
LOG.info('Started exporting admin_level_0 boundaries!')
for layer, feature in lyr_read.readData():
# get data
osm_id = feature.GetField('osm_id')
admin_level = feature.GetField('admin_level')
name = feature.GetField('name')
name_en = feature.GetField('name:en')
geom = feature.GetGeometryRef()
bad_geom = check_bad_geom(geom, osm_id)
# BONKERS features usually crash QGIS, we need to skip those
if bad_geom and bad_geom != 'BONKERS!':
feature_data = [
('osm_id', osm_id),
('name', name),
('adminlevel', admin_level),
('reason', bad_geom)
]
lyr_discard.saveFeature(feature_data, geom)
# skip further processing
continue
elif bad_geom == 'BONKERS!':
continue
# osm_id is crucial for establishing feature relationship
osm_id = prepare_osm_id(feature, layer)
if not osm_id:
LOG.warning('Feature without OSM_ID, discarding... "%s"', name)
feature_data = [
('osm_id', osm_id),
('name', name),
('adminlevel', admin_level),
('reason', 'Feature without OSM_ID!')
]
lyr_discard.saveFeature(feature_data, geom)
continue
feature_data = [
('osm_id', osm_id),
('name', name),
('name_en', name_en),
('adminlevel', admin_level),
('is_in', None)
]
# process national level boundary
if admin_level == '1':
LOG.debug('Writing admin_level=1, feature %s', osm_id)
lyr_save1.saveFeature(feature_data, geom)
if admin_level == '2':
LOG.debug('Writing admin_level=2, feature %s', osm_id)
lyr_save2.saveFeature(feature_data, geom)
if admin_level == '3':
LOG.debug('Writing admin_level=3, feature %s', osm_id)
lyr_save3.saveFeature(feature_data, geom)
if admin_level == '4':
LOG.debug('Writing admin_level=4, feature %s', osm_id)
lyr_save4.saveFeature(feature_data, geom)
if admin_level == '5':
LOG.debug('Writing admin_level=5, feature %s', osm_id)
lyr_save5.saveFeature(feature_data, geom)
if admin_level == '6':
LOG.debug('Writing admin_level=6, feature %s', osm_id)
lyr_save6.saveFeature(feature_data, geom)
if admin_level == '7':
LOG.debug('Writing admin_level=7, feature %s', osm_id)
lyr_save7.saveFeature(feature_data, geom)
if admin_level == '8':
LOG.debug('Writing admin_level=8, feature %s', osm_id)
lyr_save8.saveFeature(feature_data, geom)
if admin_level == '9':
LOG.debug('Writing admin_level=9, feature %s', osm_id)
lyr_save9.saveFeature(feature_data, geom)
if admin_level == '10':
LOG.debug('Writing admin_level=10, feature %s', osm_id)
lyr_save10.saveFeature(feature_data, geom)
lyr_read.datasource = None
lyr_save1.datasource = None
lyr_save2.datasource = None
lyr_save3.datasource = None
lyr_save4.datasource = None
lyr_save5.datasource = None
lyr_save6.datasource = None
lyr_save7.datasource = None
lyr_save8.datasource = None
lyr_save9.datasource = None
lyr_save10.datasource = None
lyr_discard.datasource = None