def test_multiple_zip(self):
path = fixture('multiple.zip')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(tmpdirs), 4)
self.assertEqual(len(data_sources), 5)
paths = [
os.path.join(tmpdirs[0], 'bar.shp'),
os.path.join(tmpdirs[0], 'foo.shp'),
os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp'),
os.path.join(tmpdirs[1], 'foo.shp'),
os.path.join(tmpdirs[2], 'foo.shp'),
os.path.join(tmpdirs[3], 'dir.zip', 'foo.shp'),
]
zipfiles = [
path,
os.path.join(tmpdirs[0], 'flat.zip'),
os.path.join(tmpdirs[0], 'nested.zip'),
]
for data_source in data_sources:
self.assertIn(data_source.name, paths)
self.assertEqual(data_source.layer_count, 1)
if hasattr(data_source, 'zipfile'):
self.assertIn(data_source.zipfile, zipfiles)
def test_nested(self):
path = fixture("nested")
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(tmpdirs, [])
self.assertEqual(data_sources[0].name, os.path.join(path, "dir.zip", "foo.shp"))
self.assertEqual(data_sources[0].layer_count, 1)
def test_foo_shp(self):
path = fixture('foo.shp')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(tmpdirs, [])
self.assertEqual(data_sources[0].name, path)
self.assertEqual(data_sources[0].layer_count, 1)
def test_foo_shp(self):
path = fixture('foo.shp')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(tmpdirs, [])
self.assertEqual(data_sources[0].name, path)
self.assertEqual(data_sources[0].layer_count, 1)
def test_flat_zip(self):
path = fixture('flat.zip') # foo.shp, etc.
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(len(tmpdirs), 1)
self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'foo.shp'))
self.assertEqual(data_sources[0].layer_count, 1)
def test_multiple_zip(self):
path = fixture('multiple.zip')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(tmpdirs), 4)
self.assertEqual(len(data_sources), 5)
paths = [
os.path.join(tmpdirs[0], 'bar.shp'),
os.path.join(tmpdirs[0], 'foo.shp'),
os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp'),
os.path.join(tmpdirs[1], 'foo.shp'),
os.path.join(tmpdirs[2], 'foo.shp'),
os.path.join(tmpdirs[3], 'dir.zip', 'foo.shp'),
]
zipfiles = [
path,
os.path.join(tmpdirs[0], 'flat.zip'),
os.path.join(tmpdirs[0], 'nested.zip'),
]
for data_source in data_sources:
self.assertIn(data_source.name, paths)
self.assertEqual(data_source.layer_count, 1)
if hasattr(data_source, 'zipfile'):
self.assertIn(data_source.zipfile, zipfiles)
def test_returns_shapefile(self):
path = fixture('foo.shp')
data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False)
self.assertEqual(tmpdirs, [])
self.assertEqual(len(data_sources), 1)
self.assertEqual(data_sources[0].name, path)
self.assertEqual(data_sources[0].layer_count, 1)
def test_returns_shapefile_from_zip(self):
path = fixture('flat.zip')
data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False)
self.assertEqual(len(tmpdirs), 1)
self.assertEqual(len(data_sources), 1)
self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'foo.shp'))
self.assertEqual(data_sources[0].layer_count, 1)
def test_nested_zip(self):
path = fixture('nested.zip')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(len(tmpdirs), 1)
self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp'))
self.assertEqual(data_sources[0].layer_count, 1)
def test_flat_zip(self):
path = fixture('flat.zip') # foo.shp, etc.
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(len(tmpdirs), 1)
self.assertEqual(data_sources[0].name,
os.path.join(tmpdirs[0], 'foo.shp'))
self.assertEqual(data_sources[0].layer_count, 1)
def test_nested_zip(self):
path = fixture('nested.zip')
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(data_sources), 1)
self.assertEqual(len(tmpdirs), 1)
self.assertEqual(data_sources[0].name,
os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp'))
self.assertEqual(data_sources[0].layer_count, 1)
def test_multiple(self):
path = fixture("multiple")
data_sources, tmpdirs = create_data_sources(path)
self.assertEqual(len(tmpdirs), 3)
self.assertEqual(len(data_sources), 5)
paths = [
os.path.join(path, "bar.shp"),
os.path.join(path, "foo.shp"),
os.path.join(path, "dir.zip", "foo.shp"),
os.path.join(tmpdirs[0], "foo.shp"),
os.path.join(tmpdirs[1], "foo.shp"),
os.path.join(tmpdirs[2], "dir.zip", "foo.shp"),
]
zipfiles = [os.path.join(path, "flat.zip"), os.path.join(path, "nested.zip")]
for data_source in data_sources:
self.assertIn(data_source.name, paths)
self.assertEqual(data_source.layer_count, 1)
if hasattr(data_source, "zipfile"):
self.assertIn(data_source.zipfile, zipfiles)
def test_returns_shapefiles_from_directory(self):
path = fixture('multiple')
data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False)
self.assertEqual(len(tmpdirs), 2)
self.assertEqual(len(data_sources), 4)
paths = [
os.path.join(path, 'bar.shp'),
os.path.join(path, 'foo.shp'),
os.path.join(tmpdirs[0], 'foo.shp'),
os.path.join(tmpdirs[1], 'foo.shp'),
]
zipfiles = [
os.path.join(path, 'flat.zip'),
os.path.join(path, 'nested.zip')
]
for data_source in data_sources:
self.assertTrue(data_source.name in paths)
self.assertEqual(data_source.layer_count, 1)
if hasattr(data_source, 'zipfile'):
self.assertTrue(data_source.zipfile in zipfiles)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug in sorted(boundaries.registry):
name = slug
slug = slugify(slug)
definition = boundaries.registry[name]
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
def test_returns_nothing_from_empty_zip(self):
result = create_data_sources({}, fixture('empty.zip'), False)
self.assertIsNone(result)
def test_empty(self):
data_sources, tmpdirs = create_data_sources(fixture('empty'))
self.assertEqual(data_sources, [])
self.assertEqual(len(tmpdirs), 1)
def test_returns_nothing_from_nested_directory(self):
data_sources, tmpdirs = create_data_sources({}, fixture('nested'), False)
self.assertEqual(tmpdirs, [])
self.assertEqual(data_sources, [])
def test_empty_zip(self):
data_sources, tmpdirs = create_data_sources(
fixture('empty.zip')) # empty.txt
self.assertEqual(data_sources, [])
self.assertEqual(len(tmpdirs), 1)
def test_empty(self):
data_sources, tmpdirs = create_data_sources(fixture('empty'))
self.assertEqual(data_sources, [])
self.assertEqual(len(tmpdirs), 1)
def import_shapefile(country_id: str, shapefile_url: str):
slug = shapefile_url.rsplit('/', 1)[-1].split('.')[0]
print(f"Importing {shapefile_url}")
country = Country.objects.get(id=country_id)
shapefile_path, etags = download_shapefile(country_id, shapefile_url)
try:
source_notes = Path(shapefile_path[:-4] + '-COPYRIGHT').read_text()
except FileNotFoundError:
source_notes = ''
try:
# Update the ETags
shapefile, _ = Shapefile.objects.select_for_update().get_or_create(
url=shapefile_url)
if shapefile.etags == etags:
return
load_shapefiles_command = LoadShapefilesCommand()
options = {'merge': None, 'clean': False}
definition = Definition({
'file':
shapefile_path,
'name':
'{} boundaries for {} ({})'.format(slug, country.label,
country.id),
'singular':
'boundary',
'encoding':
'utf-8',
'last_updated':
datetime.datetime.now(),
'name_func':
lambda feature: feature.get('WIKIDATA'),
'id_func':
lambda feature: feature.get('WIKIDATA'),
'notes':
source_notes
})
data_sources, tmpdirs = create_data_sources(
definition['file'],
encoding=definition['encoding'],
convert_3d_to_2d=options['clean'])
flatten_data_sources(data_sources)
load_shapefiles_command.load_boundary_set(slug=country_id + '-' + slug,
definition=definition,
data_sources=data_sources,
options=options)
# Update the ETags
shapefile.etags = etags
shapefile.save()
print("Done: {}".format(
sizeof_fmt(
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024)))
boundary_set = BoundarySet.objects.get(slug=country_id + '-' + slug)
update_wikidata_boundary_links(boundary_set)
finally:
# Always clear up the download directory
shutil.rmtree(os.path.dirname(shapefile_path))
def test_empty_zip(self):
data_sources, tmpdirs = create_data_sources(fixture('empty.zip')) # empty.txt
self.assertEqual(data_sources, [])
self.assertEqual(len(tmpdirs), 1)
