def test_multiple_zip(self): path = fixture('multiple.zip') data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(tmpdirs), 4) self.assertEqual(len(data_sources), 5) paths = [ os.path.join(tmpdirs[0], 'bar.shp'), os.path.join(tmpdirs[0], 'foo.shp'), os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp'), os.path.join(tmpdirs[1], 'foo.shp'), os.path.join(tmpdirs[2], 'foo.shp'), os.path.join(tmpdirs[3], 'dir.zip', 'foo.shp'), ] zipfiles = [ path, os.path.join(tmpdirs[0], 'flat.zip'), os.path.join(tmpdirs[0], 'nested.zip'), ] for data_source in data_sources: self.assertIn(data_source.name, paths) self.assertEqual(data_source.layer_count, 1) if hasattr(data_source, 'zipfile'): self.assertIn(data_source.zipfile, zipfiles)
def test_nested(self): path = fixture("nested") data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(data_sources), 1) self.assertEqual(tmpdirs, []) self.assertEqual(data_sources[0].name, os.path.join(path, "dir.zip", "foo.shp")) self.assertEqual(data_sources[0].layer_count, 1)
def test_foo_shp(self): path = fixture('foo.shp') data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(data_sources), 1) self.assertEqual(tmpdirs, []) self.assertEqual(data_sources[0].name, path) self.assertEqual(data_sources[0].layer_count, 1)
def test_flat_zip(self): path = fixture('flat.zip') # foo.shp, etc. data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(data_sources), 1) self.assertEqual(len(tmpdirs), 1) self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'foo.shp')) self.assertEqual(data_sources[0].layer_count, 1)
def test_returns_shapefile(self): path = fixture('foo.shp') data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False) self.assertEqual(tmpdirs, []) self.assertEqual(len(data_sources), 1) self.assertEqual(data_sources[0].name, path) self.assertEqual(data_sources[0].layer_count, 1)
def test_returns_shapefile_from_zip(self): path = fixture('flat.zip') data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False) self.assertEqual(len(tmpdirs), 1) self.assertEqual(len(data_sources), 1) self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'foo.shp')) self.assertEqual(data_sources[0].layer_count, 1)
def test_nested_zip(self): path = fixture('nested.zip') data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(data_sources), 1) self.assertEqual(len(tmpdirs), 1) self.assertEqual(data_sources[0].name, os.path.join(tmpdirs[0], 'dir.zip', 'foo.shp')) self.assertEqual(data_sources[0].layer_count, 1)
def test_multiple(self): path = fixture("multiple") data_sources, tmpdirs = create_data_sources(path) self.assertEqual(len(tmpdirs), 3) self.assertEqual(len(data_sources), 5) paths = [ os.path.join(path, "bar.shp"), os.path.join(path, "foo.shp"), os.path.join(path, "dir.zip", "foo.shp"), os.path.join(tmpdirs[0], "foo.shp"), os.path.join(tmpdirs[1], "foo.shp"), os.path.join(tmpdirs[2], "dir.zip", "foo.shp"), ] zipfiles = [os.path.join(path, "flat.zip"), os.path.join(path, "nested.zip")] for data_source in data_sources: self.assertIn(data_source.name, paths) self.assertEqual(data_source.layer_count, 1) if hasattr(data_source, "zipfile"): self.assertIn(data_source.zipfile, zipfiles)
def test_returns_shapefiles_from_directory(self): path = fixture('multiple') data_sources, tmpdirs = create_data_sources({'encoding': 'ascii'}, path, False) self.assertEqual(len(tmpdirs), 2) self.assertEqual(len(data_sources), 4) paths = [ os.path.join(path, 'bar.shp'), os.path.join(path, 'foo.shp'), os.path.join(tmpdirs[0], 'foo.shp'), os.path.join(tmpdirs[1], 'foo.shp'), ] zipfiles = [ os.path.join(path, 'flat.zip'), os.path.join(path, 'nested.zip') ] for data_source in data_sources: self.assertTrue(data_source.name in paths) self.assertEqual(data_source.layer_count, 1) if hasattr(data_source, 'zipfile'): self.assertTrue(data_source.zipfile in zipfiles)
def handle(self, *args, **options): boundaries.autodiscover(options['data_dir']) for slug in sorted(boundaries.registry): name = slug slug = slugify(slug) definition = boundaries.registry[name] # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: features = OrderedDict() for data_source in data_sources: features[slug] = [] layer = data_source[0] for feature in layer: feature = Feature(feature, definition) if feature.is_valid(): features[slug].append((feature.id, feature.name)) for slug, features in features.items(): print('\n%s: %d' % (slug, len(features))) for properties in sorted(features): print('%s: %s' % properties) finally: for tmpdir in tmpdirs: rmtree(tmpdir)
def handle(self, *args, **options): boundaries.autodiscover(options['data_dir']) for slug, definition in boundaries.registry.items(): name = slug slug = slugify(slug) # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: features = OrderedDict() for data_source in data_sources: features[slug] = [] layer = data_source[0] for feature in layer: feature = Feature(feature, definition) if feature.is_valid(): features[slug].append((feature.id, feature.name)) for slug, features in features.items(): print('\n%s: %d' % (slug, len(features))) for properties in sorted(features): print('%s: %s' % properties) finally: for tmpdir in tmpdirs: rmtree(tmpdir)
def test_returns_nothing_from_empty_zip(self): result = create_data_sources({}, fixture('empty.zip'), False) self.assertIsNone(result)
def test_empty(self): data_sources, tmpdirs = create_data_sources(fixture('empty')) self.assertEqual(data_sources, []) self.assertEqual(len(tmpdirs), 1)
def test_returns_nothing_from_nested_directory(self): data_sources, tmpdirs = create_data_sources({}, fixture('nested'), False) self.assertEqual(tmpdirs, []) self.assertEqual(data_sources, [])
def test_empty_zip(self): data_sources, tmpdirs = create_data_sources( fixture('empty.zip')) # empty.txt self.assertEqual(data_sources, []) self.assertEqual(len(tmpdirs), 1)
def import_shapefile(country_id: str, shapefile_url: str): slug = shapefile_url.rsplit('/', 1)[-1].split('.')[0] print(f"Importing {shapefile_url}") country = Country.objects.get(id=country_id) shapefile_path, etags = download_shapefile(country_id, shapefile_url) try: source_notes = Path(shapefile_path[:-4] + '-COPYRIGHT').read_text() except FileNotFoundError: source_notes = '' try: # Update the ETags shapefile, _ = Shapefile.objects.select_for_update().get_or_create( url=shapefile_url) if shapefile.etags == etags: return load_shapefiles_command = LoadShapefilesCommand() options = {'merge': None, 'clean': False} definition = Definition({ 'file': shapefile_path, 'name': '{} boundaries for {} ({})'.format(slug, country.label, country.id), 'singular': 'boundary', 'encoding': 'utf-8', 'last_updated': datetime.datetime.now(), 'name_func': lambda feature: feature.get('WIKIDATA'), 'id_func': lambda feature: feature.get('WIKIDATA'), 'notes': source_notes }) data_sources, tmpdirs = create_data_sources( definition['file'], encoding=definition['encoding'], convert_3d_to_2d=options['clean']) flatten_data_sources(data_sources) load_shapefiles_command.load_boundary_set(slug=country_id + '-' + slug, definition=definition, data_sources=data_sources, options=options) # Update the ETags shapefile.etags = etags shapefile.save() print("Done: {}".format( sizeof_fmt( resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024))) boundary_set = BoundarySet.objects.get(slug=country_id + '-' + slug) update_wikidata_boundary_links(boundary_set) finally: # Always clear up the download directory shutil.rmtree(os.path.dirname(shapefile_path))
def test_empty_zip(self): data_sources, tmpdirs = create_data_sources(fixture('empty.zip')) # empty.txt self.assertEqual(data_sources, []) self.assertEqual(len(tmpdirs), 1)