def test_get(self):
definition = Definition({
'name': '',
'name_func': lambda feature: None,
})
self.assertEqual(definition.get('name'), '')
self.assertEqual(definition.get('nonexistent', 'default'), 'default')
def test_get(self):
definition = Definition({
'name': '',
'name_func': lambda feature: None,
})
self.assertEqual(definition.get('name'), '')
self.assertEqual(definition.get('nonexistent', 'default'), 'default')
def setUp(self):
self.serializer_class = serializers.CountrySerializer
self.country = models.Country(id='Q145')
self.definition = Definition({
'name_func': lambda feature: 'name',
'id_func': lambda feature: 'id',
'name': 'test-set'
})
self.boundary_set = BoundarySet.objects.create(
slug='test-set', last_updated=datetime.datetime.now())
self.feature = DataSource(
json.dumps({
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [
[
[30, 10],
[40, 40],
[20, 40],
[10, 20],
[30, 10],
],
],
},
}))[0][0]
self.boundary = Feature(self.feature,
boundary_set=self.boundary_set,
definition=self.definition).create_boundary()
self.request_factory = APIRequestFactory()
def test_str(self):
definition = Definition({
'name': 'Test',
'name_func': lambda feature: '',
})
self.assertEqual(str(definition), 'Test')
def test_singular_override(self):
definition = Definition({
'name': 'Districts',
'name_func': lambda feature: None,
'singular': 'Singular',
})
self.assertEqual(definition['singular'], 'Singular')
def test_extra_default(self):
definition = Definition({
'name': '',
'name_func': lambda feature: None,
'metadata': {
'geographic_code': '01'
},
})
self.assertEqual(definition['extra'], {'geographic_code': '01'})
self.assertNotIn('metadata', definition)
def test_extra_default(self):
definition = Definition({
'name': '',
'name_func': lambda feature: None,
'extra': {
'id': 'ocd-division/country:ca'
},
})
self.assertEqual(definition['extra'],
{'id': 'ocd-division/country:ca'})
def handle(self, *args, **options):
if settings.DEBUG:
print(
_('DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]'
))
if input().lower() != 'y':
return
boundaries.autodiscover(options['data_dir'])
if options['only']:
whitelist = set(options['only'].split(','))
else:
whitelist = set()
if options['except']:
blacklist = set(options['except'].split(','))
else:
blacklist = set()
with alive_bar(len(boundaries.registry.items())) as bar:
for slug, definition in boundaries.registry.items():
bar()
name = slug
slug = slugify(slug)
if self.loadable(slug, definition['last_updated'], whitelist,
blacklist, options['reload']):
#log.info(_('Processing %(slug)s.') % {'slug': slug})
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(
definition['file'],
encoding=definition['encoding'],
convert_3d_to_2d=options['clean'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
self.load_boundary_set(slug, definition,
data_sources, options)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
else:
log.debug(_('Skipping %(slug)s.') % {'slug': slug})
def setUp(self):
self.definition = definition = Definition({
'last_updated':
date(2000, 1, 1),
'encoding':
'utf-8',
'name':
'Districts',
'name_func':
clean_attr('Name'),
'id_func':
attr('ID'),
'slug_func':
attr('Code'),
'is_valid_func':
lambda f: f.get('ID') == '1',
'label_point_func':
lambda f: Point(0, 1),
})
self.fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
self.boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
self.feature = Feature(FeatureProxy(self.fields), definition)
self.other = Feature(
FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), self.boundary_set)
def test_defaults(self):
definition = Definition({
'name': '',
'name_func': lambda feature: 'Test',
})
self.assertCountEqual(definition.dictionary.keys(), [
'name',
'name_func',
'encoding',
'domain',
'authority',
'source_url',
'licence_url',
'start_date',
'end_date',
'notes',
'extra',
'id_func',
'slug_func',
'is_valid_func',
'label_point_func',
])
self.assertEqual(definition['name'], '')
self.assertEqual(definition['name_func']({}), 'Test')
self.assertEqual(definition['encoding'], 'ascii')
self.assertEqual(definition['domain'], '')
self.assertEqual(definition['authority'], '')
self.assertEqual(definition['source_url'], '')
self.assertEqual(definition['licence_url'], '')
self.assertEqual(definition['start_date'], None)
self.assertEqual(definition['end_date'], None)
self.assertEqual(definition['notes'], '')
self.assertEqual(definition['extra'], {})
self.assertEqual(definition['id_func']({}), '')
self.assertEqual(definition['slug_func']({}), 'Test')
self.assertEqual(definition['is_valid_func']({}), True)
self.assertEqual(definition['label_point_func']({}), None)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
class LoadBoundaryTestCase(TestCase):
definition = Definition({
'last_updated': date(2000, 1, 1),
'name': 'Districts',
'name_func': lambda feature: 'Test',
})
boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
feature = Feature(FeatureProxy({}), definition, boundary_set=boundary_set)
def test_no_merge_strategy(self):
boundary = Command().load_boundary(self.feature)
self.assertEqual(boundary.set, self.boundary_set)
self.assertEqual(boundary.set_name, 'District')
self.assertEqual(boundary.external_id, '')
self.assertEqual(boundary.name, 'Test')
self.assertEqual(boundary.slug, 'test')
self.assertEqual(boundary.metadata, {})
self.assertEqual(boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
self.assertEqual(boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666666)')
self.assertEqual(boundary.extent,
(0.0, 0.0, 4.999999999999999, 4.999999999999999))
self.assertEqual(boundary.label_point, None)
def test_invalid_merge_strategy_when_nothing_to_merge(self):
try:
Command().load_boundary(self.feature, 'invalid')
except Exception as e:
self.fail('Exception %s raised: %s %s' %
(type(e).__name__, e, traceback.format_exc()))
def test_invalid_merge_strategy(self):
Command().load_boundary(self.feature, 'invalid')
self.assertRaisesRegexp(
ValueError,
r"\AThe merge strategy 'invalid' must be 'combine' or 'union'.\Z",
Command().load_boundary, self.feature, 'invalid')
def test_combine_merge_strategy(self):
self.boundary_set.save()
Command().load_boundary(self.feature, 'invalid')
boundary = Command().load_boundary(self.feature, 'combine')
self.assertEqual(
boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)),((0 0,0.0001 0.0001,0 5,5 5,0 0)))'
)
self.assertEqual(
boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0 5,5 5,0 0)),((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666667)')
self.assertEqual(boundary.extent, (0.0, 0.0, 5.0, 5.0))
def test_union_merge_strategy(self):
self.boundary_set.save()
Command().load_boundary(self.feature, 'invalid')
boundary = Command().load_boundary(self.feature, 'union')
self.assertEqual(
boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0.0001 0.0001,0 5,0 5,0 5,5 5,5 5,0.0001 0.0001)))'
)
self.assertEqual(
boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0.0001 0.0001,0 5,5 5,5 5,0.0001 0.0001)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666667)')
self.assertEqual(boundary.extent, (0.0, 0.0001, 5.0, 5.0))
def test_overrides(self):
definition = Definition({
'name': 'Federal',
'name_func': lambda feature: 'Name',
'encoding': 'iso-8859-1',
'domain': 'Canada',
'authority': 'Her Majesty the Queen in Right of Canada',
'source_url':
'http://data.gc.ca/data/en/dataset/48f10fb9-78a2-43a9-92ab-354c28d30674',
'licence_url':
'http://data.gc.ca/eng/open-government-licence-canada',
'start_date': date(2000, 1, 1),
'end_date': date(2010, 1, 1),
'notes': 'Notes',
'extra': {
'geographic_code': '01'
},
'id_func': lambda feature: 'ID',
'slug_func': lambda feature: 'Slug',
'is_valid_func': lambda feature: False,
'label_point_func': lambda feature: '',
})
self.assertCountEqual(definition.dictionary.keys(), [
'name',
'name_func',
'encoding',
'domain',
'authority',
'source_url',
'licence_url',
'start_date',
'end_date',
'notes',
'extra',
'id_func',
'slug_func',
'is_valid_func',
'label_point_func',
])
self.assertEqual(definition['name'], 'Federal')
self.assertEqual(definition['name_func']({}), 'Name')
self.assertEqual(definition['encoding'], 'iso-8859-1')
self.assertEqual(definition['domain'], 'Canada')
self.assertEqual(definition['authority'],
'Her Majesty the Queen in Right of Canada')
self.assertEqual(
definition['source_url'],
'http://data.gc.ca/data/en/dataset/48f10fb9-78a2-43a9-92ab-354c28d30674'
)
self.assertEqual(
definition['licence_url'],
'http://data.gc.ca/eng/open-government-licence-canada')
self.assertEqual(definition['start_date'], date(2000, 1, 1))
self.assertEqual(definition['end_date'], date(2010, 1, 1))
self.assertEqual(definition['notes'], 'Notes')
self.assertEqual(definition['extra'], {'geographic_code': '01'})
self.assertEqual(definition['id_func']({}), 'ID')
self.assertEqual(definition['slug_func']({}), 'Slug')
self.assertEqual(definition['is_valid_func']({}), False)
self.assertEqual(definition['label_point_func']({}), '')
def test_singular_default(self):
definition = Definition({
'name': 'Districts',
'name_func': lambda feature: None,
})
self.assertEqual(definition['singular'], 'District')
def import_shapefile(country_id: str, shapefile_url: str):
slug = shapefile_url.rsplit('/', 1)[-1].split('.')[0]
print(f"Importing {shapefile_url}")
country = Country.objects.get(id=country_id)
shapefile_path, etags = download_shapefile(country_id, shapefile_url)
try:
source_notes = Path(shapefile_path[:-4] + '-COPYRIGHT').read_text()
except FileNotFoundError:
source_notes = ''
try:
# Update the ETags
shapefile, _ = Shapefile.objects.select_for_update().get_or_create(
url=shapefile_url)
if shapefile.etags == etags:
return
load_shapefiles_command = LoadShapefilesCommand()
options = {'merge': None, 'clean': False}
definition = Definition({
'file':
shapefile_path,
'name':
'{} boundaries for {} ({})'.format(slug, country.label,
country.id),
'singular':
'boundary',
'encoding':
'utf-8',
'last_updated':
datetime.datetime.now(),
'name_func':
lambda feature: feature.get('WIKIDATA'),
'id_func':
lambda feature: feature.get('WIKIDATA'),
'notes':
source_notes
})
data_sources, tmpdirs = create_data_sources(
definition['file'],
encoding=definition['encoding'],
convert_3d_to_2d=options['clean'])
flatten_data_sources(data_sources)
load_shapefiles_command.load_boundary_set(slug=country_id + '-' + slug,
definition=definition,
data_sources=data_sources,
options=options)
# Update the ETags
shapefile.etags = etags
shapefile.save()
print("Done: {}".format(
sizeof_fmt(
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024)))
boundary_set = BoundarySet.objects.get(slug=country_id + '-' + slug)
update_wikidata_boundary_links(boundary_set)
finally:
# Always clear up the download directory
shutil.rmtree(os.path.dirname(shapefile_path))
class FeatureTestCase(TestCase):
definition = Definition({
'last_updated': date(2000, 1, 1),
'encoding': 'utf-8',
'name': 'Districts',
'name_func': clean_attr('Name'),
'id_func': attr('ID'),
'slug_func': attr('Code'),
'is_valid_func': lambda f: f.get('ID') == '1',
'label_point_func': lambda f: Point(0, 1),
})
fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
feature = Feature(FeatureProxy(fields), definition)
other = Feature(FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), boundary_set)
def test_init(self):
self.assertEqual(self.feature.boundary_set, None)
self.assertEqual(self.other.boundary_set, self.boundary_set)
def test_str(self):
self.assertEqual(str(self.feature), 'Valid')
self.assertEqual(str(self.other), 'Invalid')
def test_get(self):
self.assertEqual(self.feature.get('Name'), 'VALID')
def test_is_valid(self):
self.assertTrue(self.feature.is_valid())
self.assertFalse(self.other.is_valid())
def test_name(self):
self.assertEqual(self.feature.name, 'Valid')
self.assertEqual(self.other.name, 'Invalid')
def test_id(self):
self.assertEqual(self.feature.id, '1')
self.assertEqual(self.other.id, '100')
def test_slug(self):
self.assertEqual(self.feature.slug, 'foo-bar-baz-bzz')
self.assertEqual(self.other.slug, '3')
def test_label_point(self):
self.assertEqual(self.feature.label_point, Point(0, 1))
def test_metadata(self):
self.assertEqual(self.feature.metadata, self.fields)
def test_boundary_set(self):
self.feature.boundary_set = self.boundary_set
self.assertEqual(self.feature.boundary_set, self.boundary_set)
self.feature.boundary_set = None
def test_create_boundary(self):
self.feature.boundary_set = self.boundary_set
self.boundary_set.save()
boundary = self.feature.create_boundary()
self.assertEqual(boundary.set, self.boundary_set)
self.assertEqual(boundary.set_name, 'District')
self.assertEqual(boundary.external_id, '1')
self.assertEqual(boundary.name, 'Valid')
self.assertEqual(boundary.slug, 'foo-bar-baz-bzz')
self.assertEqual(boundary.metadata, self.fields)
self.assertEqual(boundary.shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
self.assertEqual(boundary.simple_shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt, 'POINT (1.6667 3.333366666666666)')
self.assertEqual(boundary.extent, (0.0, 0.0, 4.999999999999999, 4.999999999999999))
self.assertEqual(boundary.label_point, Point(0, 1))
self.feature.boundary_set = None