def setUp(self):
self.definition = definition = Definition({
'last_updated': date(2000, 1, 1),
'encoding': 'utf-8',
'name': 'Districts',
'name_func': clean_attr('Name'),
'id_func': attr('ID'),
'slug_func': attr('Code'),
'is_valid_func': lambda f: f.get('ID') == '1',
'label_point_func': lambda f: Point(0, 1),
})
self.fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
self.boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
self.feature = Feature(FeatureProxy(self.fields), definition)
self.other = Feature(FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), self.boundary_set)
def load_boundary_set(self, slug, definition, data_sources, options):
BoundarySet.objects.filter(
slug=slug).delete() # also deletes boundaries
boundary_set = BoundarySet.objects.create(
slug=slug,
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
domain=definition['domain'],
authority=definition['authority'],
source_url=definition['source_url'],
licence_url=definition['licence_url'],
start_date=definition['start_date'],
end_date=definition['end_date'],
notes=definition['notes'],
extra=definition['extra'],
)
boundary_set.extent = [None, None, None,
None] # [xmin, ymin, xmax, ymax]
for data_source in data_sources:
log.info(
_('Loading %(slug)s from %(source)s') % {
'slug': slug,
'source': data_source.name
})
layer = data_source[0]
layer.source = data_source # to trace the layer back to its source
if definition.get('srid'):
srs = SpatialReference(definition['srid'])
else:
srs = layer.srs
for feature in layer:
feature = Feature(feature, definition, srs, boundary_set)
feature.layer = layer # to trace the feature back to its source
if feature.is_valid():
log.info(_('%(slug)s...') % {'slug': feature.slug})
boundary = self.load_boundary(feature, options['merge'])
boundary_set.extend(boundary.extent)
if None not in boundary_set.extent: # unless there are no features
boundary_set.save()
log.info(
_('%(slug)s count: %(count)i') % {
'slug': slug,
'count': Boundary.objects.filter(set=boundary_set).count()
})
def setUp(self):
self.serializer_class = serializers.CountrySerializer
self.country = models.Country(id='Q145')
self.definition = Definition({
'name_func': lambda feature: 'name',
'id_func': lambda feature: 'id',
'name': 'test-set'
})
self.boundary_set = BoundarySet.objects.create(
slug='test-set', last_updated=datetime.datetime.now())
self.feature = DataSource(
json.dumps({
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [
[
[30, 10],
[40, 40],
[20, 40],
[10, 20],
[30, 10],
],
],
},
}))[0][0]
self.boundary = Feature(self.feature,
boundary_set=self.boundary_set,
definition=self.definition).create_boundary()
self.request_factory = APIRequestFactory()
def load_boundary_set(self, slug, definition, data_sources, options):
BoundarySet.objects.filter(slug=slug).delete() # also deletes boundaries
boundary_set = BoundarySet.objects.create(
slug=slug,
last_updated=definition["last_updated"],
name=definition["name"],
singular=definition["singular"],
domain=definition["domain"],
authority=definition["authority"],
source_url=definition["source_url"],
licence_url=definition["licence_url"],
start_date=definition["start_date"],
end_date=definition["end_date"],
notes=definition["notes"],
extra=definition["extra"],
)
boundary_set.extent = [None, None, None, None] # [xmin, ymin, xmax, ymax]
for data_source in data_sources:
log.info(_("Loading %(slug)s from %(source)s") % {"slug": slug, "source": data_source.name})
layer = data_source[0]
layer.source = data_source # to trace the layer back to its source
if definition.get("srid"):
srs = SpatialReference(definition["srid"])
else:
srs = layer.srs
for feature in layer:
feature = Feature(feature, definition, srs, boundary_set)
feature.layer = layer # to trace the feature back to its source
if feature.is_valid():
log.info(_("%(slug)s...") % {"slug": feature.slug})
boundary = self.load_boundary(feature, options["merge"])
boundary_set.extend(boundary.extent)
if None not in boundary_set.extent: # unless there are no features
boundary_set.save()
log.info(
_("%(slug)s count: %(count)i") % {"slug": slug, "count": Boundary.objects.filter(set=boundary_set).count()}
)
def load_boundary_set(self, slug, definition, data_sources, options):
BoundarySet.objects.filter(slug=slug).delete() # also deletes boundaries
boundary_set = BoundarySet.objects.create(
slug=slug,
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
domain=definition['domain'],
authority=definition['authority'],
source_url=definition['source_url'],
licence_url=definition['licence_url'],
start_date=definition['start_date'],
end_date=definition['end_date'],
notes=definition['notes'],
extra=definition['extra'],
)
boundary_set.extent = [None, None, None, None] # [xmin, ymin, xmax, ymax]
for data_source in data_sources:
log.info(_('Loading %(slug)s from %(source)s') % {'slug': slug, 'source': data_source.name})
layer = data_source[0]
layer.source = data_source # to trace the layer back to its source
if definition.get('srid'):
srs = SpatialReference(definition['srid'])
else:
srs = layer.srs
for feature in layer:
feature = Feature(feature, definition, srs, boundary_set)
feature.layer = layer # to trace the feature back to its source
if feature.is_valid():
log.info(_('%(slug)s...') % {'slug': feature.slug})
boundary = self.load_boundary(feature, options['merge'])
boundary_set.extend(boundary.extent)
if None not in boundary_set.extent: # unless there are no features
boundary_set.save()
log.info(_('%(slug)s count: %(count)i') % {'slug': slug, 'count': Boundary.objects.filter(set=boundary_set).count()})
def setUp(self):
self.definition = definition = Definition({
'last_updated':
date(2000, 1, 1),
'encoding':
'utf-8',
'name':
'Districts',
'name_func':
clean_attr('Name'),
'id_func':
attr('ID'),
'slug_func':
attr('Code'),
'is_valid_func':
lambda f: f.get('ID') == '1',
'label_point_func':
lambda f: Point(0, 1),
})
self.fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
self.boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
self.feature = Feature(FeatureProxy(self.fields), definition)
self.other = Feature(
FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), self.boundary_set)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug in sorted(boundaries.registry):
name = slug
slug = slugify(slug)
definition = boundaries.registry[name]
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
class LoadBoundaryTestCase(TestCase):
definition = Definition({
'last_updated': date(2000, 1, 1),
'name': 'Districts',
'name_func': lambda feature: 'Test',
})
boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
feature = Feature(FeatureProxy({}), definition, boundary_set=boundary_set)
def test_no_merge_strategy(self):
boundary = Command().load_boundary(self.feature)
self.assertEqual(boundary.set, self.boundary_set)
self.assertEqual(boundary.set_name, 'District')
self.assertEqual(boundary.external_id, '')
self.assertEqual(boundary.name, 'Test')
self.assertEqual(boundary.slug, 'test')
self.assertEqual(boundary.metadata, {})
self.assertEqual(boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
self.assertEqual(boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666666)')
self.assertEqual(boundary.extent,
(0.0, 0.0, 4.999999999999999, 4.999999999999999))
self.assertEqual(boundary.label_point, None)
def test_invalid_merge_strategy_when_nothing_to_merge(self):
try:
Command().load_boundary(self.feature, 'invalid')
except Exception as e:
self.fail('Exception %s raised: %s %s' %
(type(e).__name__, e, traceback.format_exc()))
def test_invalid_merge_strategy(self):
Command().load_boundary(self.feature, 'invalid')
self.assertRaisesRegexp(
ValueError,
r"\AThe merge strategy 'invalid' must be 'combine' or 'union'.\Z",
Command().load_boundary, self.feature, 'invalid')
def test_combine_merge_strategy(self):
self.boundary_set.save()
Command().load_boundary(self.feature, 'invalid')
boundary = Command().load_boundary(self.feature, 'combine')
self.assertEqual(
boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)),((0 0,0.0001 0.0001,0 5,5 5,0 0)))'
)
self.assertEqual(
boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0 0,0 5,5 5,0 0)),((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666667)')
self.assertEqual(boundary.extent, (0.0, 0.0, 5.0, 5.0))
def test_union_merge_strategy(self):
self.boundary_set.save()
Command().load_boundary(self.feature, 'invalid')
boundary = Command().load_boundary(self.feature, 'union')
self.assertEqual(
boundary.shape.ogr.wkt,
'MULTIPOLYGON (((0.0001 0.0001,0 5,0 5,0 5,5 5,5 5,0.0001 0.0001)))'
)
self.assertEqual(
boundary.simple_shape.ogr.wkt,
'MULTIPOLYGON (((0.0001 0.0001,0 5,5 5,5 5,0.0001 0.0001)))')
self.assertEqual(boundary.centroid.ogr.wkt,
'POINT (1.6667 3.333366666666667)')
self.assertEqual(boundary.extent, (0.0, 0.0001, 5.0, 5.0))
class FeatureTestCase(TestCase):
def setUp(self):
self.definition = definition = Definition({
'last_updated': date(2000, 1, 1),
'encoding': 'utf-8',
'name': 'Districts',
'name_func': clean_attr('Name'),
'id_func': attr('ID'),
'slug_func': attr('Code'),
'is_valid_func': lambda f: f.get('ID') == '1',
'label_point_func': lambda f: Point(0, 1),
})
self.fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
self.boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
self.feature = Feature(FeatureProxy(self.fields), definition)
self.other = Feature(FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), self.boundary_set)
def test_init(self):
self.assertEqual(self.feature.boundary_set, None)
self.assertEqual(self.other.boundary_set, self.boundary_set)
def test_str(self):
self.assertEqual(str(self.feature), 'Valid')
self.assertEqual(str(self.other), 'Invalid')
def test_get(self):
self.assertEqual(self.feature.get('Name'), 'VALID')
def test_is_valid(self):
self.assertTrue(self.feature.is_valid())
self.assertFalse(self.other.is_valid())
def test_name(self):
self.assertEqual(self.feature.name, 'Valid')
self.assertEqual(self.other.name, 'Invalid')
def test_id(self):
self.assertEqual(self.feature.id, '1')
self.assertEqual(self.other.id, '100')
def test_slug(self):
self.assertEqual(self.feature.slug, 'foo-bar-baz-bzz')
self.assertEqual(self.other.slug, '3')
def test_label_point(self):
self.assertEqual(self.feature.label_point, Point(0, 1))
def test_metadata(self):
self.assertEqual(self.feature.metadata, self.fields)
def test_boundary_set(self):
self.feature.boundary_set = self.boundary_set
self.assertEqual(self.feature.boundary_set, self.boundary_set)
self.feature.boundary_set = None
def test_create_boundary(self):
self.feature.boundary_set = self.boundary_set
self.boundary_set.save()
boundary = self.feature.create_boundary()
self.assertEqual(boundary.set, self.boundary_set)
self.assertEqual(boundary.set_name, 'District')
self.assertEqual(boundary.external_id, '1')
self.assertEqual(boundary.name, 'Valid')
self.assertEqual(boundary.slug, 'foo-bar-baz-bzz')
self.assertEqual(boundary.metadata, self.fields)
self.assertEqual(boundary.shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
self.assertEqual(boundary.simple_shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt, 'POINT (1.6667 3.333366666666666)')
self.assertEqual(boundary.extent, (0.0, 0.0, 4.999999999999999, 4.999999999999999))
self.assertEqual(boundary.label_point, Point(0, 1, srid=4326))
self.assertEqual(boundary.start_date, None)
self.assertEqual(boundary.end_date, None)
self.feature.boundary_set = None
class FeatureTestCase(TestCase):
definition = Definition({
'last_updated': date(2000, 1, 1),
'encoding': 'utf-8',
'name': 'Districts',
'name_func': clean_attr('Name'),
'id_func': attr('ID'),
'slug_func': attr('Code'),
'is_valid_func': lambda f: f.get('ID') == '1',
'label_point_func': lambda f: Point(0, 1),
})
fields = {
'Name': 'VALID',
'ID': '1',
'Code': '\tFoo—Bar–Baz \r Bzz\n', # m-dash, n-dash
}
boundary_set = BoundarySet(
last_updated=definition['last_updated'],
name=definition['name'],
singular=definition['singular'],
)
feature = Feature(FeatureProxy(fields), definition)
other = Feature(FeatureProxy({
'Name': 'INVALID',
'ID': 100,
'Code': 3,
}), definition, SpatialReference(4269), boundary_set)
def test_init(self):
self.assertEqual(self.feature.boundary_set, None)
self.assertEqual(self.other.boundary_set, self.boundary_set)
def test_str(self):
self.assertEqual(str(self.feature), 'Valid')
self.assertEqual(str(self.other), 'Invalid')
def test_get(self):
self.assertEqual(self.feature.get('Name'), 'VALID')
def test_is_valid(self):
self.assertTrue(self.feature.is_valid())
self.assertFalse(self.other.is_valid())
def test_name(self):
self.assertEqual(self.feature.name, 'Valid')
self.assertEqual(self.other.name, 'Invalid')
def test_id(self):
self.assertEqual(self.feature.id, '1')
self.assertEqual(self.other.id, '100')
def test_slug(self):
self.assertEqual(self.feature.slug, 'foo-bar-baz-bzz')
self.assertEqual(self.other.slug, '3')
def test_label_point(self):
self.assertEqual(self.feature.label_point, Point(0, 1))
def test_metadata(self):
self.assertEqual(self.feature.metadata, self.fields)
def test_boundary_set(self):
self.feature.boundary_set = self.boundary_set
self.assertEqual(self.feature.boundary_set, self.boundary_set)
self.feature.boundary_set = None
def test_create_boundary(self):
self.feature.boundary_set = self.boundary_set
self.boundary_set.save()
boundary = self.feature.create_boundary()
self.assertEqual(boundary.set, self.boundary_set)
self.assertEqual(boundary.set_name, 'District')
self.assertEqual(boundary.external_id, '1')
self.assertEqual(boundary.name, 'Valid')
self.assertEqual(boundary.slug, 'foo-bar-baz-bzz')
self.assertEqual(boundary.metadata, self.fields)
self.assertEqual(boundary.shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
self.assertEqual(boundary.simple_shape.ogr.wkt, 'MULTIPOLYGON (((0 0,0 5,5 5,0 0)))')
self.assertEqual(boundary.centroid.ogr.wkt, 'POINT (1.6667 3.333366666666666)')
self.assertEqual(boundary.extent, (0.0, 0.0, 4.999999999999999, 4.999999999999999))
self.assertEqual(boundary.label_point, Point(0, 1))
self.feature.boundary_set = None