def test_out_of_date(self):
BoundarySet.objects.create(name='Foo', last_updated=date(2010, 1, 1))
self.assertTrue(Command().loadable('foo', date(2020, 1, 1)))
def test_get_version(self):
try:
Command().get_version()
except Exception as e:
self.fail('Exception %s raised: %s %s' %
(type(e).__name__, e, traceback.format_exc()))
def test_invalid_merge_strategy_when_nothing_to_merge(self):
try:
Command().load_boundary(self.feature, 'invalid')
except Exception as e:
self.fail('Exception %s raised: %s %s' %
(type(e).__name__, e, traceback.format_exc()))
def test_nonexisting(self):
self.assertTrue(Command().loadable('foo', date(2000, 1, 1)))
BoundarySet.objects.create(name='Foo', last_updated=date(2010, 1, 1))
self.assertFalse(Command().loadable('foo', date(2000, 1, 1)))
def test_up_to_date(self):
BoundarySet.objects.create(name='Foo', last_updated=date(2010, 1, 1))
self.assertFalse(Command().loadable('foo', date(2000, 1, 1)))
def import_shapefile(country_id: str, shapefile_url: str):
slug = shapefile_url.rsplit('/', 1)[-1].split('.')[0]
print(f"Importing {shapefile_url}")
country = Country.objects.get(id=country_id)
shapefile_path, etags = download_shapefile(country_id, shapefile_url)
try:
source_notes = Path(shapefile_path[:-4] + '-COPYRIGHT').read_text()
except FileNotFoundError:
source_notes = ''
try:
# Update the ETags
shapefile, _ = Shapefile.objects.select_for_update().get_or_create(
url=shapefile_url)
if shapefile.etags == etags:
return
load_shapefiles_command = LoadShapefilesCommand()
options = {'merge': None, 'clean': False}
definition = Definition({
'file':
shapefile_path,
'name':
'{} boundaries for {} ({})'.format(slug, country.label,
country.id),
'singular':
'boundary',
'encoding':
'utf-8',
'last_updated':
datetime.datetime.now(),
'name_func':
lambda feature: feature.get('WIKIDATA'),
'id_func':
lambda feature: feature.get('WIKIDATA'),
'notes':
source_notes
})
data_sources, tmpdirs = create_data_sources(
definition['file'],
encoding=definition['encoding'],
convert_3d_to_2d=options['clean'])
flatten_data_sources(data_sources)
load_shapefiles_command.load_boundary_set(slug=country_id + '-' + slug,
definition=definition,
data_sources=data_sources,
options=options)
# Update the ETags
shapefile.etags = etags
shapefile.save()
print("Done: {}".format(
sizeof_fmt(
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024)))
boundary_set = BoundarySet.objects.get(slug=country_id + '-' + slug)
update_wikidata_boundary_links(boundary_set)
finally:
# Always clear up the download directory
shutil.rmtree(os.path.dirname(shapefile_path))