def load(filename, drop=False):
'''Load a GeoZones Bundle'''
tmp = tempfile.mkdtemp()
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, join(tmp, 'geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp)
log.info('Loading GeoZones levels')
if (drop):
log.info('Dropping existing levels')
GeoLevel.drop_collection()
log.info('Loading levels.json')
total = 0
with open(join(tmp, 'levels.json')) as fp:
levels = json.load(fp)
for level in levels:
GeoLevel.objects.create(id=level['id'], name=level['label'],
parents=level['parents'])
total += 1
log.info('Loaded {0} levels'.format(total))
if (drop):
log.info('Dropping existing spatial zones')
GeoZone.drop_collection()
log.info('Loading zones.json')
total = 0
with open(join(tmp, 'zones.json')) as fp:
geozones = json.load(fp)
for zone in geozones['features']:
props = zone['properties']
GeoZone.objects.create(
id=zone['id'],
level=props['level'],
code=props['code'],
name=props['name'],
keys=props['keys'],
parents=props['parents'],
population=props.get('population'),
area=props.get('area'),
geom=zone['geometry']
)
total += 1
log.info('Loaded {0} zones'.format(total))
log.info('Cleaning temporary working directory')
shutil.rmtree(tmp)
def load(filename, drop=False):
'''Load a GeoZones Bundle'''
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
log.info('Loading GeoZones levels')
if drop:
log.info('Dropping existing levels')
GeoLevel.drop_collection()
log.info('Loading levels.msgpack')
levels_filepath = tmp.path('levels.msgpack')
with open(levels_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
for i, level in enumerate(unpacker, start=1):
GeoLevel.objects.create(
id=level['id'],
name=level['label'],
parents=level['parents']
)
os.remove(levels_filepath)
log.info('Loaded {total} levels'.format(total=i))
if drop:
log.info('Dropping existing spatial zones')
GeoZone.drop_collection()
log.info('Loading zones.msgpack')
zones_filepath = tmp.path('zones.msgpack')
with open(zones_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
for i, geozone in enumerate(unpacker, start=1):
GeoZone.objects.create(
id=geozone['_id'],
level=geozone['level'],
code=geozone['code'],
name=geozone['name'],
keys=geozone.get('keys'),
parents=geozone.get('parents'),
population=geozone.get('population'),
dbpedia=geozone.get('dbpedia'),
logo=geozone.get('flag') or geozone.get('blazon'),
wikipedia=geozone.get('wikipedia'),
area=geozone.get('area'),
geom=geozone['geom']
)
os.remove(zones_filepath)
log.info('Loaded {total} zones'.format(total=i))
shutil.rmtree(tmp.path('translations')) # Not in use for now.
def load(filename, drop=False):
'''Load a GeoZones Bundle'''
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
log.info('Loading GeoZones levels')
if drop:
log.info('Dropping existing levels')
GeoLevel.drop_collection()
log.info('Loading levels.msgpack')
levels_filepath = tmp.path('levels.msgpack')
with open(levels_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
for i, level in enumerate(unpacker, start=1):
GeoLevel.objects.create(id=level['id'],
name=level['label'],
parents=level['parents'],
admin_level=level.get('admin_level'))
os.remove(levels_filepath)
log.info('Loaded {total} levels'.format(total=i))
if drop:
log.info('Dropping existing spatial zones')
GeoZone.drop_collection()
log.info('Loading zones.msgpack')
zones_filepath = tmp.path('zones.msgpack')
with open(zones_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
for i, geozone in enumerate(unpacker, start=1):
GeoZone.objects.create(id=geozone['_id'],
level=geozone['level'],
code=geozone['code'],
name=geozone['name'],
keys=geozone.get('keys'),
parents=geozone.get('parents'),
population=geozone.get('population'),
dbpedia=geozone.get('dbpedia'),
logo=geozone.get('flag')
or geozone.get('blazon'),
wikipedia=geozone.get('wikipedia'),
area=geozone.get('area'),
geom=geozone['geom'])
os.remove(zones_filepath)
log.info('Loaded {total} zones'.format(total=i))
shutil.rmtree(tmp.path('translations')) # Not in use for now.
def load(filename, drop=False):
'''Load a GeoZones Bundle'''
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
log.info('Loading GeoZones levels')
if drop:
log.info('Dropping existing levels')
GeoLevel.drop_collection()
log.info('Loading levels.msgpack')
levels_filepath = tmp.path('levels.msgpack')
with open(levels_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
for i, level in enumerate(unpacker, start=1):
GeoLevel.objects.create(
id=level['id'],
name=level['label'],
parents=level['parents'],
admin_level=level.get('admin_level')
)
os.remove(levels_filepath)
log.info('Loaded {total} levels'.format(total=i))
if drop:
log.info('Dropping existing spatial zones')
GeoZone.drop_collection()
log.info('Loading zones.msgpack')
zones_filepath = tmp.path('zones.msgpack')
with open(zones_filepath) as fp:
unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
unpacker.next() # Skip headers.
for i, geozone in enumerate(unpacker):
if not geozone.get('geom') or (
geozone['geom']['type'] == 'GeometryCollection' and
not geozone['geom']['geometries']):
geom = None
else:
geom = geozone['geom']
params = {
'id': geozone['_id'],
'slug': slugify.slugify(geozone['name'], separator='-'),
'level': geozone['level'],
'code': geozone['code'],
'name': geozone['name'],
'keys': geozone.get('keys'),
'parents': geozone.get('parents'),
'ancestors': geozone.get('ancestors'),
'successors': geozone.get('successors'),
'validity': geozone.get('validity'),
'population': geozone.get('population'),
'dbpedia': geozone.get('dbpedia'),
'flag': geozone.get('flag'),
'blazon': geozone.get('blazon'),
'wikipedia': geozone.get('wikipedia'),
'area': geozone.get('area'),
'geom': geom
}
try:
GeoZone.objects.create(**params)
except errors.ValidationError as e:
log.warning('Validation error (%s) for %s with %s',
e, geozone, params)
continue
os.remove(zones_filepath)
log.info('Loaded {total} zones'.format(total=i))
shutil.rmtree(tmp.path('translations')) # Not in use for now.
