def dummytoken(**kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == 'token').execute()
Token.create(session=session.id,
access_token="token",
expires_in=3600 * 24)
report('Created token', 'token', report.NOTICE)
def create(fail_silently=False, **kwargs):
"""Create database tables.
fail_silently Do not raise error if table already exists.
"""
for model in models:
model.create_table(fail_silently=fail_silently)
report('Created', model.__name__, report.NOTICE)
def add_cea(row):
ign = row.get('ID_ADR')
laposte = row.get('HEXACLE_1')
if laposte == 'NR':
return report('Missing CEA', ign, report.ERROR)
query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign)
done = query.execute()
if not done:
return report('IGN id not found', ign, report.ERROR)
report('Done', ign, report.NOTICE)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
code = metadata.get('postcode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = metadata['type']
klass = Street if kind == 'street' else Locality
instance = klass.select().where(klass.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(klass.__name__), {
name: name,
fantoir: fantoir
}, report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
validator = PostCode.validator(code=code,
version=1,
name=municipality.name,
municipality=municipality)
if validator.errors:
report('Invalid postcode', code, report.ERROR)
postcode = None
else:
with PostCode._meta.database.atomic():
try:
postcode = validator.save()
except peewee.IntegrityError:
# Another thread created it?
postcode = PostCode.get(PostCode.code == code)
else:
report('Created postcode', code, report.NOTICE)
data = dict(
name=name,
fantoir=fantoir,
municipality=municipality.id,
version=1,
)
validator = klass.validator(**data)
if not validator.errors:
item = validator.save()
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata, postcode)
else:
report('Street error', validator.errors, report.ERROR)
def resources(path, **kwargs):
"""Export database as resources in json stream format.
path path of file where to write resources
"""
resources = [models.PostCode, models.Municipality, models.Locality,
models.Street, models.HouseNumber]
with Path(path).open(mode='w', encoding='utf-8') as f:
for resource in resources:
for data in resource.select().as_resource_list():
f.write(dumps(data) + '\n')
# Memory consumption when exporting all France housenumbers?
report(resource.__name__, data, report.NOTICE)
def add_housenumber(parent, id, metadata, postcode):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
ign = metadata.get('id')
data = dict(number=number,
ordinal=ordinal,
version=1,
parent=parent.id,
ign=ign)
if postcode:
data['ancestors'] = [postcode]
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center,
version=1,
kind=Position.ENTRANCE,
housenumber=housenumber.id)
if not validator.errors:
validator.save()
report('Position', validator.instance, report.NOTICE)
else:
report('Position error', validator.errors, report.ERROR)
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def add_housenumber(parent, id, metadata, postcode):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
ign = metadata.get('id')
data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk,
ign=ign)
if postcode:
data['postcodes'] = [postcode]
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center, version=1,
kind=Position.ENTRANCE,
housenumber=housenumber.pk)
if not validator.errors:
validator.save()
report('Position', validator.instance, report.NOTICE)
else:
report('Position error', validator.errors, report.ERROR)
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
insee = line[6:11]
code = line[89:94]
name = line[90:]
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality Not Existing', insee, report.WARNING)
postcode, created = PostCode.get_or_create(code=code, name=name,
municipality=municipality,
defaults={'version': 1})
if created:
report('PostCode Added', postcode, report.NOTICE)
def resources(path, **kwargs):
"""Export database as resources in json stream format.
path path of file where to write resources
"""
resources = [
models.PostCode, models.Municipality, models.Locality, models.Street,
models.HouseNumber
]
with Path(path).open(mode='w', encoding='utf-8') as f:
for resource in resources:
for data in resource.select().as_resource_list():
f.write(dumps(data) + '\n')
# Memory consumption when exporting all France housenumbers?
report(resource.__name__, data, report.NOTICE)
def add_housenumber(parent, id, metadata):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
data = dict(number=number, ordinal=ordinal, version=1)
data[parent.__class__.__name__.lower()] = parent.id
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center, version=1,
housenumber=housenumber.id)
if not validator.errors:
validator.save()
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def add_municipality(data, update=False):
insee = data.get('insee')
name = data.get('nom_com')
siren = data.get('siren_com')
version = 1
try:
instance = models.Municipality.get(models.Municipality.insee == insee)
except models.Municipality.DoesNotExist:
instance = None
if instance and not update:
return report('Existing', name, report.WARNING)
data = dict(insee=insee, name=name, siren=siren, version=version)
validator = models.Municipality.validator(instance=instance, **data)
if not validator.errors:
instance = validator.save()
report('Processed', instance, report.NOTICE)
else:
report('Error', validator.errors, report.ERROR)
def add_municipality(data, update=False):
insee = data.get('insee')
name = data.get('nom_com')
siren = data.get('siren_com')
version = 1
try:
instance = models.Municipality.get(models.Municipality.insee == insee)
except models.Municipality.DoesNotExist:
instance = None
if instance and not update:
return report('Existing', name, report.WARNING)
data = dict(insee=insee, name=name, siren=siren, version=version)
validator = models.Municipality.validator(instance=instance, **data)
if not validator.errors:
instance = validator.save()
report('Processed', instance, report.NOTICE)
else:
report('Error', validator.errors, report.ERROR)
def createuser(username=None, email=None, is_staff=False, **kwargs):
"""Create a user.
is_staff set user staff
"""
if not username:
username = helpers.prompt('Username')
if not email:
email = helpers.prompt('Email')
password = helpers.prompt('Password', confirmation=True, hidden=True)
validator = User.validator(username=username, email=email)
if not validator.errors:
user = validator.save()
user.set_password(password)
if is_staff:
user.is_staff = True
user.save()
report('Created', user, report.NOTICE)
else:
report('Errored', validator.errors, report.ERROR)
def createuser(username=None, email=None, is_staff=False, **kwargs):
"""Create a user.
is_staff set user staff
"""
if not username:
username = helpers.prompt('Username')
if not email:
email = helpers.prompt('Email')
password = helpers.prompt('Password', confirmation=True, hidden=True)
validator = User.validator(username=username, email=email)
if not validator.errors:
user = validator.save()
user.set_password(password)
if is_staff:
user.is_staff = True
user.save()
report('Created', user, report.NOTICE)
else:
report('Errored', validator.errors, report.ERROR)
def truncate(force=False, names=[], **kwargs):
"""Truncate database tables.
force Do not ask for confirm.
names List of model names to truncate (in the given order).
"""
if not names:
# We expect names, not classes.
names = [m.__name__.lower() for m in models]
msg = 'Truncate all tables?'
else:
msg = 'Truncate tables: {}'.format(', '.join(names))
if not force and not helpers.confirm(msg, default=False):
helpers.abort('Aborted.')
# Delete in reverse way not to break FK constraints.
for model in models[::-1]:
name = model.__name__.lower()
if name not in names:
continue
model.delete().execute()
report('Truncated', name, report.NOTICE)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
insee = line[6:11]
code = line[89:94]
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality Not Existing', insee, report.WARNING)
postcode, created = PostCode.get_or_create(code=code, version=1)
if created:
report('PostCode Added', postcode, report.NOTICE)
try:
postcode.municipalities.add(municipality)
except peewee.IntegrityError:
report('Association Already Exists', postcode, report.WARNING)
else:
report('Association Added', postcode, report.NOTICE)
def add_cea(row):
ign = row.get('ID_ADR')
laposte = row.get('HEXACLE_1')
if laposte == 'NR':
return report('Missing CEA', ign, report.ERROR)
query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign)
try:
done = query.execute()
except peewee.IntegrityError:
report('Duplicate CEA', laposte, report.ERROR)
else:
if not done:
return report('IGN id not found', ign, report.ERROR)
report('Done', ign, report.NOTICE)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
municipality = 'insee:{}'.format(line[6:11])
code = line[89:94]
name = line[90:]
validator = PostCode.validator(code=code, name=name,
municipality=municipality)
if validator.errors:
return report('PostCode Error', validator.errors, report.ERROR)
else:
with PostCode._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError:
report('Already created', (code, municipality), report.WARNING)
else:
report('PostCode created', code, report.NOTICE)
def process_housenumber(line):
matricule = line[:8]
number = line[8:12].strip()
ordinal = line[13:23].strip()
laposte = line[23:33]
group_laposte = MATRICULE_TO_CEA.get(matricule)
group = 'laposte:{}'.format(group_laposte)
if not group_laposte:
return report('Missing group CEA', matricule, report.ERROR)
if not number:
return report('Not a housenumber', laposte, report.NOTICE)
validator = HouseNumber.validator(number=number, ordinal=ordinal,
laposte=laposte, parent=group)
if validator.errors:
report('Housenumber error', validator.errors, report.ERROR)
else:
validator.save()
report('Housenumber created', laposte, report.NOTICE)
def process_group(line):
if not line[0] == 'V':
return report('Not a street', line, report.NOTICE)
name = line[60:92]
matricule = line[12:20]
laposte = MATRICULE_TO_CEA.get(matricule)
if not laposte:
return report('Missing CEA', matricule, report.ERROR)
municipality = 'insee:{}'.format(line[7:12])
kind = guess_kind(name, line[92:96].strip())
validator = Group.validator(name=name, laposte=laposte,
municipality=municipality, kind=kind)
if validator.errors:
report('Error', validator.errors, report.ERROR)
else:
validator.save()
report('Success', name, report.NOTICE)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = metadata['type']
klass = Street if kind == 'street' else Locality
instance = klass.select().where(klass.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(klass.__name__),
{name: name, fantoir: fantoir},
report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
data = dict(
name=name,
fantoir=fantoir,
municipality=municipality.id,
version=1,
)
validator = klass.validator(**data)
if not validator.errors:
item = validator.save()
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata)
else:
report('Street error', validator.errors, report.ERROR)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
code = metadata.get('postcode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = 'area' if metadata['type'] == 'locality' else 'way'
instance = Group.select().where(Group.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(Group.__name__),
{name: name, fantoir: fantoir},
report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
validator = PostCode.validator(code=code, version=1,
name=municipality.name,
municipality=municipality)
if validator.errors:
report('Invalid postcode', code, report.ERROR)
postcode = None
else:
with PostCode._meta.database.atomic():
try:
postcode = validator.save()
except peewee.IntegrityError:
# Another thread created it?
postcode = PostCode.get(PostCode.code == code)
else:
report('Created postcode', code, report.NOTICE)
validator = Group.validator(name=name, fantoir=fantoir, kind=kind,
municipality=municipality.pk, version=1)
if not validator.errors:
try:
item = validator.save()
except peewee.IntegrityError:
return report('Duplicate group', fantoir, report.ERROR)
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata, postcode)
else:
report('Street error', validator.errors, report.ERROR)
def dummytoken(**kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == 'token').execute()
Token.create(session=session.id, access_token="token", expires_in=3600*24)
report('Created token', 'token', report.NOTICE)
