def process_postcode(row):
insee = row['municipality:insee']
municipality = 'insee:{}'.format(insee)
source = row.get('source')
attributes = {}
if source:
attributes = {'source': row.pop('source')}
name = row.get('name')
code = row.get('postcode')
complement = row.get('complement')
data = dict(name=name,
code=code,
municipality=municipality,
version=1,
attributes=attributes,
complement=complement)
instance = PostCode.select().join(Municipality).where(
PostCode.complement == complement, PostCode.code == code,
Municipality.insee == insee).first()
if instance:
return reporter.notice('PostCode already exists', code)
validator = PostCode.validator(**data)
if validator.errors:
return reporter.error('PostCode errors',
(validator.errors, code, insee))
validator.save()
reporter.notice('Imported PostCode', code)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
code = metadata.get('postcode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = metadata['type']
klass = Street if kind == 'street' else Locality
instance = klass.select().where(klass.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(klass.__name__), {
name: name,
fantoir: fantoir
}, report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
validator = PostCode.validator(code=code,
version=1,
name=municipality.name,
municipality=municipality)
if validator.errors:
report('Invalid postcode', code, report.ERROR)
postcode = None
else:
with PostCode._meta.database.atomic():
try:
postcode = validator.save()
except peewee.IntegrityError:
# Another thread created it?
postcode = PostCode.get(PostCode.code == code)
else:
report('Created postcode', code, report.NOTICE)
data = dict(
name=name,
fantoir=fantoir,
municipality=municipality.id,
version=1,
)
validator = klass.validator(**data)
if not validator.errors:
item = validator.save()
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata, postcode)
else:
report('Street error', validator.errors, report.ERROR)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
code = metadata.get('postcode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = 'area' if metadata['type'] == 'locality' else 'way'
instance = Group.select().where(Group.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(Group.__name__),
{name: name, fantoir: fantoir},
report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
validator = PostCode.validator(code=code, version=1,
name=municipality.name,
municipality=municipality)
if validator.errors:
report('Invalid postcode', code, report.ERROR)
postcode = None
else:
with PostCode._meta.database.atomic():
try:
postcode = validator.save()
except peewee.IntegrityError:
# Another thread created it?
postcode = PostCode.get(PostCode.code == code)
else:
report('Created postcode', code, report.NOTICE)
validator = Group.validator(name=name, fantoir=fantoir, kind=kind,
municipality=municipality.pk, version=1)
if not validator.errors:
try:
item = validator.save()
except peewee.IntegrityError:
return report('Duplicate group', fantoir, report.ERROR)
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata, postcode)
else:
report('Street error', validator.errors, report.ERROR)
def process_postcode(row):
insee = row['municipality:insee']
municipality = 'insee:{}'.format(insee)
attributes = {'source': row.pop('source')}
name = row.get('name')
code = row.get('postcode')
data = dict(name=name, code=code, municipality=municipality,
version=1, attributes=attributes)
instance = PostCode.select().join(Municipality).where(
PostCode.code == code, Municipality.insee == insee).first()
if instance:
return reporter.notice('PostCode already exists', code)
validator = PostCode.validator(**data)
if validator.errors:
return reporter.error('PostCode errors', (validator.errors,
code, insee))
validator.save()
reporter.notice('Imported PostCode', code)
def process_postcode(row):
name = row.get('libelle')
municipality = 'insee:{}'.format(row.get('code_insee'))
code = row.get('code_post')
validator = PostCode.validator(name=name, municipality=municipality,
code=code)
if validator.errors:
reporter.error('Postcode error', validator.errors)
else:
validator.save()
reporter.notice('Postcode saved', code)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
insee = line[6:11]
code = line[89:94]
name = line[90:]
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality Not Existing', insee, report.WARNING)
postcode, created = PostCode.get_or_create(code=code, name=name,
municipality=municipality,
defaults={'version': 1})
if created:
report('PostCode Added', postcode, report.NOTICE)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
insee = line[6:11]
code = line[89:94]
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality Not Existing', insee, report.WARNING)
postcode, created = PostCode.get_or_create(code=code, version=1)
if created:
report('PostCode Added', postcode, report.NOTICE)
try:
postcode.municipalities.add(municipality)
except peewee.IntegrityError:
report('Association Already Exists', postcode, report.WARNING)
else:
report('Association Added', postcode, report.NOTICE)
def process_postcode(line):
if line[50] != 'M':
return report('Cedex postcode', line, report.WARNING)
municipality = 'insee:{}'.format(line[6:11])
code = line[89:94]
name = line[90:]
validator = PostCode.validator(code=code, name=name,
municipality=municipality)
if validator.errors:
return report('PostCode Error', validator.errors, report.ERROR)
else:
with PostCode._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError:
report('Already created', (code, municipality), report.WARNING)
else:
report('PostCode created', code, report.NOTICE)
from pathlib import Path
import os
import peewee
from ban.commands import command
from ban.core.encoder import dumps
from ban.core.models import (Group, HouseNumber, Municipality, Position,
PostCode)
from ban.db import database
from . import helpers
QUERIES = {
'PostCode': PostCode.select(),
'Municipality': Municipality.select(),
'Group': Group.select(),
'HouseNumber': HouseNumber.select(),
'Position': Position.select()
}
@command
def resources(resource, path, **kwargs):
"""Export database as resources in json stream format.
path path of file where to write resources
resource Municipality, PostCode, Group, HouseNumber or Position
"""
resources = [
'Municipality', 'PostCode', 'Group', 'HouseNumber', 'Position'
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
number = row.get('number')
ordinal = row.get('ordinal')
source = row.get('source')
attributes = row.get('attributes', {})
if source:
attributes['source'] = source
data['attributes'] = attributes
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
complement = row.get('postcode:complement')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code, Municipality.insee == insee,
PostCode.complement == complement).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = row.get('ign')
laposte = row.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = row.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists',
(instance.cia, instance.ign, instance.laposte))
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
if fantoir:
if not insee:
insee = fantoir[:5]
number = data.get('number')
ordinal = data.get('ordinal')
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if data.get('cia'):
data['cia'] = computed_cia
source = row.get('source')
data['attributes'] = {'source': source}
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = data.get('ign')
laposte = data.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
if instance and compute_cia:
if cia != computed_cia:
# Means new values are changing one of the four values of the
# cia (insee, fantoir, number, ordinal). Make sure we are not
# creating a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = data.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def process_housenumber(row):
number = row.get('numero')
ordinal = row.get('ordinal') or None
fantoir = row.get('group:fantoir')
cia = row.get('cia')
if not fantoir and cia:
# 12xxx.json is missing group:fantoir.
fantoir = '{}{}'.format(cia[:5], cia[6:10])
insee = fantoir[:5]
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if not cia:
cia = computed_cia
parent = 'fantoir:{}'.format(fantoir)
source = row.get('source')
attributes = {'source': source}
data = dict(number=number, ordinal=ordinal, version=1, parent=parent,
attributes=attributes)
# Only override if key is present (even if value is null).
if 'ref:ign' in row:
data['ign'] = row['ref:ign']
if 'poste:cea' in row:
data['laposte'] = row['poste:cea']
if 'postcode' in row:
code = row.get('postcode')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
instance = HouseNumber.first(HouseNumber.cia == cia)
update = False
if instance:
if cia != computed_cia:
# Means new values are changing one of the four values of the cia
# (insee, fantoir, number, ordinal). Make sure we are not creating
# a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError:
reporter.warning('HouseNumber DB error', cia)
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))