def process_group(row, id, name, insee, group_id, fantoir):
municipality = 'insee:{}'.format(insee)
data = dict(name=name, fantoir=fantoir, municipality=municipality)
instance = None # Means creation.
if id:
try:
instance = Group.get(Group.id == id)
except Group.DoesNotExist:
return reporter.error('Group id not found', id)
elif data['fantoir']:
try:
instance = Group.get(Group.fantoir == data['fantoir'])
except Group.DoesNotExist:
pass # Let's create it.
if instance:
data['kind'] = instance.kind
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
else:
data['kind'] = Group.WAY
validator = Group.validator(instance=instance, **data)
if validator.errors:
reporter.error('Invalid data', validator.errors)
else:
street = validator.save()
msg = 'Created group' if not instance else 'Updated group'
reporter.notice(msg, street.id)
if row.get('lat') and row.get('long'):
process_housenumber(row, id, name, insee, group_id, fantoir)
def process_group(row, id, name, insee, group_id, fantoir):
municipality = 'insee:{}'.format(insee)
data = dict(name=name, fantoir=fantoir, municipality=municipality)
instance = None # Means creation.
if id:
try:
instance = Group.get(Group.id == id)
except Group.DoesNotExist:
return reporter.error('Group id not found', id)
elif data['fantoir']:
try:
instance = Group.get(Group.fantoir == data['fantoir'])
except Group.DoesNotExist:
pass # Let's create it.
if instance:
data['kind'] = instance.kind
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
else:
data['kind'] = Group.WAY
validator = Group.validator(instance=instance, **data)
if validator.errors:
reporter.error('Invalid data', validator.errors)
else:
street = validator.save()
msg = 'Created group' if not instance else 'Updated group'
reporter.notice(msg, street.id)
if row.get('lat') and row.get('long'):
process_housenumber(row, id, name, insee, group_id, fantoir)
def process_row(metadata):
name = metadata.get('name')
id = metadata.get('id')
insee = metadata.get('citycode')
code = metadata.get('postcode')
fantoir = ''.join(id.split('_')[:2])[:9]
kind = 'area' if metadata['type'] == 'locality' else 'way'
instance = Group.select().where(Group.fantoir == fantoir).first()
if instance:
return report('Existing {}'.format(Group.__name__),
{name: name, fantoir: fantoir},
report.WARNING)
try:
municipality = Municipality.get(Municipality.insee == insee)
except Municipality.DoesNotExist:
return report('Municipality does not exist', insee, report.ERROR)
validator = PostCode.validator(code=code, version=1,
name=municipality.name,
municipality=municipality)
if validator.errors:
report('Invalid postcode', code, report.ERROR)
postcode = None
else:
with PostCode._meta.database.atomic():
try:
postcode = validator.save()
except peewee.IntegrityError:
# Another thread created it?
postcode = PostCode.get(PostCode.code == code)
else:
report('Created postcode', code, report.NOTICE)
validator = Group.validator(name=name, fantoir=fantoir, kind=kind,
municipality=municipality.pk, version=1)
if not validator.errors:
try:
item = validator.save()
except peewee.IntegrityError:
return report('Duplicate group', fantoir, report.ERROR)
report(kind, item, report.NOTICE)
housenumbers = metadata.get('housenumbers')
if housenumbers:
for id, metadata in housenumbers.items():
add_housenumber(item, id, metadata, postcode)
else:
report('Street error', validator.errors, report.ERROR)
def process_group(row):
data = dict(version=1)
keys = ['name', ('group', 'kind'), 'laposte', 'ign', 'fantoir', 'alias']
populate(keys, row, data)
insee = row.get('municipality:insee')
if insee:
data['municipality'] = 'insee:{}'.format(insee)
source = row.get('source')
attributes = row.get('attributes', {})
if source:
attributes['source'] = source
data['attributes'] = attributes
if 'addressing' in row:
if hasattr(Group, row['addressing'].upper()):
data['addressing'] = row['addressing']
update = False
ign = data.get('ign')
fantoir = data.get('fantoir')
laposte = data.get('laposte')
if fantoir:
instance = Group.first(Group.fantoir == fantoir)
elif ign:
instance = Group.first(Group.ign == ign)
elif laposte:
instance = Group.first(Group.laposte == laposte)
else:
reporter.error('Missing group unique id', row)
return
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') and attributes.get('source') == source:
# Reimporting same data?
reporter.warning('Group already exist', fantoir)
return
data['version'] = instance.version + 1
if attributes:
attributes.update(data['attributes'])
data['attributes'] = attributes
update = True
validator = Group.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('Invalid group data', (validator.errors, row))
else:
try:
validator.save()
except peewee.IntegrityError:
reporter.error('Integrity Error', fantoir)
else:
msg = 'Group updated' if instance else 'Group created'
reporter.notice(msg, fantoir)
def process_group(row):
data = dict(version=1)
keys = ['name', ('group', 'kind'), 'laposte', 'ign', 'fantoir']
populate(keys, row, data)
insee = row.get('municipality:insee')
if insee:
data['municipality'] = 'insee:{}'.format(insee)
source = row.get('source')
attributes = row.get('attributes', {})
attributes['source'] = source
data['attributes'] = attributes
if 'addressing' in row:
if hasattr(Group, row['addressing'].upper()):
data['addressing'] = row['addressing']
update = False
ign = data.get('ign')
fantoir = data.get('fantoir')
laposte = data.get('laposte')
if fantoir:
instance = Group.first(Group.fantoir == fantoir)
elif ign:
instance = Group.first(Group.ign == ign)
elif laposte:
instance = Group.first(Group.laposte == laposte)
else:
reporter.error('Missing group unique id', row)
return
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('Group already exist', fantoir)
return
data['version'] = instance.version + 1
if attributes:
attributes.update(data['attributes'])
data['attributes'] = attributes
update = True
validator = Group.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('Invalid group data', (validator.errors, row))
else:
try:
validator.save()
except peewee.IntegrityError:
reporter.error('Integrity Error', fantoir)
else:
msg = 'Group updated' if instance else 'Group created'
reporter.notice(msg, fantoir)
def process_group(row):
fantoir = row.get('group:fantoir')
data = dict(version=1, fantoir=fantoir)
name = row.get('name')
if name:
data['name'] = name
kind = row.get('group')
if kind:
data['kind'] = kind
insee = row.get('municipality:insee')
if insee:
data['municipality'] = 'insee:{}'.format(insee)
laposte = row.get('poste:matricule')
if laposte:
data['laposte'] = laposte
source = row.get('source')
attributes = row.get('attributes', {})
attributes['source'] = source
data['attributes'] = attributes
update = False
instance = Group.first(Group.fantoir == fantoir)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('Group already exist', fantoir)
return
data['version'] = instance.version + 1
if attributes:
attributes.update(data['attributes'])
data['attributes'] = attributes
update = True
validator = Group.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('Invalid group data', (validator.errors, fantoir))
else:
try:
validator.save()
except peewee.IntegrityError:
reporter.error('Integrity Error', fantoir)
else:
msg = 'Group updated' if instance else 'Group created'
reporter.notice(msg, fantoir)
def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def process_group(row):
name = row.get('nom')
fantoir = row.get('id_fantoir')
municipality = 'insee:{}'.format(row.get('code_insee'))
ign = row.get('identifiant_fpb')
data = dict(name=name, fantoir=fantoir, municipality=municipality, ign=ign,
kind=Group.WAY)
laposte = row.get('id_poste') or None
if laposte:
data['laposte'] = laposte
validator = Group.validator(**data)
if validator.errors:
reporter.error('Je suis pas content', validator.errors)
else:
validator.save()
reporter.notice('Je suis content', name)
def process_group(line):
if not line[0] == 'V':
return report('Not a street', line, report.NOTICE)
name = line[60:92]
matricule = line[12:20]
laposte = MATRICULE_TO_CEA.get(matricule)
if not laposte:
return report('Missing CEA', matricule, report.ERROR)
municipality = 'insee:{}'.format(line[7:12])
kind = guess_kind(name, line[92:96].strip())
validator = Group.validator(name=name, laposte=laposte,
municipality=municipality, kind=kind)
if validator.errors:
report('Error', validator.errors, report.ERROR)
else:
validator.save()
report('Success', name, report.NOTICE)
import os
import peewee
from ban.commands import command
from ban.core.encoder import dumps
from ban.core.models import (Group, HouseNumber, Municipality, Position,
PostCode)
from ban.db import database
from . import helpers
QUERIES = {
'PostCode': PostCode.select(),
'Municipality': Municipality.select(),
'Group': Group.select(),
'HouseNumber': HouseNumber.select(),
'Position': Position.select()
}
@command
def resources(resource, path, **kwargs):
"""Export database as resources in json stream format.
path path of file where to write resources
resource Municipality, PostCode, Group, HouseNumber or Position
"""
resources = [
'Municipality', 'PostCode', 'Group', 'HouseNumber', 'Position'
]
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
number = row.get('number')
ordinal = row.get('ordinal')
source = row.get('source')
attributes = row.get('attributes', {})
if source:
attributes['source'] = source
data['attributes'] = attributes
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
complement = row.get('postcode:complement')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code, Municipality.insee == insee,
PostCode.complement == complement).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = row.get('ign')
laposte = row.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = row.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists',
(instance.cia, instance.ign, instance.laposte))
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
if fantoir:
if not insee:
insee = fantoir[:5]
number = data.get('number')
ordinal = data.get('ordinal')
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if data.get('cia'):
data['cia'] = computed_cia
source = row.get('source')
data['attributes'] = {'source': source}
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = data.get('ign')
laposte = data.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
if instance and compute_cia:
if cia != computed_cia:
# Means new values are changing one of the four values of the
# cia (insee, fantoir, number, ordinal). Make sure we are not
# creating a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = data.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)