def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def process_position(row):
positioning = row.get('positioning')
if not positioning or not hasattr(Position, positioning.upper()):
positioning = Position.OTHER
source = row.get('source')
cia = row.get('housenumber:cia')
housenumber_ign = row.get('housenumber:ign')
housenumber = None
if cia:
cia = cia.upper()
housenumber = HouseNumber.first(HouseNumber.cia == cia)
elif housenumber_ign:
housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign)
if not housenumber:
reporter.error('Unable to find parent housenumber', row)
return
instance = None
if 'ign' in row:
# The only situation where we want to avoid creating new position is
# when we have the ign identifier.
instance = Position.first(Position.ign == row['ign'])
version = instance.version + 1 if instance else 1
data = dict(source=source,
housenumber=housenumber,
positioning=positioning,
version=version)
kind = row.get('kind', '')
data['attributes'] = row.get('attributes', {})
if hasattr(Position, kind.upper()):
data['kind'] = kind
elif not instance:
# We are creating a new position (not updating), kind is mandatory.
kind = Position.UNKNOWN
if kind:
data['kind'] = kind
populate(['ign', 'name', ('geometry', 'center')], row, data)
validator = Position.validator(instance=instance,
update=bool(instance),
**data)
if validator.errors:
reporter.error('Position error', validator.errors)
else:
try:
position = validator.save()
except peewee.IntegrityError as e:
reporter.error('Integrity error', (str(e), data))
else:
msg = 'Position updated' if instance else 'Position created'
reporter.notice(msg, position.id)
def process_position(row):
kind = row.get('kind')
if not hasattr(Position, kind.upper()):
kind = Position.UNKNOWN
positioning = row.get('positionning') # two "n" in the data.
if not positioning or not hasattr(Position, positioning.upper()):
positioning = Position.OTHER
source = row.get('source')
cia = row.get('housenumber:cia').upper()
center = row.get('geometry')
housenumber = HouseNumber.first(HouseNumber.cia == cia)
if not housenumber:
reporter.error('Position housenumber does not exist', cia)
return
instance = Position.first(Position.housenumber == housenumber,
Position.kind == kind, Position.source == source)
version = instance.version + 1 if instance else 1
data = dict(kind=kind, source=source, housenumber=housenumber,
center=center, positioning=positioning, version=version)
if 'ref:ign' in row:
data['ign'] = row.get('ref:ign')
validator = Position.validator(instance=instance, **data)
if validator.errors:
reporter.error('Position error', validator.errors)
else:
try:
position = validator.save()
except peewee.IntegrityError:
reporter.error('Integrity error', row)
else:
msg = 'Position updated' if instance else 'Position created'
reporter.notice(msg, position.id)
def add_housenumber(parent, id, metadata, postcode):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
ign = metadata.get('id')
data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk,
ign=ign)
if postcode:
data['postcodes'] = [postcode]
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center, version=1,
kind=Position.ENTRANCE,
housenumber=housenumber.pk)
if not validator.errors:
validator.save()
report('Position', validator.instance, report.NOTICE)
else:
report('Position error', validator.errors, report.ERROR)
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def add_housenumber(parent, id, metadata, postcode):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
ign = metadata.get('id')
data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk,
ign=ign)
if postcode:
data['postcode'] = postcode
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center, version=1,
kind=Position.ENTRANCE,
positioning=Position.OTHER,
housenumber=housenumber.pk)
if not validator.errors:
validator.save()
reporter.notice('Position', validator.instance)
else:
reporter.error('Position error', validator.errors)
reporter.notice('Housenumber created', housenumber)
else:
reporter.error('Housenumber error', validator.errors)
def process_housenumber(row):
number = row.get('numero')
ordinal = row.get('rep')
parent = 'ign:{}'.format(row.get('identifiant_fpb'))
postcode = 'code:{}'.format(row.get('code_post'))
complement = row.get('designation_de_l_entree')
ign = row.get('id')
lat = row.get('lat')
lon = row.get('lon')
localisation_type = row.get('type_de_localisation')
data = dict(number=number, ordinal=ordinal, ign=ign, parent=parent,
postcode=postcode)
laposte = row.get('cea')
if laposte:
data['laposte'] = laposte
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber error', validator.errors)
else:
try:
housenumber = validator.save()
except peewee.IntegrityError as e:
reporter.error('SQL Error ', str(e))
else:
reporter.notice('HouseNumber created', (number, ordinal))
if localisation_type != 'Au centre commune':
process_position(housenumber, (lon, lat), localisation_type,
{'complement': complement})
else:
reporter.notice('Skipped centre commune', str(housenumber))
def add_housenumber(parent, id, metadata, postcode):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
ign = metadata.get('id')
data = dict(number=number,
ordinal=ordinal,
version=1,
parent=parent.id,
ign=ign)
if postcode:
data['ancestors'] = [postcode]
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center,
version=1,
kind=Position.ENTRANCE,
housenumber=housenumber.id)
if not validator.errors:
validator.save()
report('Position', validator.instance, report.NOTICE)
else:
report('Position error', validator.errors, report.ERROR)
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def process_position(row):
positioning = row.get('positionning') # two "n" in the data.
if not positioning or not hasattr(Position, positioning.upper()):
positioning = Position.OTHER
source = row.get('source')
cia = row.get('housenumber:cia')
housenumber_ign = row.get('housenumber:ign')
housenumber = None
if cia:
cia = cia.upper()
housenumber = HouseNumber.first(HouseNumber.cia == cia)
elif housenumber_ign:
housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign)
if not housenumber:
reporter.error('Unable to find parent housenumber', row)
return
instance = None
if 'ign' in row:
# The only situation where we want to avoid creating new position is
# when we have the ign identifier.
instance = Position.first(Position.ign == row['ign'])
version = instance.version + 1 if instance else 1
data = dict(source=source, housenumber=housenumber,
positioning=positioning, version=version)
kind = row.get('kind', '')
if hasattr(Position, kind.upper()):
data['kind'] = kind
elif not instance:
# We are creating a new position (not updating), kind is mandatory.
kind = Position.UNKNOWN
if kind:
data['kind'] = kind
populate(['ign', 'name', ('geometry', 'center')], row, data)
validator = Position.validator(instance=instance, update=bool(instance),
**data)
if validator.errors:
reporter.error('Position error', validator.errors)
else:
try:
position = validator.save()
except peewee.IntegrityError as e:
reporter.error('Integrity error', (str(e), data))
else:
msg = 'Position updated' if instance else 'Position created'
reporter.notice(msg, position.id)
def add_cea(row):
ign = row.get('ID_ADR')
laposte = row.get('HEXACLE_1')
if laposte == 'NR':
return report('Missing CEA', ign, report.ERROR)
query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign)
done = query.execute()
if not done:
return report('IGN id not found', ign, report.ERROR)
report('Done', ign, report.NOTICE)
def add_cea(row):
ign = row.get('ID_ADR')
laposte = row.get('HEXACLE_1')
if laposte == 'NR':
return report('Missing CEA', ign, report.ERROR)
query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign)
try:
done = query.execute()
except peewee.IntegrityError:
report('Duplicate CEA', laposte, report.ERROR)
else:
if not done:
return report('IGN id not found', ign, report.ERROR)
report('Done', ign, report.NOTICE)
def add_housenumber(parent, id, metadata):
number, *ordinal = id.split(' ')
ordinal = ordinal[0] if ordinal else ''
center = [metadata['lon'], metadata['lat']]
data = dict(number=number, ordinal=ordinal, version=1)
data[parent.__class__.__name__.lower()] = parent.id
validator = HouseNumber.validator(**data)
if not validator.errors:
housenumber = validator.save()
validator = Position.validator(center=center, version=1,
housenumber=housenumber.id)
if not validator.errors:
validator.save()
report('Housenumber', housenumber, report.NOTICE)
else:
report('Housenumber error', validator.errors, report.ERROR)
def process_housenumber(line):
matricule = line[:8]
number = line[8:12].strip()
ordinal = line[13:23].strip()
laposte = line[23:33]
group_laposte = MATRICULE_TO_CEA.get(matricule)
group = 'laposte:{}'.format(group_laposte)
if not group_laposte:
return report('Missing group CEA', matricule, report.ERROR)
if not number:
return report('Not a housenumber', laposte, report.NOTICE)
validator = HouseNumber.validator(number=number, ordinal=ordinal,
laposte=laposte, parent=group)
if validator.errors:
report('Housenumber error', validator.errors, report.ERROR)
else:
validator.save()
report('Housenumber created', laposte, report.NOTICE)
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
if fantoir:
if not insee:
insee = fantoir[:5]
number = data.get('number')
ordinal = data.get('ordinal')
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if data.get('cia'):
data['cia'] = computed_cia
source = row.get('source')
data['attributes'] = {'source': source}
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = data.get('ign')
laposte = data.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
if instance and compute_cia:
if cia != computed_cia:
# Means new values are changing one of the four values of the
# cia (insee, fantoir, number, ordinal). Make sure we are not
# creating a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = data.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
import peewee
from ban.commands import command
from ban.core.encoder import dumps
from ban.core.models import (Group, HouseNumber, Municipality, Position,
PostCode)
from ban.db import database
from . import helpers
QUERIES = {
'PostCode': PostCode.select(),
'Municipality': Municipality.select(),
'Group': Group.select(),
'HouseNumber': HouseNumber.select(),
'Position': Position.select()
}
@command
def resources(resource, path, **kwargs):
"""Export database as resources in json stream format.
path path of file where to write resources
resource Municipality, PostCode, Group, HouseNumber or Position
"""
resources = [
'Municipality', 'PostCode', 'Group', 'HouseNumber', 'Position'
]
if resource not in resources:
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
number = row.get('number')
ordinal = row.get('ordinal')
source = row.get('source')
attributes = row.get('attributes', {})
if source:
attributes['source'] = source
data['attributes'] = attributes
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
complement = row.get('postcode:complement')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code, Municipality.insee == insee,
PostCode.complement == complement).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = row.get('ign')
laposte = row.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = row.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists',
(instance.cia, instance.ign, instance.laposte))
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def process_housenumber(row):
number = row.get('numero')
ordinal = row.get('ordinal') or None
fantoir = row.get('group:fantoir')
cia = row.get('cia')
if not fantoir and cia:
# 12xxx.json is missing group:fantoir.
fantoir = '{}{}'.format(cia[:5], cia[6:10])
insee = fantoir[:5]
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if not cia:
cia = computed_cia
parent = 'fantoir:{}'.format(fantoir)
source = row.get('source')
attributes = {'source': source}
data = dict(number=number, ordinal=ordinal, version=1, parent=parent,
attributes=attributes)
# Only override if key is present (even if value is null).
if 'ref:ign' in row:
data['ign'] = row['ref:ign']
if 'poste:cea' in row:
data['laposte'] = row['poste:cea']
if 'postcode' in row:
code = row.get('postcode')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
instance = HouseNumber.first(HouseNumber.cia == cia)
update = False
if instance:
if cia != computed_cia:
# Means new values are changing one of the four values of the cia
# (insee, fantoir, number, ordinal). Make sure we are not creating
# a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError:
reporter.warning('HouseNumber DB error', cia)
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
