def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def process_housenumber(row, id, name, insee, group_id, fantoir):
number = row.get('numero')
if number == '99999':
# Means it's a group address point, according to AITF weird specs.
number = None
ordinal = row.get('suffixe') or None
lat = row.get('lat')
lon = row.get('long')
kind = row.get('position')
cia = None
instance = None
data = dict(number=number, ordinal=ordinal)
if id:
instance = HouseNumber.where(HouseNumber.id == id).first()
if not instance:
return reporter.error('HouseNumber id not found', id)
parent = instance.parent
elif fantoir:
parent = 'fantoir:{}'.format(fantoir)
cia = compute_cia(insee, fantoir[5:], number, ordinal)
elif group_id:
parent = Group.where(Group.id == group_id).first()
if not parent:
return reporter.error('Group id not found', group_id)
if parent.fantoir:
cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
ordinal)
else:
return reporter.error('Missing group id and fantoir', id)
if cia:
instance = HouseNumber.where(HouseNumber.cia == cia).first()
if instance:
# Well… the BAL can't give us a BAN reference version, be kind for now.
# See https://github.com/etalab/ban/issues/91#issuecomment-198432574
# and https://github.com/etalab/ban/issues/94
data['version'] = instance.version + 1
data['instance'] = instance
data['parent'] = parent
validator = HouseNumber.validator(**data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
else:
try:
housenumber = validator.save()
except peewee.IntegrityError:
return reporter.error('Duplicate housenumber',
(number, ordinal, parent))
if lon and lat:
process_position(housenumber, (lon, lat), kind)
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def compute_cia(self):
return compute_cia(str(self.parent.municipality.insee),
self.parent.get_fantoir(), self.number,
self.ordinal)
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
if fantoir:
if not insee:
insee = fantoir[:5]
number = data.get('number')
ordinal = data.get('ordinal')
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if data.get('cia'):
data['cia'] = computed_cia
source = row.get('source')
data['attributes'] = {'source': source}
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = data.get('ign')
laposte = data.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
if instance and compute_cia:
if cia != computed_cia:
# Means new values are changing one of the four values of the
# cia (insee, fantoir, number, ordinal). Make sure we are not
# creating a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = data.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def compute_cia(self):
return compute_cia(self.parent.fantoir[:5], self.parent.fantoir[5:],
self.number,
self.ordinal) if self.parent.fantoir else None
def process_housenumber(row):
number = row.get('numero')
ordinal = row.get('ordinal') or None
fantoir = row.get('group:fantoir')
cia = row.get('cia')
if not fantoir and cia:
# 12xxx.json is missing group:fantoir.
fantoir = '{}{}'.format(cia[:5], cia[6:10])
insee = fantoir[:5]
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if not cia:
cia = computed_cia
parent = 'fantoir:{}'.format(fantoir)
source = row.get('source')
attributes = {'source': source}
data = dict(number=number, ordinal=ordinal, version=1, parent=parent,
attributes=attributes)
# Only override if key is present (even if value is null).
if 'ref:ign' in row:
data['ign'] = row['ref:ign']
if 'poste:cea' in row:
data['laposte'] = row['poste:cea']
if 'postcode' in row:
code = row.get('postcode')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code,
Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
instance = HouseNumber.first(HouseNumber.cia == cia)
update = False
if instance:
if cia != computed_cia:
# Means new values are changing one of the four values of the cia
# (insee, fantoir, number, ordinal). Make sure we are not creating
# a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, parent))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError:
reporter.warning('HouseNumber DB error', cia)
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, (number, ordinal, parent))
def compute_cia(self):
return compute_cia(str(self.parent.municipality.insee),
self.parent.get_fantoir(),
self.number, self.ordinal)
def process_housenumber(row):
data = dict(version=1)
keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
populate(keys, row, data)
fantoir = row.get('group:fantoir')
cia = row.get('cia')
insee = row.get('municipality:insee')
computed_cia = None
if fantoir:
if not insee:
insee = fantoir[:5]
number = data.get('number')
ordinal = data.get('ordinal')
computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
if data.get('cia'):
data['cia'] = computed_cia
source = row.get('source')
data['attributes'] = {'source': source}
# Only override if key is present (even if value is null).
if 'postcode:code' in row:
code = row.get('postcode:code')
postcode = PostCode.select().join(Municipality).where(
PostCode.code == code, Municipality.insee == insee).first()
if not postcode:
reporter.error('HouseNumber postcode not found', (cia, code))
else:
data['postcode'] = postcode
group_ign = row.get('group:ign')
group_laposte = row.get('group:laposte')
parent = None
if fantoir:
parent = 'fantoir:{}'.format(fantoir)
elif group_ign:
parent = 'ign:{}'.format(group_ign)
elif group_laposte:
parent = 'laposte:{}'.format(group_laposte)
if parent:
try:
parent = Group.coerce(parent)
except Group.DoesNotExist:
reporter.error('Parent given but not found', parent)
parent = None
else:
data['parent'] = parent
update = False
instance = None
ign = data.get('ign')
laposte = data.get('laposte')
if cia:
instance = HouseNumber.first(HouseNumber.cia == cia)
if instance and compute_cia:
if cia != computed_cia:
# Means new values are changing one of the four values of the
# cia (insee, fantoir, number, ordinal). Make sure we are not
# creating a duplicate.
duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
if duplicate:
msg = 'Duplicate CIA'
reporter.error(msg, (cia, computed_cia))
return
elif ign:
instance = HouseNumber.first(HouseNumber.ign == ign)
elif laposte:
instance = HouseNumber.first(HouseNumber.laposte == laposte)
if parent and not instance:
# Data is not coerced yet, we want None for empty strings.
ordinal = data.get('ordinal') or None
instance = HouseNumber.first(HouseNumber.parent == parent,
HouseNumber.number == data['number'],
HouseNumber.ordinal == ordinal)
if instance:
attributes = getattr(instance, 'attributes') or {}
if attributes.get('source') == source:
# Reimporting same data?
reporter.warning('HouseNumber already exists', instance.cia)
return
data['version'] = instance.version + 1
update = True
if not instance and not parent:
reporter.error('No matching instance and missing parent reference',
row)
return
validator = HouseNumber.validator(instance=instance, update=update, **data)
if validator.errors:
reporter.error('HouseNumber errors', (validator.errors, data))
return
with HouseNumber._meta.database.atomic():
try:
validator.save()
except peewee.IntegrityError as e:
reporter.warning('HouseNumber DB error', (data, str(e)))
else:
msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
reporter.notice(msg, data)
def compute_cia(self):
get_fantoir = getattr(self.parent, 'get_fantoir', None)
if not get_fantoir:
return None
return compute_cia(str(self.parent.municipality.insee),
get_fantoir(), self.number, self.ordinal)
