def process_housenumber(row, id, name, insee, group_id, fantoir): number = row.get('numero') if number == '99999': # Means it's a group address point, according to AITF weird specs. number = None ordinal = row.get('suffixe') or None lat = row.get('lat') lon = row.get('long') kind = row.get('position') cia = None instance = None data = dict(number=number, ordinal=ordinal) if id: instance = HouseNumber.where(HouseNumber.id == id).first() if not instance: return reporter.error('HouseNumber id not found', id) parent = instance.parent elif fantoir: parent = 'fantoir:{}'.format(fantoir) cia = compute_cia(insee, fantoir[5:], number, ordinal) elif group_id: parent = Group.where(Group.id == group_id).first() if not parent: return reporter.error('Group id not found', group_id) if parent.fantoir: cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number, ordinal) else: return reporter.error('Missing group id and fantoir', id) if cia: instance = HouseNumber.where(HouseNumber.cia == cia).first() if instance: # Well… the BAL can't give us a BAN reference version, be kind for now. # See https://github.com/etalab/ban/issues/91#issuecomment-198432574 # and https://github.com/etalab/ban/issues/94 data['version'] = instance.version + 1 data['instance'] = instance data['parent'] = parent validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) else: try: housenumber = validator.save() except peewee.IntegrityError: return reporter.error('Duplicate housenumber', (number, ordinal, parent)) if lon and lat: process_position(housenumber, (lon, lat), kind) msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def compute_cia(self): return compute_cia(str(self.parent.municipality.insee), self.parent.get_fantoir(), self.number, self.ordinal)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None if fantoir: if not insee: insee = fantoir[:5] number = data.get('number') ordinal = data.get('ordinal') computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if data.get('cia'): data['cia'] = computed_cia source = row.get('source') data['attributes'] = {'source': source} # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = data.get('ign') laposte = data.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) if instance and compute_cia: if cia != computed_cia: # Means new values are changing one of the four values of the # cia (insee, fantoir, number, ordinal). Make sure we are not # creating a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = data.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def compute_cia(self): return compute_cia(self.parent.fantoir[:5], self.parent.fantoir[5:], self.number, self.ordinal) if self.parent.fantoir else None
def process_housenumber(row): number = row.get('numero') ordinal = row.get('ordinal') or None fantoir = row.get('group:fantoir') cia = row.get('cia') if not fantoir and cia: # 12xxx.json is missing group:fantoir. fantoir = '{}{}'.format(cia[:5], cia[6:10]) insee = fantoir[:5] computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if not cia: cia = computed_cia parent = 'fantoir:{}'.format(fantoir) source = row.get('source') attributes = {'source': source} data = dict(number=number, ordinal=ordinal, version=1, parent=parent, attributes=attributes) # Only override if key is present (even if value is null). if 'ref:ign' in row: data['ign'] = row['ref:ign'] if 'poste:cea' in row: data['laposte'] = row['poste:cea'] if 'postcode' in row: code = row.get('postcode') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode instance = HouseNumber.first(HouseNumber.cia == cia) update = False if instance: if cia != computed_cia: # Means new values are changing one of the four values of the cia # (insee, fantoir, number, ordinal). Make sure we are not creating # a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('HouseNumber DB error', cia) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def compute_cia(self): get_fantoir = getattr(self.parent, 'get_fantoir', None) if not get_fantoir: return None return compute_cia(str(self.parent.municipality.insee), get_fantoir(), self.number, self.ordinal)