def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.id, ign=ign) if postcode: data['ancestors'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.id) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcodes'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.pk) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('rep') parent = 'ign:{}'.format(row.get('identifiant_fpb')) postcode = 'code:{}'.format(row.get('code_post')) complement = row.get('designation_de_l_entree') ign = row.get('id') lat = row.get('lat') lon = row.get('lon') localisation_type = row.get('type_de_localisation') data = dict(number=number, ordinal=ordinal, ign=ign, parent=parent, postcode=postcode) laposte = row.get('cea') if laposte: data['laposte'] = laposte validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber error', validator.errors) else: try: housenumber = validator.save() except peewee.IntegrityError as e: reporter.error('SQL Error ', str(e)) else: reporter.notice('HouseNumber created', (number, ordinal)) if localisation_type != 'Au centre commune': process_position(housenumber, (lon, lat), localisation_type, {'complement': complement}) else: reporter.notice('Skipped centre commune', str(housenumber))
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcode'] = postcode validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, positioning=Position.OTHER, housenumber=housenumber.pk) if not validator.errors: validator.save() reporter.notice('Position', validator.instance) else: reporter.error('Position error', validator.errors) reporter.notice('Housenumber created', housenumber) else: reporter.error('Housenumber error', validator.errors)
def process_housenumber(row, id, name, insee, group_id, fantoir): number = row.get('numero') if number == '99999': # Means it's a group address point, according to AITF weird specs. number = None ordinal = row.get('suffixe') or None lat = row.get('lat') lon = row.get('long') kind = row.get('position') cia = None instance = None data = dict(number=number, ordinal=ordinal) if id: instance = HouseNumber.where(HouseNumber.id == id).first() if not instance: return reporter.error('HouseNumber id not found', id) parent = instance.parent elif fantoir: parent = 'fantoir:{}'.format(fantoir) cia = compute_cia(insee, fantoir[5:], number, ordinal) elif group_id: parent = Group.where(Group.id == group_id).first() if not parent: return reporter.error('Group id not found', group_id) if parent.fantoir: cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number, ordinal) else: return reporter.error('Missing group id and fantoir', id) if cia: instance = HouseNumber.where(HouseNumber.cia == cia).first() if instance: # Well… the BAL can't give us a BAN reference version, be kind for now. # See https://github.com/etalab/ban/issues/91#issuecomment-198432574 # and https://github.com/etalab/ban/issues/94 data['version'] = instance.version + 1 data['instance'] = instance data['parent'] = parent validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) else: try: housenumber = validator.save() except peewee.IntegrityError: return reporter.error('Duplicate housenumber', (number, ordinal, parent)) if lon and lat: process_position(housenumber, (lon, lat), kind) msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def add_housenumber(parent, id, metadata): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] data = dict(number=number, ordinal=ordinal, version=1) data[parent.__class__.__name__.lower()] = parent.id validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, housenumber=housenumber.id) if not validator.errors: validator.save() report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def process_housenumber(line): matricule = line[:8] number = line[8:12].strip() ordinal = line[13:23].strip() laposte = line[23:33] group_laposte = MATRICULE_TO_CEA.get(matricule) group = 'laposte:{}'.format(group_laposte) if not group_laposte: return report('Missing group CEA', matricule, report.ERROR) if not number: return report('Not a housenumber', laposte, report.NOTICE) validator = HouseNumber.validator(number=number, ordinal=ordinal, laposte=laposte, parent=group) if validator.errors: report('Housenumber error', validator.errors, report.ERROR) else: validator.save() report('Housenumber created', laposte, report.NOTICE)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None number = row.get('number') ordinal = row.get('ordinal') source = row.get('source') attributes = row.get('attributes', {}) if source: attributes['source'] = source data['attributes'] = attributes # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') complement = row.get('postcode:complement') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee, PostCode.complement == complement).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = row.get('ign') laposte = row.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = row.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', (instance.cia, instance.ign, instance.laposte)) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None if fantoir: if not insee: insee = fantoir[:5] number = data.get('number') ordinal = data.get('ordinal') computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if data.get('cia'): data['cia'] = computed_cia source = row.get('source') data['attributes'] = {'source': source} # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = data.get('ign') laposte = data.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) if instance and compute_cia: if cia != computed_cia: # Means new values are changing one of the four values of the # cia (insee, fantoir, number, ordinal). Make sure we are not # creating a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = data.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('ordinal') or None fantoir = row.get('group:fantoir') cia = row.get('cia') if not fantoir and cia: # 12xxx.json is missing group:fantoir. fantoir = '{}{}'.format(cia[:5], cia[6:10]) insee = fantoir[:5] computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if not cia: cia = computed_cia parent = 'fantoir:{}'.format(fantoir) source = row.get('source') attributes = {'source': source} data = dict(number=number, ordinal=ordinal, version=1, parent=parent, attributes=attributes) # Only override if key is present (even if value is null). if 'ref:ign' in row: data['ign'] = row['ref:ign'] if 'poste:cea' in row: data['laposte'] = row['poste:cea'] if 'postcode' in row: code = row.get('postcode') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode instance = HouseNumber.first(HouseNumber.cia == cia) update = False if instance: if cia != computed_cia: # Means new values are changing one of the four values of the cia # (insee, fantoir, number, ordinal). Make sure we are not creating # a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('HouseNumber DB error', cia) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))