def process_housenumber(row, id, name, insee, group_id, fantoir): number = row.get('numero') if number == '99999': # Means it's a group address point, according to AITF weird specs. number = None ordinal = row.get('suffixe') or None lat = row.get('lat') lon = row.get('long') kind = row.get('position') cia = None instance = None data = dict(number=number, ordinal=ordinal) if id: instance = HouseNumber.where(HouseNumber.id == id).first() if not instance: return reporter.error('HouseNumber id not found', id) parent = instance.parent elif fantoir: parent = 'fantoir:{}'.format(fantoir) cia = compute_cia(insee, fantoir[5:], number, ordinal) elif group_id: parent = Group.where(Group.id == group_id).first() if not parent: return reporter.error('Group id not found', group_id) if parent.fantoir: cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number, ordinal) else: return reporter.error('Missing group id and fantoir', id) if cia: instance = HouseNumber.where(HouseNumber.cia == cia).first() if instance: # Well… the BAL can't give us a BAN reference version, be kind for now. # See https://github.com/etalab/ban/issues/91#issuecomment-198432574 # and https://github.com/etalab/ban/issues/94 data['version'] = instance.version + 1 data['instance'] = instance data['parent'] = parent validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) else: try: housenumber = validator.save() except peewee.IntegrityError: return reporter.error('Duplicate housenumber', (number, ordinal, parent)) if lon and lat: process_position(housenumber, (lon, lat), kind) msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def process_position(row): positioning = row.get('positioning') if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') data['attributes'] = row.get('attributes', {}) if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_position(row): kind = row.get('kind') if not hasattr(Position, kind.upper()): kind = Position.UNKNOWN positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia').upper() center = row.get('geometry') housenumber = HouseNumber.first(HouseNumber.cia == cia) if not housenumber: reporter.error('Position housenumber does not exist', cia) return instance = Position.first(Position.housenumber == housenumber, Position.kind == kind, Position.source == source) version = instance.version + 1 if instance else 1 data = dict(kind=kind, source=source, housenumber=housenumber, center=center, positioning=positioning, version=version) if 'ref:ign' in row: data['ign'] = row.get('ref:ign') validator = Position.validator(instance=instance, **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError: reporter.error('Integrity error', row) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcodes'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.pk) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcode'] = postcode validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, positioning=Position.OTHER, housenumber=housenumber.pk) if not validator.errors: validator.save() reporter.notice('Position', validator.instance) else: reporter.error('Position error', validator.errors) reporter.notice('Housenumber created', housenumber) else: reporter.error('Housenumber error', validator.errors)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('rep') parent = 'ign:{}'.format(row.get('identifiant_fpb')) postcode = 'code:{}'.format(row.get('code_post')) complement = row.get('designation_de_l_entree') ign = row.get('id') lat = row.get('lat') lon = row.get('lon') localisation_type = row.get('type_de_localisation') data = dict(number=number, ordinal=ordinal, ign=ign, parent=parent, postcode=postcode) laposte = row.get('cea') if laposte: data['laposte'] = laposte validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber error', validator.errors) else: try: housenumber = validator.save() except peewee.IntegrityError as e: reporter.error('SQL Error ', str(e)) else: reporter.notice('HouseNumber created', (number, ordinal)) if localisation_type != 'Au centre commune': process_position(housenumber, (lon, lat), localisation_type, {'complement': complement}) else: reporter.notice('Skipped centre commune', str(housenumber))
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.id, ign=ign) if postcode: data['ancestors'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.id) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def process_position(row): positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def add_cea(row): ign = row.get('ID_ADR') laposte = row.get('HEXACLE_1') if laposte == 'NR': return report('Missing CEA', ign, report.ERROR) query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign) done = query.execute() if not done: return report('IGN id not found', ign, report.ERROR) report('Done', ign, report.NOTICE)
def add_cea(row): ign = row.get('ID_ADR') laposte = row.get('HEXACLE_1') if laposte == 'NR': return report('Missing CEA', ign, report.ERROR) query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign) try: done = query.execute() except peewee.IntegrityError: report('Duplicate CEA', laposte, report.ERROR) else: if not done: return report('IGN id not found', ign, report.ERROR) report('Done', ign, report.NOTICE)
def add_housenumber(parent, id, metadata): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] data = dict(number=number, ordinal=ordinal, version=1) data[parent.__class__.__name__.lower()] = parent.id validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, housenumber=housenumber.id) if not validator.errors: validator.save() report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def process_housenumber(line): matricule = line[:8] number = line[8:12].strip() ordinal = line[13:23].strip() laposte = line[23:33] group_laposte = MATRICULE_TO_CEA.get(matricule) group = 'laposte:{}'.format(group_laposte) if not group_laposte: return report('Missing group CEA', matricule, report.ERROR) if not number: return report('Not a housenumber', laposte, report.NOTICE) validator = HouseNumber.validator(number=number, ordinal=ordinal, laposte=laposte, parent=group) if validator.errors: report('Housenumber error', validator.errors, report.ERROR) else: validator.save() report('Housenumber created', laposte, report.NOTICE)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None if fantoir: if not insee: insee = fantoir[:5] number = data.get('number') ordinal = data.get('ordinal') computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if data.get('cia'): data['cia'] = computed_cia source = row.get('source') data['attributes'] = {'source': source} # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = data.get('ign') laposte = data.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) if instance and compute_cia: if cia != computed_cia: # Means new values are changing one of the four values of the # cia (insee, fantoir, number, ordinal). Make sure we are not # creating a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = data.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
import peewee from ban.commands import command from ban.core.encoder import dumps from ban.core.models import (Group, HouseNumber, Municipality, Position, PostCode) from ban.db import database from . import helpers QUERIES = { 'PostCode': PostCode.select(), 'Municipality': Municipality.select(), 'Group': Group.select(), 'HouseNumber': HouseNumber.select(), 'Position': Position.select() } @command def resources(resource, path, **kwargs): """Export database as resources in json stream format. path path of file where to write resources resource Municipality, PostCode, Group, HouseNumber or Position """ resources = [ 'Municipality', 'PostCode', 'Group', 'HouseNumber', 'Position' ] if resource not in resources:
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None number = row.get('number') ordinal = row.get('ordinal') source = row.get('source') attributes = row.get('attributes', {}) if source: attributes['source'] = source data['attributes'] = attributes # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') complement = row.get('postcode:complement') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee, PostCode.complement == complement).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = row.get('ign') laposte = row.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = row.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', (instance.cia, instance.ign, instance.laposte)) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('ordinal') or None fantoir = row.get('group:fantoir') cia = row.get('cia') if not fantoir and cia: # 12xxx.json is missing group:fantoir. fantoir = '{}{}'.format(cia[:5], cia[6:10]) insee = fantoir[:5] computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if not cia: cia = computed_cia parent = 'fantoir:{}'.format(fantoir) source = row.get('source') attributes = {'source': source} data = dict(number=number, ordinal=ordinal, version=1, parent=parent, attributes=attributes) # Only override if key is present (even if value is null). if 'ref:ign' in row: data['ign'] = row['ref:ign'] if 'poste:cea' in row: data['laposte'] = row['poste:cea'] if 'postcode' in row: code = row.get('postcode') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode instance = HouseNumber.first(HouseNumber.cia == cia) update = False if instance: if cia != computed_cia: # Means new values are changing one of the four values of the cia # (insee, fantoir, number, ordinal). Make sure we are not creating # a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('HouseNumber DB error', cia) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))