def process_position(row): positioning = row.get('positioning') if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') data['attributes'] = row.get('attributes', {}) if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_position(row): kind = row.get('kind') if not hasattr(Position, kind.upper()): kind = Position.UNKNOWN positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia').upper() center = row.get('geometry') housenumber = HouseNumber.first(HouseNumber.cia == cia) if not housenumber: reporter.error('Position housenumber does not exist', cia) return instance = Position.first(Position.housenumber == housenumber, Position.kind == kind, Position.source == source) version = instance.version + 1 if instance else 1 data = dict(kind=kind, source=source, housenumber=housenumber, center=center, positioning=positioning, version=version) if 'ref:ign' in row: data['ign'] = row.get('ref:ign') validator = Position.validator(instance=instance, **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError: reporter.error('Integrity error', row) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_position(row): positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None number = row.get('number') ordinal = row.get('ordinal') source = row.get('source') attributes = row.get('attributes', {}) if source: attributes['source'] = source data['attributes'] = attributes # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') complement = row.get('postcode:complement') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee, PostCode.complement == complement).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = row.get('ign') laposte = row.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = row.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', (instance.cia, instance.ign, instance.laposte)) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None if fantoir: if not insee: insee = fantoir[:5] number = data.get('number') ordinal = data.get('ordinal') computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if data.get('cia'): data['cia'] = computed_cia source = row.get('source') data['attributes'] = {'source': source} # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = data.get('ign') laposte = data.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) if instance and compute_cia: if cia != computed_cia: # Means new values are changing one of the four values of the # cia (insee, fantoir, number, ordinal). Make sure we are not # creating a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = data.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('ordinal') or None fantoir = row.get('group:fantoir') cia = row.get('cia') if not fantoir and cia: # 12xxx.json is missing group:fantoir. fantoir = '{}{}'.format(cia[:5], cia[6:10]) insee = fantoir[:5] computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if not cia: cia = computed_cia parent = 'fantoir:{}'.format(fantoir) source = row.get('source') attributes = {'source': source} data = dict(number=number, ordinal=ordinal, version=1, parent=parent, attributes=attributes) # Only override if key is present (even if value is null). if 'ref:ign' in row: data['ign'] = row['ref:ign'] if 'poste:cea' in row: data['laposte'] = row['poste:cea'] if 'postcode' in row: code = row.get('postcode') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode instance = HouseNumber.first(HouseNumber.cia == cia) update = False if instance: if cia != computed_cia: # Means new values are changing one of the four values of the cia # (insee, fantoir, number, ordinal). Make sure we are not creating # a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('HouseNumber DB error', cia) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))