def process_group(row, id, name, insee, group_id, fantoir): municipality = 'insee:{}'.format(insee) data = dict(name=name, fantoir=fantoir, municipality=municipality) instance = None # Means creation. if id: try: instance = Group.get(Group.id == id) except Group.DoesNotExist: return reporter.error('Group id not found', id) elif data['fantoir']: try: instance = Group.get(Group.fantoir == data['fantoir']) except Group.DoesNotExist: pass # Let's create it. if instance: data['kind'] = instance.kind # Well… the BAL can't give us a BAN reference version, be kind for now. # See https://github.com/etalab/ban/issues/91#issuecomment-198432574 # and https://github.com/etalab/ban/issues/94 data['version'] = instance.version + 1 else: data['kind'] = Group.WAY validator = Group.validator(instance=instance, **data) if validator.errors: reporter.error('Invalid data', validator.errors) else: street = validator.save() msg = 'Created group' if not instance else 'Updated group' reporter.notice(msg, street.id) if row.get('lat') and row.get('long'): process_housenumber(row, id, name, insee, group_id, fantoir)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcodes'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, positioning=Position.OTHER, housenumber=housenumber.pk) if not validator.errors: validator.save() reporter.notice('Position', validator.instance) else: reporter.error('Position error', validator.errors) reporter.notice('Housenumber created', housenumber) else: reporter.error('Housenumber error', validator.errors)
def group_to_municipality(destination, source, areas, label): validator = models.Group.validator(name=source.name, municipality=destination, version=1, kind=models.Group.AREA, attributes={'insee': source.insee}) if validator.errors: reporter.error('Errors', validator) else: reporter.notice('Created', validator) gr_area = validator.save() areas.append(gr_area) for group in source.groups: # Check the Group is not one created by the validator above if group not in areas: move_group(destination, group) for housenumber in group.housenumber_set: validator = models.HouseNumber.validator( instance=housenumber, ancestors=[gr_area], update=True, version=housenumber.version + 1) if validator.errors: reporter.error('Errors', housenumber) else: validator.save() reporter.notice('Ancestor redirected', housenumber)
def invalidatetoken(client=None, user=None, **kwargs): """Invalidate a token. Provide at least a client or a user client client_id or client_secret of an existing client user username or email of an existing user """ if not client and not user: helpers.abort("""Provide at least a client (client_id or client_secret) or a user (username or email)""") if user: user_inst = User.first((User.username == user) | (User.email == user)) if not user_inst: return reporter.error('User not found', user) where_clause = (Session.user_id == user_inst.pk) else: client_inst = Client.first((Client.client_id == client) | (Client.client_secret == client)) if not client_inst: return reporter.error('Client not found', user) where_clause = (Session.client_id == client_inst.pk) tokens = Token.select().join(Session).where(where_clause).where( Token.expires > utcnow()) for token in tokens: token.expires = utcnow() token.save() reporter.notice('Invalidate {} tokens'.format(len(tokens)), tokens)
def move_group(destination, group): validator = models.Group.validator(instance=group, municipality=destination, update=True, version=group.version + 1) if validator.errors: reporter.error('Errors', validator) else: validator.save() reporter.notice('Municipality modified', group)
def add_cea(row): ign = row.get('ID_ADR') laposte = row.get('HEXACLE_1') if laposte == 'NR': return reporter.error('Missing CEA', ign) query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign) try: done = query.execute() except peewee.IntegrityError: reporter.error('Duplicate CEA', laposte) else: if not done: return reporter.error('IGN id not found', ign) reporter.notice('Done', ign)
def process_municipality(row): row['attributes'] = {'source': row.pop('source')} validator = Municipality.validator(**row) if validator.errors: return reporter.error('Municipality errors', validator.errors) validator.save() reporter.notice('Imported Municipality', row['insee'])
def process_position(housenumber, center, kind): kind = KIND_MAPPING.get(kind, kind) instance = Position.where(Position.housenumber == housenumber, Position.kind == kind).first() version = instance.version + 1 if instance else 1 validator = Position.validator(housenumber=housenumber, center=center, source='BAL', # Use siren from filename? positioning=Position.IMAGERY, kind=kind, instance=instance, version=version) if validator.errors: reporter.error('Position error', validator.errors) else: position = validator.save() msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_postcode(row): insee = row['municipality:insee'] municipality = 'insee:{}'.format(insee) source = row.get('source') attributes = {} if source: attributes = {'source': row.pop('source')} name = row.get('name') code = row.get('postcode') complement = row.get('complement') data = dict(name=name, code=code, municipality=municipality, version=1, attributes=attributes, complement=complement) instance = PostCode.select().join(Municipality).where( PostCode.complement == complement, PostCode.code == code, Municipality.insee == insee).first() if instance: return reporter.notice('PostCode already exists', code) validator = PostCode.validator(**data) if validator.errors: return reporter.error('PostCode errors', (validator.errors, code, insee)) validator.save() reporter.notice('Imported PostCode', code)
def process_housenumber(line): matricule = line[:8] number = line[8:12].strip() ordinal = line[13:23].strip() laposte = line[23:33] group_laposte = MATRICULE_TO_CEA.get(matricule) group = 'laposte:{}'.format(group_laposte) if not group_laposte: return reporter.error('Missing group CEA', matricule) if not number: return reporter.notice('Not a housenumber', laposte) validator = HouseNumber.validator(number=number, ordinal=ordinal, laposte=laposte, parent=group) if validator.errors: reporter.error('Housenumber error', validator.errors) else: validator.save() reporter.notice('Housenumber created', laposte)
def createuser(username=None, email=None, is_staff=False, **kwargs): """Create a user. is_staff set user staff """ if not username: username = helpers.prompt('Username') if not email: email = helpers.prompt('Email') validator = User.validator(username=username, email=email) if not validator.errors: user = validator.save() if is_staff: user.is_staff = True user.save() reporter.notice('Created', user) else: reporter.error('Errored', validator.errors)
def process_group(line): if not line[0] == 'V': return reporter.warning('Not a street', line) name = line[60:92] matricule = line[12:20] laposte = MATRICULE_TO_CEA.get(matricule) if not laposte: return reporter.error('Missing CEA', matricule) municipality = 'insee:{}'.format(line[7:12]) kind = guess_kind(name, line[92:96].strip()) validator = Group.validator(name=name, laposte=laposte, municipality=municipality, kind=kind) if validator.errors: reporter.error('Error', validator.errors) else: validator.save() reporter.notice('Success', name)
def add_municipality(data, update=False): insee = data.get('insee') name = data.get('nom_com') siren = data.get('siren_com') version = 1 try: instance = models.Municipality.get(models.Municipality.insee == insee) except models.Municipality.DoesNotExist: instance = None if instance and not update: return reporter.warning('Existing', name) data = dict(insee=insee, name=name, siren=siren, version=version) validator = models.Municipality.validator(instance=instance, **data) if not validator.errors: instance = validator.save() reporter.notice('Processed', instance) else: reporter.error('Error', validator.errors)
def createclient(name=None, user=None, **kwargs): """Create a client. name name of the client to create user username or email of an existing user """ if not name: name = helpers.prompt('Client name') if not user: user = helpers.prompt('User username or email') user_inst = User.first((User.username == user) | (User.email == user)) if not user_inst: return reporter.error('User not found', user) validator = Client.validator(name=name, user=user_inst) if validator.errors: return reporter.error('Errored', validator.errors) client = validator.save() reporter.notice('Created', client) listclients()
def process_group(row): fantoir = row.get('group:fantoir') data = dict(version=1, fantoir=fantoir) name = row.get('name') if name: data['name'] = name kind = row.get('group') if kind: data['kind'] = kind insee = row.get('municipality:insee') if insee: data['municipality'] = 'insee:{}'.format(insee) laposte = row.get('poste:matricule') if laposte: data['laposte'] = laposte source = row.get('source') attributes = row.get('attributes', {}) attributes['source'] = source data['attributes'] = attributes update = False instance = Group.first(Group.fantoir == fantoir) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('Group already exist', fantoir) return data['version'] = instance.version + 1 if attributes: attributes.update(data['attributes']) data['attributes'] = attributes update = True validator = Group.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('Invalid group data', (validator.errors, fantoir)) else: try: validator.save() except peewee.IntegrityError: reporter.error('Integrity Error', fantoir) else: msg = 'Group updated' if instance else 'Group created' reporter.notice(msg, fantoir)
def createuser(username=None, email=None, is_staff=False, **kwargs): """Create a user. is_staff set user staff """ if not username: username = helpers.prompt('Username') if not email: email = helpers.prompt('Email') password = helpers.prompt('Password', confirmation=True, hidden=True) validator = User.validator(username=username, email=email) if not validator.errors: user = validator.save() user.set_password(password) if is_staff: user.is_staff = True user.save() reporter.notice('Created', user) else: reporter.error('Errored', validator.errors)
def process_postcode(destination, source, label): for postcode in source.postcodes: if postcode.complement is None: validator = models.PostCode.validator(instance=postcode, municipality=destination, update=True, complement=postcode.name, name=label, version=postcode.version + 1) else: validator = models.PostCode.validator(instance=postcode, municipality=destination, update=True, name=label, version=postcode.version + 1) if validator.errors: reporter.error('Errors', validator) else: validator.save() reporter.notice('Label and municipality modified', postcode)
def process_position(row): kind = row.get('kind') if not hasattr(Position, kind.upper()): kind = Position.UNKNOWN positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia').upper() center = row.get('geometry') housenumber = HouseNumber.first(HouseNumber.cia == cia) if not housenumber: reporter.error('Position housenumber does not exist', cia) return instance = Position.first(Position.housenumber == housenumber, Position.kind == kind, Position.source == source) version = instance.version + 1 if instance else 1 data = dict(kind=kind, source=source, housenumber=housenumber, center=center, positioning=positioning, version=version) if 'ref:ign' in row: data['ign'] = row.get('ref:ign') validator = Position.validator(instance=instance, **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError: reporter.error('Integrity error', row) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def createclient(name=None, user=None, scopes=[], contributor_types=[], **kwargs): """Create a client. name name of the client to create user username or email of an existing user """ if not name: name = helpers.prompt('Client name') if not user: user = helpers.prompt('User username or email') user_inst = User.first((User.username == user) | (User.email == user)) if not user_inst: return reporter.error('User not found', user) if not scopes: scopes = helpers.prompt('Scopes (separated by spaces)', default='').split() if not contributor_types: contributor_types = helpers.prompt( 'Contributor types (separated by spaces)', default='viewer').split() for ct in contributor_types: if ct not in Client.CONTRIBUTOR_TYPE: return reporter.error( '{} not in {}'.format(ct, str(Client.CONTRIBUTOR_TYPE)), contributor_types) if contributor_types != ['viewer']: contributor_types.append('viewer') validator = Client.validator(name=name, user=user_inst, scopes=scopes, contributor_types=contributor_types) if validator.errors: return reporter.error('Errored', validator.errors) client = validator.save() reporter.notice('Created', client) listclients()
def process_row(row): kind = row.pop('type') if kind == "municipality": return process_municipality(row) elif kind == "group": return process_group(row) elif kind == "postcode": return process_postcode(row) elif kind == "housenumber": return process_housenumber(row) elif kind == "position": return process_position(row) else: return reporter.error('Missing "type" key', row)
def process_housenumber(row, id, name, insee, group_id, fantoir): number = row.get('numero') if number == '99999': # Means it's a group address point, according to AITF weird specs. number = None ordinal = row.get('suffixe') or None lat = row.get('lat') lon = row.get('long') kind = row.get('position') cia = None instance = None data = dict(number=number, ordinal=ordinal) if id: instance = HouseNumber.where(HouseNumber.id == id).first() if not instance: return reporter.error('HouseNumber id not found', id) parent = instance.parent elif fantoir: parent = 'fantoir:{}'.format(fantoir) cia = compute_cia(insee, fantoir[5:], number, ordinal) elif group_id: parent = Group.where(Group.id == group_id).first() if not parent: return reporter.error('Group id not found', group_id) if parent.fantoir: cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number, ordinal) else: return reporter.error('Missing group id and fantoir', id) if cia: instance = HouseNumber.where(HouseNumber.cia == cia).first() if instance: # Well… the BAL can't give us a BAN reference version, be kind for now. # See https://github.com/etalab/ban/issues/91#issuecomment-198432574 # and https://github.com/etalab/ban/issues/94 data['version'] = instance.version + 1 data['instance'] = instance data['parent'] = parent validator = HouseNumber.validator(**data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) else: try: housenumber = validator.save() except peewee.IntegrityError: return reporter.error('Duplicate housenumber', (number, ordinal, parent)) if lon and lat: process_position(housenumber, (lon, lat), kind) msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def process_postcode(line): if line[50] != 'M': return reporter.warning('Cedex postcode', line) municipality = 'insee:{}'.format(line[6:11]) code = line[89:94] name = line[90:] validator = PostCode.validator(code=code, name=name, municipality=municipality) if validator.errors: return reporter.error('PostCode Error', validator.errors) else: with PostCode._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('Already created', (code, municipality)) else: reporter.notice('PostCode created', code)
def process_postcode(row): insee = row['municipality:insee'] municipality = 'insee:{}'.format(insee) attributes = {'source': row.pop('source')} name = row.get('name') code = row.get('postcode') data = dict(name=name, code=code, municipality=municipality, version=1, attributes=attributes) instance = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if instance: return reporter.notice('PostCode already exists', code) validator = PostCode.validator(**data) if validator.errors: return reporter.error('PostCode errors', (validator.errors, code, insee)) validator.save() reporter.notice('Imported PostCode', code)
def process_group(row): data = dict(version=1) keys = ['name', ('group', 'kind'), 'laposte', 'ign', 'fantoir', 'alias'] populate(keys, row, data) insee = row.get('municipality:insee') if insee: data['municipality'] = 'insee:{}'.format(insee) source = row.get('source') attributes = row.get('attributes', {}) if source: attributes['source'] = source data['attributes'] = attributes if 'addressing' in row: if hasattr(Group, row['addressing'].upper()): data['addressing'] = row['addressing'] update = False ign = data.get('ign') fantoir = data.get('fantoir') laposte = data.get('laposte') if fantoir: instance = Group.first(Group.fantoir == fantoir) elif ign: instance = Group.first(Group.ign == ign) elif laposte: instance = Group.first(Group.laposte == laposte) else: reporter.error('Missing group unique id', row) return if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') and attributes.get('source') == source: # Reimporting same data? reporter.warning('Group already exist', fantoir) return data['version'] = instance.version + 1 if attributes: attributes.update(data['attributes']) data['attributes'] = attributes update = True validator = Group.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('Invalid group data', (validator.errors, row)) else: try: validator.save() except peewee.IntegrityError: reporter.error('Integrity Error', fantoir) else: msg = 'Group updated' if instance else 'Group created' reporter.notice(msg, fantoir)
def process_row(metadata): name = metadata.get('name') id = metadata.get('id') insee = metadata.get('citycode') code = metadata.get('postcode') fantoir = ''.join(id.split('_')[:2])[:9] kind = 'area' if metadata['type'] == 'locality' else 'way' instance = Group.select().where(Group.fantoir == fantoir).first() if instance: return reporter.warning('Existing group', {name: name, fantoir: fantoir}) try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return reporter.error('Municipality does not exist', insee) validator = PostCode.validator(code=code, version=1, name=municipality.name, municipality=municipality) if validator.errors: reporter.error('Invalid postcode', code) postcode = None else: with PostCode._meta.database.atomic(): try: postcode = validator.save() except peewee.IntegrityError: # Another thread created it? postcode = PostCode.get(PostCode.code == code) else: reporter.notice('Created postcode', code) validator = Group.validator(name=name, fantoir=fantoir, kind=kind, municipality=municipality.pk, version=1) if not validator.errors: try: item = validator.save() except peewee.IntegrityError: return reporter.error('Duplicate group', fantoir) reporter.notice(kind, item) housenumbers = metadata.get('housenumbers') if housenumbers: for id, metadata in housenumbers.items(): add_housenumber(item, id, metadata, postcode) else: reporter.error('Street error', validator.errors)
def process_group(row): data = dict(version=1) keys = ['name', ('group', 'kind'), 'laposte', 'ign', 'fantoir'] populate(keys, row, data) insee = row.get('municipality:insee') if insee: data['municipality'] = 'insee:{}'.format(insee) source = row.get('source') attributes = row.get('attributes', {}) attributes['source'] = source data['attributes'] = attributes if 'addressing' in row: if hasattr(Group, row['addressing'].upper()): data['addressing'] = row['addressing'] update = False ign = data.get('ign') fantoir = data.get('fantoir') laposte = data.get('laposte') if fantoir: instance = Group.first(Group.fantoir == fantoir) elif ign: instance = Group.first(Group.ign == ign) elif laposte: instance = Group.first(Group.laposte == laposte) else: reporter.error('Missing group unique id', row) return if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('Group already exist', fantoir) return data['version'] = instance.version + 1 if attributes: attributes.update(data['attributes']) data['attributes'] = attributes update = True validator = Group.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('Invalid group data', (validator.errors, row)) else: try: validator.save() except peewee.IntegrityError: reporter.error('Integrity Error', fantoir) else: msg = 'Group updated' if instance else 'Group created' reporter.notice(msg, fantoir)
def process_position(row): positioning = row.get('positioning') if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') data['attributes'] = row.get('attributes', {}) if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_position(row): positioning = row.get('positionning') # two "n" in the data. if not positioning or not hasattr(Position, positioning.upper()): positioning = Position.OTHER source = row.get('source') cia = row.get('housenumber:cia') housenumber_ign = row.get('housenumber:ign') housenumber = None if cia: cia = cia.upper() housenumber = HouseNumber.first(HouseNumber.cia == cia) elif housenumber_ign: housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign) if not housenumber: reporter.error('Unable to find parent housenumber', row) return instance = None if 'ign' in row: # The only situation where we want to avoid creating new position is # when we have the ign identifier. instance = Position.first(Position.ign == row['ign']) version = instance.version + 1 if instance else 1 data = dict(source=source, housenumber=housenumber, positioning=positioning, version=version) kind = row.get('kind', '') if hasattr(Position, kind.upper()): data['kind'] = kind elif not instance: # We are creating a new position (not updating), kind is mandatory. kind = Position.UNKNOWN if kind: data['kind'] = kind populate(['ign', 'name', ('geometry', 'center')], row, data) validator = Position.validator(instance=instance, update=bool(instance), **data) if validator.errors: reporter.error('Position error', validator.errors) else: try: position = validator.save() except peewee.IntegrityError as e: reporter.error('Integrity error', (str(e), data)) else: msg = 'Position updated' if instance else 'Position created' reporter.notice(msg, position.id)
def process_housenumber(row): number = row.get('numero') ordinal = row.get('ordinal') or None fantoir = row.get('group:fantoir') cia = row.get('cia') if not fantoir and cia: # 12xxx.json is missing group:fantoir. fantoir = '{}{}'.format(cia[:5], cia[6:10]) insee = fantoir[:5] computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if not cia: cia = computed_cia parent = 'fantoir:{}'.format(fantoir) source = row.get('source') attributes = {'source': source} data = dict(number=number, ordinal=ordinal, version=1, parent=parent, attributes=attributes) # Only override if key is present (even if value is null). if 'ref:ign' in row: data['ign'] = row['ref:ign'] if 'poste:cea' in row: data['laposte'] = row['poste:cea'] if 'postcode' in row: code = row.get('postcode') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode instance = HouseNumber.first(HouseNumber.cia == cia) update = False if instance: if cia != computed_cia: # Means new values are changing one of the four values of the cia # (insee, fantoir, number, ordinal). Make sure we are not creating # a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, parent)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: reporter.warning('HouseNumber DB error', cia) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, (number, ordinal, parent))
def merge(destination, sources=[], name='', label='', **kwargs): """ Municipality merge command. Steps: - for each Municipality to be removed: - redirect the Municipality by insee and id - create a new Group with its name - attach all HouseNumbers of this Municipality to this new Group - attach sources Groups and sources PostCodes to destination - for each PostCode of the Municipality: - change the name and the complement (if the complement is null) - remove the Municipality """ if destination in sources: helpers.abort('Destination in sources') if not name or not name.split(): helpers.abort('Name should not be empty') if not label or not label.split(): helpers.abort('Label should not be empty') try: destination = models.Municipality.get( models.Municipality.insee == destination) except models.Municipality.DoesNotExist: helpers.abort('Destination does not exist') if not sources: helpers.abort('No sources') sources_inst = [] # Make sure all sources exist before processing any of them. for source in sources: print(source) try: source = models.Municipality.get( models.Municipality.insee == source) except models.Municipality.DoesNotExist: helpers.abort('Source {} does not exist'.format(source)) else: sources_inst.append(source) source_done = [] areas = [] db = models.Municipality._meta.database with db.atomic(): process_postcode(destination, destination, label) group_to_municipality(destination, destination, areas, label) for source in sources_inst: # Make sure that the source has not be treated yet if source.insee not in source_done: source_done.append(source.insee) process_source(destination, source, areas, label) validator = models.Municipality.validator(instance=destination, name=name, version=destination.version + 1, update=True) if validator.errors: reporter.error('Errors', validator) else: validator.save() reporter.notice('Mame modified', destination) print(reporter) if helpers.confirm('Are you confident with those changes ?') is False: db.rollback() reporter.clear('Action cancelled')
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None if fantoir: if not insee: insee = fantoir[:5] number = data.get('number') ordinal = data.get('ordinal') computed_cia = compute_cia(insee, fantoir[5:], number, ordinal) if data.get('cia'): data['cia'] = computed_cia source = row.get('source') data['attributes'] = {'source': source} # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = data.get('ign') laposte = data.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) if instance and compute_cia: if cia != computed_cia: # Means new values are changing one of the four values of the # cia (insee, fantoir, number, ordinal). Make sure we are not # creating a duplicate. duplicate = HouseNumber.first(HouseNumber.cia == computed_cia) if duplicate: msg = 'Duplicate CIA' reporter.error(msg, (cia, computed_cia)) return elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = data.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', instance.cia) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)
def process_housenumber(row): data = dict(version=1) keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia'] populate(keys, row, data) fantoir = row.get('group:fantoir') cia = row.get('cia') insee = row.get('municipality:insee') computed_cia = None number = row.get('number') ordinal = row.get('ordinal') source = row.get('source') attributes = row.get('attributes', {}) if source: attributes['source'] = source data['attributes'] = attributes # Only override if key is present (even if value is null). if 'postcode:code' in row: code = row.get('postcode:code') complement = row.get('postcode:complement') postcode = PostCode.select().join(Municipality).where( PostCode.code == code, Municipality.insee == insee, PostCode.complement == complement).first() if not postcode: reporter.error('HouseNumber postcode not found', (cia, code)) else: data['postcode'] = postcode group_ign = row.get('group:ign') group_laposte = row.get('group:laposte') parent = None if fantoir: parent = 'fantoir:{}'.format(fantoir) elif group_ign: parent = 'ign:{}'.format(group_ign) elif group_laposte: parent = 'laposte:{}'.format(group_laposte) if parent: try: parent = Group.coerce(parent) except Group.DoesNotExist: reporter.error('Parent given but not found', parent) parent = None else: data['parent'] = parent update = False instance = None ign = row.get('ign') laposte = row.get('laposte') if cia: instance = HouseNumber.first(HouseNumber.cia == cia) elif ign: instance = HouseNumber.first(HouseNumber.ign == ign) elif laposte: instance = HouseNumber.first(HouseNumber.laposte == laposte) if parent and not instance: # Data is not coerced yet, we want None for empty strings. ordinal = row.get('ordinal') or None instance = HouseNumber.first(HouseNumber.parent == parent, HouseNumber.number == data['number'], HouseNumber.ordinal == ordinal) if instance: attributes = getattr(instance, 'attributes') or {} if attributes.get('source') == source: # Reimporting same data? reporter.warning('HouseNumber already exists', (instance.cia, instance.ign, instance.laposte)) return data['version'] = instance.version + 1 update = True if not instance and not parent: reporter.error('No matching instance and missing parent reference', row) return validator = HouseNumber.validator(instance=instance, update=update, **data) if validator.errors: reporter.error('HouseNumber errors', (validator.errors, data)) return with HouseNumber._meta.database.atomic(): try: validator.save() except peewee.IntegrityError as e: reporter.warning('HouseNumber DB error', (data, str(e))) else: msg = 'HouseNumber Updated' if instance else 'HouseNumber created' reporter.notice(msg, data)