Ejemplo n.º 1
0
def dummytoken(token, **kwargs):
    """Create a dummy token for dev."""
    session = context.get('session')
    Token.delete().where(Token.access_token == token).execute()
    Token.create(session=session.pk, access_token=token, expires_in=3600*24,
                 token_type='Bearer', scope='*')
    reporter.notice('Created token', token)
Ejemplo n.º 2
0
Archivo: init.py Proyecto: odorie/ban
def process_position(row):
    kind = row.get('kind')
    if not hasattr(Position, kind.upper()):
        kind = Position.UNKNOWN
    positioning = row.get('positionning')  # two "n" in the data.
    if not positioning or not hasattr(Position, positioning.upper()):
        positioning = Position.OTHER
    source = row.get('source')
    cia = row.get('housenumber:cia').upper()
    center = row.get('geometry')
    housenumber = HouseNumber.first(HouseNumber.cia == cia)
    if not housenumber:
        reporter.error('Position housenumber does not exist', cia)
        return
    instance = Position.first(Position.housenumber == housenumber,
                              Position.kind == kind, Position.source == source)
    version = instance.version + 1 if instance else 1
    data = dict(kind=kind, source=source, housenumber=housenumber,
                center=center, positioning=positioning, version=version)
    if 'ref:ign' in row:
        data['ign'] = row.get('ref:ign')
    validator = Position.validator(instance=instance, **data)
    if validator.errors:
        reporter.error('Position error', validator.errors)
    else:
        try:
            position = validator.save()
        except peewee.IntegrityError:
            reporter.error('Integrity error', row)
        else:
            msg = 'Position updated' if instance else 'Position created'
            reporter.notice(msg, position.id)
Ejemplo n.º 3
0
def process_group(row, id, name, insee, group_id, fantoir):
    municipality = 'insee:{}'.format(insee)
    data = dict(name=name, fantoir=fantoir, municipality=municipality)
    instance = None  # Means creation.
    if id:
        try:
            instance = Group.get(Group.id == id)
        except Group.DoesNotExist:
            return reporter.error('Group id not found', id)
    elif data['fantoir']:
        try:
            instance = Group.get(Group.fantoir == data['fantoir'])
        except Group.DoesNotExist:
            pass  # Let's create it.
    if instance:
        data['kind'] = instance.kind
        # Well… the BAL can't give us a BAN reference version, be kind for now.
        # See https://github.com/etalab/ban/issues/91#issuecomment-198432574
        # and https://github.com/etalab/ban/issues/94
        data['version'] = instance.version + 1
    else:
        data['kind'] = Group.WAY
    validator = Group.validator(instance=instance, **data)
    if validator.errors:
        reporter.error('Invalid data', validator.errors)
    else:
        street = validator.save()
        msg = 'Created group' if not instance else 'Updated group'
        reporter.notice(msg, street.id)
        if row.get('lat') and row.get('long'):
            process_housenumber(row, id, name, insee, group_id, fantoir)
Ejemplo n.º 4
0
def process_municipality(row):
    row['attributes'] = {'source': row.pop('source')}
    validator = Municipality.validator(**row)
    if validator.errors:
        return reporter.error('Municipality errors', validator.errors)
    validator.save()
    reporter.notice('Imported Municipality', row['insee'])
Ejemplo n.º 5
0
def add_housenumber(parent, id, metadata, postcode):
    number, *ordinal = id.split(' ')
    ordinal = ordinal[0] if ordinal else ''
    center = [metadata['lon'], metadata['lat']]
    ign = metadata.get('id')
    data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk,
                ign=ign)
    if postcode:
        data['postcodes'] = [postcode]

    validator = HouseNumber.validator(**data)

    if not validator.errors:
        housenumber = validator.save()
        validator = Position.validator(center=center, version=1,
                                       kind=Position.ENTRANCE,
                                       positioning=Position.OTHER,
                                       housenumber=housenumber.pk)
        if not validator.errors:
            validator.save()
            reporter.notice('Position', validator.instance)
        else:
            reporter.error('Position error', validator.errors)
        reporter.notice('Housenumber created', housenumber)
    else:
        reporter.error('Housenumber error', validator.errors)
Ejemplo n.º 6
0
Archivo: db.py Proyecto: odorie/ban
def create(fail_silently=False, **kwargs):
    """Create database tables.

    fail_silently   Do not raise error if table already exists.
    """
    for model in models:
        model.create_table(fail_silently=fail_silently)
        reporter.notice('Created', model.__name__)
Ejemplo n.º 7
0
def resources(path, **kwargs):
    """Export database as resources in json stream format.

    path    path of file where to write resources
    """
    resources = [models.PostCode, models.Municipality, models.Group,
                 models.HouseNumber]
    with Path(path).open(mode='w', encoding='utf-8') as f:
        for resource in resources:
            for data in resource.select().serialize({'*': {}}):
                f.write(dumps(data) + '\n')
                reporter.notice(resource.__name__, data)
Ejemplo n.º 8
0
def process_housenumber(row, id, name, insee, group_id, fantoir):
    number = row.get('numero')
    if number == '99999':
        # Means it's a group address point, according to AITF weird specs.
        number = None
    ordinal = row.get('suffixe') or None
    lat = row.get('lat')
    lon = row.get('long')
    kind = row.get('position')
    cia = None
    instance = None
    data = dict(number=number, ordinal=ordinal)
    if id:
        instance = HouseNumber.where(HouseNumber.id == id).first()
        if not instance:
            return reporter.error('HouseNumber id not found', id)
        parent = instance.parent
    elif fantoir:
        parent = 'fantoir:{}'.format(fantoir)
        cia = compute_cia(insee, fantoir[5:], number, ordinal)
    elif group_id:
        parent = Group.where(Group.id == group_id).first()
        if not parent:
            return reporter.error('Group id not found', group_id)
        if parent.fantoir:
            cia = compute_cia(parent.fantoir[:5], parent.fantoir[5:], number,
                              ordinal)
    else:
        return reporter.error('Missing group id and fantoir', id)
    if cia:
        instance = HouseNumber.where(HouseNumber.cia == cia).first()
    if instance:
        # Well… the BAL can't give us a BAN reference version, be kind for now.
        # See https://github.com/etalab/ban/issues/91#issuecomment-198432574
        # and https://github.com/etalab/ban/issues/94
        data['version'] = instance.version + 1
        data['instance'] = instance
    data['parent'] = parent

    validator = HouseNumber.validator(**data)
    if validator.errors:
        reporter.error('HouseNumber errors', (validator.errors, parent))
    else:
        try:
            housenumber = validator.save()
        except peewee.IntegrityError:
            return reporter.error('Duplicate housenumber',
                                  (number, ordinal, parent))
        if lon and lat:
            process_position(housenumber, (lon, lat), kind)
        msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
        reporter.notice(msg, (number, ordinal, parent))
Ejemplo n.º 9
0
def add_cea(row):
    ign = row.get('ID_ADR')
    laposte = row.get('HEXACLE_1')
    if laposte == 'NR':
        return reporter.error('Missing CEA', ign)
    query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign)
    try:
        done = query.execute()
    except peewee.IntegrityError:
        reporter.error('Duplicate CEA', laposte)
    else:
        if not done:
            return reporter.error('IGN id not found', ign)
        reporter.notice('Done', ign)
Ejemplo n.º 10
0
def process_row(metadata):
    name = metadata.get('name')
    id = metadata.get('id')
    insee = metadata.get('citycode')
    code = metadata.get('postcode')
    fantoir = ''.join(id.split('_')[:2])[:9]

    kind = 'area' if metadata['type'] == 'locality' else 'way'
    instance = Group.select().where(Group.fantoir == fantoir).first()
    if instance:
        return reporter.warning('Existing group', {name: name,
                                                   fantoir: fantoir})

    try:
        municipality = Municipality.get(Municipality.insee == insee)
    except Municipality.DoesNotExist:
        return reporter.error('Municipality does not exist', insee)

    validator = PostCode.validator(code=code, version=1,
                                   name=municipality.name,
                                   municipality=municipality)
    if validator.errors:
        reporter.error('Invalid postcode', code)
        postcode = None
    else:
        with PostCode._meta.database.atomic():
            try:
                postcode = validator.save()
            except peewee.IntegrityError:
                # Another thread created it?
                postcode = PostCode.get(PostCode.code == code)
            else:
                reporter.notice('Created postcode', code)

    validator = Group.validator(name=name, fantoir=fantoir, kind=kind,
                                municipality=municipality.pk, version=1)

    if not validator.errors:
        try:
            item = validator.save()
        except peewee.IntegrityError:
            return reporter.error('Duplicate group', fantoir)
        reporter.notice(kind, item)
        housenumbers = metadata.get('housenumbers')
        if housenumbers:
            for id, metadata in housenumbers.items():
                add_housenumber(item, id, metadata, postcode)
    else:
        reporter.error('Street error', validator.errors)
Ejemplo n.º 11
0
def process_group(row):
    data = dict(version=1)
    keys = ['name', ('group', 'kind'), 'laposte', 'ign', 'fantoir']
    populate(keys, row, data)
    insee = row.get('municipality:insee')
    if insee:
        data['municipality'] = 'insee:{}'.format(insee)
    source = row.get('source')
    attributes = row.get('attributes', {})
    attributes['source'] = source
    data['attributes'] = attributes
    if 'addressing' in row:
        if hasattr(Group, row['addressing'].upper()):
            data['addressing'] = row['addressing']
    update = False
    ign = data.get('ign')
    fantoir = data.get('fantoir')
    laposte = data.get('laposte')
    if fantoir:
        instance = Group.first(Group.fantoir == fantoir)
    elif ign:
        instance = Group.first(Group.ign == ign)
    elif laposte:
        instance = Group.first(Group.laposte == laposte)
    else:
        reporter.error('Missing group unique id', row)
        return
    if instance:
        attributes = getattr(instance, 'attributes') or {}
        if attributes.get('source') == source:
            # Reimporting same data?
            reporter.warning('Group already exist', fantoir)
            return
        data['version'] = instance.version + 1
        if attributes:
            attributes.update(data['attributes'])
            data['attributes'] = attributes
        update = True
    validator = Group.validator(instance=instance, update=update, **data)
    if validator.errors:
        reporter.error('Invalid group data', (validator.errors, row))
    else:
        try:
            validator.save()
        except peewee.IntegrityError:
            reporter.error('Integrity Error', fantoir)
        else:
            msg = 'Group updated' if instance else 'Group created'
            reporter.notice(msg, fantoir)
Ejemplo n.º 12
0
def process_position(housenumber, center, kind):
    kind = KIND_MAPPING.get(kind, kind)
    instance = Position.where(Position.housenumber == housenumber,
                              Position.kind == kind).first()
    version = instance.version + 1 if instance else 1
    validator = Position.validator(housenumber=housenumber, center=center,
                                   source='BAL',  # Use siren from filename?
                                   positioning=Position.IMAGERY,
                                   kind=kind, instance=instance,
                                   version=version)
    if validator.errors:
        reporter.error('Position error', validator.errors)
    else:
        position = validator.save()
        msg = 'Position updated' if instance else 'Position created'
        reporter.notice(msg, position.id)
Ejemplo n.º 13
0
def process_position(row):
    positioning = row.get('positionning')  # two "n" in the data.
    if not positioning or not hasattr(Position, positioning.upper()):
        positioning = Position.OTHER
    source = row.get('source')
    cia = row.get('housenumber:cia')
    housenumber_ign = row.get('housenumber:ign')
    housenumber = None
    if cia:
        cia = cia.upper()
        housenumber = HouseNumber.first(HouseNumber.cia == cia)
    elif housenumber_ign:
        housenumber = HouseNumber.first(HouseNumber.ign == housenumber_ign)
    if not housenumber:
        reporter.error('Unable to find parent housenumber', row)
        return
    instance = None
    if 'ign' in row:
        # The only situation where we want to avoid creating new position is
        # when we have the ign identifier.
        instance = Position.first(Position.ign == row['ign'])
    version = instance.version + 1 if instance else 1
    data = dict(source=source, housenumber=housenumber,
                positioning=positioning, version=version)
    kind = row.get('kind', '')
    if hasattr(Position, kind.upper()):
        data['kind'] = kind
    elif not instance:
        # We are creating a new position (not updating), kind is mandatory.
        kind = Position.UNKNOWN
    if kind:
        data['kind'] = kind
    populate(['ign', 'name', ('geometry', 'center')], row, data)
    validator = Position.validator(instance=instance, update=bool(instance),
                                   **data)
    if validator.errors:
        reporter.error('Position error', validator.errors)
    else:
        try:
            position = validator.save()
        except peewee.IntegrityError as e:
            reporter.error('Integrity error', (str(e), data))
        else:
            msg = 'Position updated' if instance else 'Position created'
            reporter.notice(msg, position.id)
Ejemplo n.º 14
0
def process_group(line):
    if not line[0] == 'V':
        return reporter.warning('Not a street', line)
    name = line[60:92]
    matricule = line[12:20]
    laposte = MATRICULE_TO_CEA.get(matricule)
    if not laposte:
        return reporter.error('Missing CEA', matricule)
    municipality = 'insee:{}'.format(line[7:12])
    kind = guess_kind(name, line[92:96].strip())

    validator = Group.validator(name=name, laposte=laposte,
                                municipality=municipality, kind=kind)
    if validator.errors:
        reporter.error('Error', validator.errors)
    else:
        validator.save()
        reporter.notice('Success', name)
Ejemplo n.º 15
0
def process_postcode(line):
    if line[50] != 'M':
        return reporter.warning('Cedex postcode', line)
    municipality = 'insee:{}'.format(line[6:11])
    code = line[89:94]
    name = line[90:]
    validator = PostCode.validator(code=code, name=name,
                                   municipality=municipality)
    if validator.errors:
        return reporter.error('PostCode Error', validator.errors)
    else:
        with PostCode._meta.database.atomic():
            try:
                validator.save()
            except peewee.IntegrityError:
                reporter.warning('Already created', (code, municipality))
            else:
                reporter.notice('PostCode created', code)
Ejemplo n.º 16
0
def process_housenumber(line):
    matricule = line[:8]
    number = line[8:12].strip()
    ordinal = line[13:23].strip()
    laposte = line[23:33]
    group_laposte = MATRICULE_TO_CEA.get(matricule)
    group = 'laposte:{}'.format(group_laposte)
    if not group_laposte:
        return reporter.error('Missing group CEA', matricule)
    if not number:
        return reporter.notice('Not a housenumber', laposte)
    validator = HouseNumber.validator(number=number, ordinal=ordinal,
                                      laposte=laposte, parent=group)
    if validator.errors:
        reporter.error('Housenumber error', validator.errors)
    else:
        validator.save()
        reporter.notice('Housenumber created', laposte)
Ejemplo n.º 17
0
def process_postcode(row):
    insee = row['municipality:insee']
    municipality = 'insee:{}'.format(insee)
    attributes = {'source': row.pop('source')}
    name = row.get('name')
    code = row.get('postcode')
    data = dict(name=name, code=code, municipality=municipality,
                version=1, attributes=attributes)
    instance = PostCode.select().join(Municipality).where(
        PostCode.code == code, Municipality.insee == insee).first()
    if instance:
        return reporter.notice('PostCode already exists', code)
    validator = PostCode.validator(**data)
    if validator.errors:
        return reporter.error('PostCode errors', (validator.errors,
                                                  code, insee))
    validator.save()
    reporter.notice('Imported PostCode', code)
Ejemplo n.º 18
0
def createclient(name=None, user=None, **kwargs):
    """Create a client.

    name    name of the client to create
    user    username or email of an existing user
    """
    if not name:
        name = helpers.prompt('Client name')
    if not user:
        user = helpers.prompt('User username or email')
    user_inst = User.first((User.username == user) | (User.email == user))
    if not user_inst:
        return reporter.error('User not found', user)
    validator = Client.validator(name=name, user=user_inst)
    if validator.errors:
        return reporter.error('Errored', validator.errors)
    client = validator.save()
    reporter.notice('Created', client)
    listclients()
Ejemplo n.º 19
0
def add_municipality(data, update=False):
    insee = data.get('insee')
    name = data.get('nom_com')
    siren = data.get('siren_com')
    version = 1
    try:
        instance = models.Municipality.get(models.Municipality.insee == insee)
    except models.Municipality.DoesNotExist:
        instance = None
    if instance and not update:
        return reporter.warning('Existing', name)

    data = dict(insee=insee, name=name, siren=siren, version=version)
    validator = models.Municipality.validator(instance=instance, **data)
    if not validator.errors:
        instance = validator.save()
        reporter.notice('Processed', instance)
    else:
        reporter.error('Error', validator.errors)
Ejemplo n.º 20
0
Archivo: init.py Proyecto: odorie/ban
def process_group(row):
    fantoir = row.get('group:fantoir')
    data = dict(version=1, fantoir=fantoir)
    name = row.get('name')
    if name:
        data['name'] = name
    kind = row.get('group')
    if kind:
        data['kind'] = kind
    insee = row.get('municipality:insee')
    if insee:
        data['municipality'] = 'insee:{}'.format(insee)
    laposte = row.get('poste:matricule')
    if laposte:
        data['laposte'] = laposte
    source = row.get('source')
    attributes = row.get('attributes', {})
    attributes['source'] = source
    data['attributes'] = attributes
    update = False
    instance = Group.first(Group.fantoir == fantoir)
    if instance:
        attributes = getattr(instance, 'attributes') or {}
        if attributes.get('source') == source:
            # Reimporting same data?
            reporter.warning('Group already exist', fantoir)
            return
        data['version'] = instance.version + 1
        if attributes:
            attributes.update(data['attributes'])
            data['attributes'] = attributes
        update = True
    validator = Group.validator(instance=instance, update=update, **data)
    if validator.errors:
        reporter.error('Invalid group data', (validator.errors, fantoir))
    else:
        try:
            validator.save()
        except peewee.IntegrityError:
            reporter.error('Integrity Error', fantoir)
        else:
            msg = 'Group updated' if instance else 'Group created'
            reporter.notice(msg, fantoir)
Ejemplo n.º 21
0
def createuser(username=None, email=None, is_staff=False, **kwargs):
    """Create a user.

    is_staff    set user staff
    """
    if not username:
        username = helpers.prompt('Username')
    if not email:
        email = helpers.prompt('Email')
    password = helpers.prompt('Password', confirmation=True, hidden=True)
    validator = User.validator(username=username, email=email)
    if not validator.errors:
        user = validator.save()
        user.set_password(password)
        if is_staff:
            user.is_staff = True
            user.save()
        reporter.notice('Created', user)
    else:
        reporter.error('Errored', validator.errors)
Ejemplo n.º 22
0
Archivo: db.py Proyecto: odorie/ban
def truncate(*names, force=False, **kwargs):
    """Truncate database tables.

    force   Do not ask for confirm.
    names   List of model names to truncate (in the given order).
    """
    if not names:
        # We expect names, not classes.
        names = [m.__name__.lower() for m in models]
        msg = 'Truncate all tables?'
    else:
        msg = 'Truncate tables: {}'.format(', '.join(names))
    if not force and not helpers.confirm(msg, default=False):
        helpers.abort('Aborted.')
    # Delete in reverse way not to break FK constraints.
    for model in models[::-1]:
        name = model.__name__.lower()
        if name not in names:
            continue
        model.delete().execute()
        reporter.notice('Truncated', name)
Ejemplo n.º 23
0
Archivo: init.py Proyecto: odorie/ban
def process_housenumber(row):
    number = row.get('numero')
    ordinal = row.get('ordinal') or None
    fantoir = row.get('group:fantoir')
    cia = row.get('cia')
    if not fantoir and cia:
        # 12xxx.json is missing group:fantoir.
        fantoir = '{}{}'.format(cia[:5], cia[6:10])
    insee = fantoir[:5]
    computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
    if not cia:
        cia = computed_cia
    parent = 'fantoir:{}'.format(fantoir)
    source = row.get('source')
    attributes = {'source': source}
    data = dict(number=number, ordinal=ordinal, version=1, parent=parent,
                attributes=attributes)
    # Only override if key is present (even if value is null).
    if 'ref:ign' in row:
        data['ign'] = row['ref:ign']
    if 'poste:cea' in row:
        data['laposte'] = row['poste:cea']
    if 'postcode' in row:
        code = row.get('postcode')
        postcode = PostCode.select().join(Municipality).where(
            PostCode.code == code,
            Municipality.insee == insee).first()
        if not postcode:
            reporter.error('HouseNumber postcode not found', (cia, code))
        else:
            data['postcode'] = postcode
    instance = HouseNumber.first(HouseNumber.cia == cia)
    update = False
    if instance:
        if cia != computed_cia:
            # Means new values are changing one of the four values of the cia
            # (insee, fantoir, number, ordinal). Make sure we are not creating
            # a duplicate.
            duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
            if duplicate:
                msg = 'Duplicate CIA'
                reporter.error(msg, (cia, computed_cia))
                return
        attributes = getattr(instance, 'attributes') or {}
        if attributes.get('source') == source:
            # Reimporting same data?
            reporter.warning('HouseNumber already exists', instance.cia)
            return
        data['version'] = instance.version + 1
        update = True

    validator = HouseNumber.validator(instance=instance, update=update, **data)
    if validator.errors:
        reporter.error('HouseNumber errors', (validator.errors, parent))
        return
    with HouseNumber._meta.database.atomic():
        try:
            validator.save()
        except peewee.IntegrityError:
            reporter.warning('HouseNumber DB error', cia)
        else:
            msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
            reporter.notice(msg, (number, ordinal, parent))
Ejemplo n.º 24
0
Archivo: init.py Proyecto: pjegouic/ban
def process_housenumber(row):
    data = dict(version=1)
    keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
    populate(keys, row, data)
    fantoir = row.get('group:fantoir')
    cia = row.get('cia')
    insee = row.get('municipality:insee')
    computed_cia = None
    if fantoir:
        if not insee:
            insee = fantoir[:5]
        number = data.get('number')
        ordinal = data.get('ordinal')
        computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
        if data.get('cia'):
            data['cia'] = computed_cia
    source = row.get('source')
    data['attributes'] = {'source': source}
    # Only override if key is present (even if value is null).
    if 'postcode:code' in row:
        code = row.get('postcode:code')
        postcode = PostCode.select().join(Municipality).where(
            PostCode.code == code, Municipality.insee == insee).first()
        if not postcode:
            reporter.error('HouseNumber postcode not found', (cia, code))
        else:
            data['postcode'] = postcode

    group_ign = row.get('group:ign')
    group_laposte = row.get('group:laposte')
    parent = None
    if fantoir:
        parent = 'fantoir:{}'.format(fantoir)
    elif group_ign:
        parent = 'ign:{}'.format(group_ign)
    elif group_laposte:
        parent = 'laposte:{}'.format(group_laposte)
    if parent:
        try:
            parent = Group.coerce(parent)
        except Group.DoesNotExist:
            reporter.error('Parent given but not found', parent)
            parent = None
        else:
            data['parent'] = parent

    update = False
    instance = None
    ign = data.get('ign')
    laposte = data.get('laposte')
    if cia:
        instance = HouseNumber.first(HouseNumber.cia == cia)
        if instance and compute_cia:
            if cia != computed_cia:
                # Means new values are changing one of the four values of the
                # cia (insee, fantoir, number, ordinal). Make sure we are not
                # creating a duplicate.
                duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
                if duplicate:
                    msg = 'Duplicate CIA'
                    reporter.error(msg, (cia, computed_cia))
                    return
    elif ign:
        instance = HouseNumber.first(HouseNumber.ign == ign)
    elif laposte:
        instance = HouseNumber.first(HouseNumber.laposte == laposte)
    if parent and not instance:
        # Data is not coerced yet, we want None for empty strings.
        ordinal = data.get('ordinal') or None
        instance = HouseNumber.first(HouseNumber.parent == parent,
                                     HouseNumber.number == data['number'],
                                     HouseNumber.ordinal == ordinal)
    if instance:
        attributes = getattr(instance, 'attributes') or {}
        if attributes.get('source') == source:
            # Reimporting same data?
            reporter.warning('HouseNumber already exists', instance.cia)
            return
        data['version'] = instance.version + 1
        update = True

    if not instance and not parent:
        reporter.error('No matching instance and missing parent reference',
                       row)
        return

    validator = HouseNumber.validator(instance=instance, update=update, **data)
    if validator.errors:
        reporter.error('HouseNumber errors', (validator.errors, data))
        return
    with HouseNumber._meta.database.atomic():
        try:
            validator.save()
        except peewee.IntegrityError as e:
            reporter.warning('HouseNumber DB error', (data, str(e)))
        else:
            msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
            reporter.notice(msg, data)
Ejemplo n.º 25
0
def dummytoken(**kwargs):
    """Create a dummy token for dev."""
    session = context.get('session')
    Token.delete().where(Token.access_token == 'token').execute()
    Token.create(session=session.pk, access_token="token", expires_in=3600*24)
    reporter.notice('Created token', 'token')
Ejemplo n.º 26
0
def process_housenumber(row):
    data = dict(version=1)
    keys = [('numero', 'number'), 'ordinal', 'ign', 'laposte', 'cia']
    populate(keys, row, data)
    fantoir = row.get('group:fantoir')
    cia = row.get('cia')
    insee = row.get('municipality:insee')
    computed_cia = None
    if fantoir:
        if not insee:
            insee = fantoir[:5]
        number = data.get('number')
        ordinal = data.get('ordinal')
        computed_cia = compute_cia(insee, fantoir[5:], number, ordinal)
        if data.get('cia'):
            data['cia'] = computed_cia
    source = row.get('source')
    data['attributes'] = {'source': source}
    # Only override if key is present (even if value is null).
    if 'postcode:code' in row:
        code = row.get('postcode:code')
        postcode = PostCode.select().join(Municipality).where(
            PostCode.code == code,
            Municipality.insee == insee).first()
        if not postcode:
            reporter.error('HouseNumber postcode not found', (cia, code))
        else:
            data['postcode'] = postcode

    group_ign = row.get('group:ign')
    group_laposte = row.get('group:laposte')
    parent = None
    if fantoir:
        parent = 'fantoir:{}'.format(fantoir)
    elif group_ign:
        parent = 'ign:{}'.format(group_ign)
    elif group_laposte:
        parent = 'laposte:{}'.format(group_laposte)
    if parent:
        try:
            parent = Group.coerce(parent)
        except Group.DoesNotExist:
            reporter.error('Parent given but not found', parent)
            parent = None
        else:
            data['parent'] = parent

    update = False
    instance = None
    ign = data.get('ign')
    laposte = data.get('laposte')
    if cia:
        instance = HouseNumber.first(HouseNumber.cia == cia)
        if instance and compute_cia:
            if cia != computed_cia:
                # Means new values are changing one of the four values of the
                # cia (insee, fantoir, number, ordinal). Make sure we are not
                # creating a duplicate.
                duplicate = HouseNumber.first(HouseNumber.cia == computed_cia)
                if duplicate:
                    msg = 'Duplicate CIA'
                    reporter.error(msg, (cia, computed_cia))
                    return
    elif ign:
        instance = HouseNumber.first(HouseNumber.ign == ign)
    elif laposte:
        instance = HouseNumber.first(HouseNumber.laposte == laposte)
    if parent and not instance:
        # Data is not coerced yet, we want None for empty strings.
        ordinal = data.get('ordinal') or None
        instance = HouseNumber.first(HouseNumber.parent == parent,
                                     HouseNumber.number == data['number'],
                                     HouseNumber.ordinal == ordinal)
    if instance:
        attributes = getattr(instance, 'attributes') or {}
        if attributes.get('source') == source:
            # Reimporting same data?
            reporter.warning('HouseNumber already exists', instance.cia)
            return
        data['version'] = instance.version + 1
        update = True

    if not instance and not parent:
        reporter.error('No matching instance and missing parent reference',
                       row)
        return

    validator = HouseNumber.validator(instance=instance, update=update, **data)
    if validator.errors:
        reporter.error('HouseNumber errors', (validator.errors, data))
        return
    with HouseNumber._meta.database.atomic():
        try:
            validator.save()
        except peewee.IntegrityError as e:
            reporter.warning('HouseNumber DB error', (data, str(e)))
        else:
            msg = 'HouseNumber Updated' if instance else 'HouseNumber created'
            reporter.notice(msg, data)