def dummytoken(**kwargs): """Create a dummy token for dev.""" session = context.get('session') Token.delete().where(Token.access_token == 'token').execute() Token.create(session=session.id, access_token="token", expires_in=3600 * 24) report('Created token', 'token', report.NOTICE)
def create(fail_silently=False, **kwargs): """Create database tables. fail_silently Do not raise error if table already exists. """ for model in models: model.create_table(fail_silently=fail_silently) report('Created', model.__name__, report.NOTICE)
def add_cea(row): ign = row.get('ID_ADR') laposte = row.get('HEXACLE_1') if laposte == 'NR': return report('Missing CEA', ign, report.ERROR) query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign) done = query.execute() if not done: return report('IGN id not found', ign, report.ERROR) report('Done', ign, report.NOTICE)
def process_row(metadata): name = metadata.get('name') id = metadata.get('id') insee = metadata.get('citycode') code = metadata.get('postcode') fantoir = ''.join(id.split('_')[:2])[:9] kind = metadata['type'] klass = Street if kind == 'street' else Locality instance = klass.select().where(klass.fantoir == fantoir).first() if instance: return report('Existing {}'.format(klass.__name__), { name: name, fantoir: fantoir }, report.WARNING) try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return report('Municipality does not exist', insee, report.ERROR) validator = PostCode.validator(code=code, version=1, name=municipality.name, municipality=municipality) if validator.errors: report('Invalid postcode', code, report.ERROR) postcode = None else: with PostCode._meta.database.atomic(): try: postcode = validator.save() except peewee.IntegrityError: # Another thread created it? postcode = PostCode.get(PostCode.code == code) else: report('Created postcode', code, report.NOTICE) data = dict( name=name, fantoir=fantoir, municipality=municipality.id, version=1, ) validator = klass.validator(**data) if not validator.errors: item = validator.save() report(kind, item, report.NOTICE) housenumbers = metadata.get('housenumbers') if housenumbers: for id, metadata in housenumbers.items(): add_housenumber(item, id, metadata, postcode) else: report('Street error', validator.errors, report.ERROR)
def resources(path, **kwargs): """Export database as resources in json stream format. path path of file where to write resources """ resources = [models.PostCode, models.Municipality, models.Locality, models.Street, models.HouseNumber] with Path(path).open(mode='w', encoding='utf-8') as f: for resource in resources: for data in resource.select().as_resource_list(): f.write(dumps(data) + '\n') # Memory consumption when exporting all France housenumbers? report(resource.__name__, data, report.NOTICE)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.id, ign=ign) if postcode: data['ancestors'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.id) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def add_housenumber(parent, id, metadata, postcode): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] ign = metadata.get('id') data = dict(number=number, ordinal=ordinal, version=1, parent=parent.pk, ign=ign) if postcode: data['postcodes'] = [postcode] validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, kind=Position.ENTRANCE, housenumber=housenumber.pk) if not validator.errors: validator.save() report('Position', validator.instance, report.NOTICE) else: report('Position error', validator.errors, report.ERROR) report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def process_postcode(line): if line[50] != 'M': return report('Cedex postcode', line, report.WARNING) insee = line[6:11] code = line[89:94] name = line[90:] try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return report('Municipality Not Existing', insee, report.WARNING) postcode, created = PostCode.get_or_create(code=code, name=name, municipality=municipality, defaults={'version': 1}) if created: report('PostCode Added', postcode, report.NOTICE)
def resources(path, **kwargs): """Export database as resources in json stream format. path path of file where to write resources """ resources = [ models.PostCode, models.Municipality, models.Locality, models.Street, models.HouseNumber ] with Path(path).open(mode='w', encoding='utf-8') as f: for resource in resources: for data in resource.select().as_resource_list(): f.write(dumps(data) + '\n') # Memory consumption when exporting all France housenumbers? report(resource.__name__, data, report.NOTICE)
def add_housenumber(parent, id, metadata): number, *ordinal = id.split(' ') ordinal = ordinal[0] if ordinal else '' center = [metadata['lon'], metadata['lat']] data = dict(number=number, ordinal=ordinal, version=1) data[parent.__class__.__name__.lower()] = parent.id validator = HouseNumber.validator(**data) if not validator.errors: housenumber = validator.save() validator = Position.validator(center=center, version=1, housenumber=housenumber.id) if not validator.errors: validator.save() report('Housenumber', housenumber, report.NOTICE) else: report('Housenumber error', validator.errors, report.ERROR)
def add_municipality(data, update=False): insee = data.get('insee') name = data.get('nom_com') siren = data.get('siren_com') version = 1 try: instance = models.Municipality.get(models.Municipality.insee == insee) except models.Municipality.DoesNotExist: instance = None if instance and not update: return report('Existing', name, report.WARNING) data = dict(insee=insee, name=name, siren=siren, version=version) validator = models.Municipality.validator(instance=instance, **data) if not validator.errors: instance = validator.save() report('Processed', instance, report.NOTICE) else: report('Error', validator.errors, report.ERROR)
def createuser(username=None, email=None, is_staff=False, **kwargs): """Create a user. is_staff set user staff """ if not username: username = helpers.prompt('Username') if not email: email = helpers.prompt('Email') password = helpers.prompt('Password', confirmation=True, hidden=True) validator = User.validator(username=username, email=email) if not validator.errors: user = validator.save() user.set_password(password) if is_staff: user.is_staff = True user.save() report('Created', user, report.NOTICE) else: report('Errored', validator.errors, report.ERROR)
def truncate(force=False, names=[], **kwargs): """Truncate database tables. force Do not ask for confirm. names List of model names to truncate (in the given order). """ if not names: # We expect names, not classes. names = [m.__name__.lower() for m in models] msg = 'Truncate all tables?' else: msg = 'Truncate tables: {}'.format(', '.join(names)) if not force and not helpers.confirm(msg, default=False): helpers.abort('Aborted.') # Delete in reverse way not to break FK constraints. for model in models[::-1]: name = model.__name__.lower() if name not in names: continue model.delete().execute() report('Truncated', name, report.NOTICE)
def process_postcode(line): if line[50] != 'M': return report('Cedex postcode', line, report.WARNING) insee = line[6:11] code = line[89:94] try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return report('Municipality Not Existing', insee, report.WARNING) postcode, created = PostCode.get_or_create(code=code, version=1) if created: report('PostCode Added', postcode, report.NOTICE) try: postcode.municipalities.add(municipality) except peewee.IntegrityError: report('Association Already Exists', postcode, report.WARNING) else: report('Association Added', postcode, report.NOTICE)
def add_cea(row): ign = row.get('ID_ADR') laposte = row.get('HEXACLE_1') if laposte == 'NR': return report('Missing CEA', ign, report.ERROR) query = HouseNumber.update(laposte=laposte).where(HouseNumber.ign == ign) try: done = query.execute() except peewee.IntegrityError: report('Duplicate CEA', laposte, report.ERROR) else: if not done: return report('IGN id not found', ign, report.ERROR) report('Done', ign, report.NOTICE)
def process_postcode(line): if line[50] != 'M': return report('Cedex postcode', line, report.WARNING) municipality = 'insee:{}'.format(line[6:11]) code = line[89:94] name = line[90:] validator = PostCode.validator(code=code, name=name, municipality=municipality) if validator.errors: return report('PostCode Error', validator.errors, report.ERROR) else: with PostCode._meta.database.atomic(): try: validator.save() except peewee.IntegrityError: report('Already created', (code, municipality), report.WARNING) else: report('PostCode created', code, report.NOTICE)
def process_housenumber(line): matricule = line[:8] number = line[8:12].strip() ordinal = line[13:23].strip() laposte = line[23:33] group_laposte = MATRICULE_TO_CEA.get(matricule) group = 'laposte:{}'.format(group_laposte) if not group_laposte: return report('Missing group CEA', matricule, report.ERROR) if not number: return report('Not a housenumber', laposte, report.NOTICE) validator = HouseNumber.validator(number=number, ordinal=ordinal, laposte=laposte, parent=group) if validator.errors: report('Housenumber error', validator.errors, report.ERROR) else: validator.save() report('Housenumber created', laposte, report.NOTICE)
def process_group(line): if not line[0] == 'V': return report('Not a street', line, report.NOTICE) name = line[60:92] matricule = line[12:20] laposte = MATRICULE_TO_CEA.get(matricule) if not laposte: return report('Missing CEA', matricule, report.ERROR) municipality = 'insee:{}'.format(line[7:12]) kind = guess_kind(name, line[92:96].strip()) validator = Group.validator(name=name, laposte=laposte, municipality=municipality, kind=kind) if validator.errors: report('Error', validator.errors, report.ERROR) else: validator.save() report('Success', name, report.NOTICE)
def process_row(metadata): name = metadata.get('name') id = metadata.get('id') insee = metadata.get('citycode') fantoir = ''.join(id.split('_')[:2])[:9] kind = metadata['type'] klass = Street if kind == 'street' else Locality instance = klass.select().where(klass.fantoir == fantoir).first() if instance: return report('Existing {}'.format(klass.__name__), {name: name, fantoir: fantoir}, report.WARNING) try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return report('Municipality does not exist', insee, report.ERROR) data = dict( name=name, fantoir=fantoir, municipality=municipality.id, version=1, ) validator = klass.validator(**data) if not validator.errors: item = validator.save() report(kind, item, report.NOTICE) housenumbers = metadata.get('housenumbers') if housenumbers: for id, metadata in housenumbers.items(): add_housenumber(item, id, metadata) else: report('Street error', validator.errors, report.ERROR)
def process_row(metadata): name = metadata.get('name') id = metadata.get('id') insee = metadata.get('citycode') code = metadata.get('postcode') fantoir = ''.join(id.split('_')[:2])[:9] kind = 'area' if metadata['type'] == 'locality' else 'way' instance = Group.select().where(Group.fantoir == fantoir).first() if instance: return report('Existing {}'.format(Group.__name__), {name: name, fantoir: fantoir}, report.WARNING) try: municipality = Municipality.get(Municipality.insee == insee) except Municipality.DoesNotExist: return report('Municipality does not exist', insee, report.ERROR) validator = PostCode.validator(code=code, version=1, name=municipality.name, municipality=municipality) if validator.errors: report('Invalid postcode', code, report.ERROR) postcode = None else: with PostCode._meta.database.atomic(): try: postcode = validator.save() except peewee.IntegrityError: # Another thread created it? postcode = PostCode.get(PostCode.code == code) else: report('Created postcode', code, report.NOTICE) validator = Group.validator(name=name, fantoir=fantoir, kind=kind, municipality=municipality.pk, version=1) if not validator.errors: try: item = validator.save() except peewee.IntegrityError: return report('Duplicate group', fantoir, report.ERROR) report(kind, item, report.NOTICE) housenumbers = metadata.get('housenumbers') if housenumbers: for id, metadata in housenumbers.items(): add_housenumber(item, id, metadata, postcode) else: report('Street error', validator.errors, report.ERROR)
def dummytoken(**kwargs): """Create a dummy token for dev.""" session = context.get('session') Token.delete().where(Token.access_token == 'token').execute() Token.create(session=session.id, access_token="token", expires_in=3600*24) report('Created token', 'token', report.NOTICE)