def migrate(ctx): context = ctx.obj['context'] store = context.get('store') commands.prepare(context, store.internal) commands.migrate(context, store) commands.prepare(context, store) commands.migrate(context, store)
def context(mocker, config, postgresql, mongo): mocker.patch.dict(os.environ, { 'AUTHLIB_INSECURE_TRANSPORT': '1', }) Context = config.get('components', 'core', 'context', cast=importstr) Context = type('ContextForTests', (ContextForTests, Context), {}) context = Context() context.set('config', components.Config()) store = context.set('store', Store()) load_commands(config.get('commands', 'modules', cast=list)) load(context, context.get('config'), config) check(context, context.get('config')) load(context, store, config) check(context, store) prepare(context, store.internal) migrate(context, store) prepare(context, store) migrate(context, store) context.bind('auth.server', AuthorizationServer, context) context.bind('auth.resource_protector', ResourceProtector, context, BearerTokenValidator) yield context with context.enter(): # FIXME: quick and dirty workaround on `context.wipe` missing a connection, # when exception is thrown in spinta's logic. context.set('transaction', store.manifests['default'].backend.transaction(write=True)) context.set('auth.token', AdminToken()) # Remove all data after each test run. graph = collections.defaultdict(set) for model in store.manifests['default'].objects['model'].values(): if model.name not in graph: graph[model.name] = set() for prop in model.properties.values(): if prop.type.name == 'ref': graph[prop.type.object].add(model.name) for models in toposort(graph): for name in models: context.wipe(name) # Datasets does not have foreign kei constraints, so there is no need to # topologically sort them. At least for now. for dataset in store.manifests['default'].objects['dataset'].values(): for model in dataset.objects.values(): context.wipe(model) context.wipe(store.internal.objects['model']['transaction'])
def push(context: Context, model: Model, backend: PostgreSQL, data: dict, *, action: str): authorize(context, action, model, data=data) # load and check if data is a valid for it's model data = load(context, model, data) check(context, model, data) data = prepare(context, model, data) transaction = context.get('transaction') connection = transaction.connection table = backend.tables[model.manifest.name][model.name] data = { k: v for k, v in data.items() if k in table.main.columns } if action == INSERT_ACTION: result = connection.execute( table.main.insert().values(data), ) row_id = result.inserted_primary_key[0] elif action == UPDATE_ACTION: data['id'] = int(data['id']) result = connection.execute( table.main.update(). where(table.main.c.id == data['id']). values(data) ) if result.rowcount == 1: row_id = data['id'] elif result.rowcount == 0: raise Exception("Update failed, {self.obj} with {data['id']} not found.") else: raise Exception("Update failed, {self.obj} with {data['id']} has found and update {result.rowcount} rows.") elif action == DELETE_ACTION: raise NotImplementedError else: raise Exception(f"Unknown action {action!r}.") # Track changes. connection.execute( table.changes.insert().values( transaction_id=transaction.id, id=row_id, datetime=utcnow(), action=action, change={k: v for k, v in data.items() if k not in {'id'}}, ), ) return prepare(context, action, model, backend, {'id': str(row_id)})
def pull(ctx, source, model, push, export): context = ctx.obj['context'] store = context.get('store') commands.prepare(context, store.internal) commands.prepare(context, store) dataset = store.manifests['default'].objects['dataset'][source] with context.enter(): context.bind('transaction', store.backends['default'].transaction, write=push) result = commands.pull(context, dataset, models=model) result = commands.push(context, store, result) if push else result if export is None and push is False: export = 'stdout' if export: formats = { 'csv': 'csv', 'json': 'json', 'jsonl': 'jsonl', 'ascii': 'ascii', } path = None if export == 'stdout': fmt = 'ascii' elif export.startswith('stdout:'): fmt = export.split(':', 1)[1] else: path = pathlib.Path(export) fmt = export.suffix.strip('.') if fmt not in formats: raise click.UsageError(f"unknown export file type {fmt!r}") chunks = store.export(result, fmt) if path is None: for chunk in chunks: print(chunk, end='') else: with export.open('wb') as f: for chunk in chunks: f.write(chunk)
def prepare(context: Context, backend: PostgreSQL, manifest: Manifest): if manifest.name not in backend.tables: backend.tables[manifest.name] = {} # Prepare backend for models. for model in manifest.objects['model'].values(): if model.backend.name == backend.name: prepare(context, backend, model) # Prepare backend for datasets. for dataset in manifest.objects.get('dataset', {}).values(): for model in dataset.objects.values(): if model.backend.name == backend.name: prepare(context, backend, model)
def getall(context: Context, model: Model, backend: PostgreSQL, **kwargs): authorize(context, 'getall', model) connection = context.get('transaction').connection table = backend.tables[model.manifest.name][model.name].main result = connection.execute(sa.select([table])) for row in result: yield prepare(context, 'getall', model, backend, row)
def getall(context: Context, model: Model, backend: Mongo, **kwargs): authorize(context, 'getall', model) transaction = context.get('transaction') # Yield all available entries. model_collection = backend.db[model.get_type_value()] for row in model_collection.find({}): yield prepare(context, 'getall', model, backend, row)
def get(context: Context, model: Model, backend: Mongo, id: str): authorize(context, 'getone', model) transaction = context.get('transaction') model_collection = backend.db[model.get_type_value()] row = model_collection.find_one({"_id": ObjectId(id)}) return prepare(context, 'getone', model, backend, row)
def push(context: Context, model: Model, backend: Mongo, data: dict, *, action: str): authorize(context, action, model, data=data) # load and check if data is a valid for it's model data = load(context, model, data) check(context, model, data) data = prepare(context, model, data) # Push data to Mongo backend, this can be an insert, update or delete. If # `id` is not given, it is an insert if `id` is given, it is an update. # # Deletes are not yet implemented, but for deletes `data` must equal to # `{'id': 1, _delete: True}`. # # Also this must return inserted/updated/deleted id. # # Also this command must write information to changelog after change is # done. transaction = context.get('transaction') model_collection = backend.db[model.get_type_value()] # Make a copy of data, because `pymongo` changes the reference `data` # object on `insert_one()` call. # # We want to have our data intact from whatever specific mongo metadata # MongoDB may add to our object. raw_data = copy.deepcopy(data) # FIXME: before creating revision check if there's not collision clash revision_id = get_new_id('revision id') raw_data['revision'] = revision_id if 'id' in data: result = model_collection.update_one({'_id': ObjectId(raw_data['id'])}, {'$set': raw_data}) assert result.matched_count == 1 and result.modified_count == 1 data_id = data['id'] else: data_id = model_collection.insert_one(raw_data).inserted_id # parse `ObjectId` to string and add it to our object raw_data['id'] = str(data_id) return prepare(context, action, model, backend, raw_data)
def prepare(context: Context, type: Object, backend: Backend, value: dict) -> dict: # prepare objects and it's properties for datastore new_loaded_obj = {} for k, v in type.properties.items(): # only load value keys which are available in schema # FIXME: If k is not in value, then value should be NA. Do not skip a value if k in value: new_loaded_obj[k] = prepare(context, v.type, backend, value[k]) return new_loaded_obj
async def homepage(request: Request): global context context.set( 'auth.request', get_auth_request({ 'method': request.method, 'url': str(request.url), 'body': None, 'headers': request.headers, })) context.set('auth.token', get_auth_token(context)) config = context.get('config') UrlParams = config.components['urlparams']['component'] params = prepare(context, UrlParams(), Version(), request) return await create_http_response(context, params, request)
def prepare(context: Context, backend: PostgreSQL, model: Model): columns = [] for prop in model.properties.values(): column = prepare(context, backend, prop) if isinstance(column, list): columns.extend(column) elif column is not None: columns.append(column) # Create main table. main_table_name = get_table_name(backend, model.manifest.name, model.name, MAIN_TABLE) main_table = sa.Table( main_table_name, backend.schema, sa.Column('_transaction_id', sa.Integer, sa.ForeignKey('transaction.id')), *columns, ) # Create changes table. changes_table_name = get_table_name(backend, model.manifest.name, model.name, CHANGES_TABLE) changes_table = get_changes_table(backend, changes_table_name, sa.Integer) backend.tables[model.manifest.name][model.name] = ModelTables(main_table, changes_table)
def prepare(context: Context, project: Project): for model in project.objects.values(): prepare(context, model)
def prepare(context: Context, store: Store): for manifest in store.manifests.values(): prepare(context, manifest)
def prepare(context: Context, type: Array, backend: Backend, value: list) -> list: # prepare array and it's items for datastore return [prepare(context, type.items.type, backend, v) for v in value]
def prepare(context: Context, manifest: Manifest): store = context.get('store') for backend in store.backends.values(): prepare(context, backend, manifest)
def prepare(context: Context, backend: PostgreSQL, prop: Property): return prepare(context, backend, prop.type)
format='%(levelname)s: %(message)s', ) c = Config() c.read() load_commands(c.get('commands', 'modules', cast=list)) Context = c.get('components', 'core', 'context', cast=importstr) context = Context() config = context.set('config', components.Config()) store = context.set('store', Store()) load(context, config, c) check(context, config) load(context, store, c) check(context, store) wait(context, store, c) prepare(context, store.internal) prepare(context, store) context.set('auth.server', AuthorizationServer(context)) context.set('auth.resource_protector', ResourceProtector(context, BearerTokenValidator)) set_context(context) app.debug = config.debug
def get(context: Context, model: Model, backend: PostgreSQL, id: str): authorize(context, 'getone', model) connection = context.get('transaction').connection table = backend.tables[model.manifest.name][model.name].main result = backend.get(connection, table, table.c.id == int(id)) return prepare(context, 'getone', model, backend, result)
def prepare(context: Context, dataset: Dataset): for model in dataset.objects.values(): prepare(context, model)