def migrate(ctx):
context = ctx.obj['context']
store = context.get('store')
commands.prepare(context, store.internal)
commands.migrate(context, store)
commands.prepare(context, store)
commands.migrate(context, store)
def context(mocker, config, postgresql, mongo):
mocker.patch.dict(os.environ, {
'AUTHLIB_INSECURE_TRANSPORT': '1',
})
Context = config.get('components', 'core', 'context', cast=importstr)
Context = type('ContextForTests', (ContextForTests, Context), {})
context = Context()
context.set('config', components.Config())
store = context.set('store', Store())
load_commands(config.get('commands', 'modules', cast=list))
load(context, context.get('config'), config)
check(context, context.get('config'))
load(context, store, config)
check(context, store)
prepare(context, store.internal)
migrate(context, store)
prepare(context, store)
migrate(context, store)
context.bind('auth.server', AuthorizationServer, context)
context.bind('auth.resource_protector', ResourceProtector, context,
BearerTokenValidator)
yield context
with context.enter():
# FIXME: quick and dirty workaround on `context.wipe` missing a connection,
# when exception is thrown in spinta's logic.
context.set('transaction',
store.manifests['default'].backend.transaction(write=True))
context.set('auth.token', AdminToken())
# Remove all data after each test run.
graph = collections.defaultdict(set)
for model in store.manifests['default'].objects['model'].values():
if model.name not in graph:
graph[model.name] = set()
for prop in model.properties.values():
if prop.type.name == 'ref':
graph[prop.type.object].add(model.name)
for models in toposort(graph):
for name in models:
context.wipe(name)
# Datasets does not have foreign kei constraints, so there is no need to
# topologically sort them. At least for now.
for dataset in store.manifests['default'].objects['dataset'].values():
for model in dataset.objects.values():
context.wipe(model)
context.wipe(store.internal.objects['model']['transaction'])
def push(context: Context, model: Model, backend: PostgreSQL, data: dict, *, action: str):
authorize(context, action, model, data=data)
# load and check if data is a valid for it's model
data = load(context, model, data)
check(context, model, data)
data = prepare(context, model, data)
transaction = context.get('transaction')
connection = transaction.connection
table = backend.tables[model.manifest.name][model.name]
data = {
k: v for k, v in data.items() if k in table.main.columns
}
if action == INSERT_ACTION:
result = connection.execute(
table.main.insert().values(data),
)
row_id = result.inserted_primary_key[0]
elif action == UPDATE_ACTION:
data['id'] = int(data['id'])
result = connection.execute(
table.main.update().
where(table.main.c.id == data['id']).
values(data)
)
if result.rowcount == 1:
row_id = data['id']
elif result.rowcount == 0:
raise Exception("Update failed, {self.obj} with {data['id']} not found.")
else:
raise Exception("Update failed, {self.obj} with {data['id']} has found and update {result.rowcount} rows.")
elif action == DELETE_ACTION:
raise NotImplementedError
else:
raise Exception(f"Unknown action {action!r}.")
# Track changes.
connection.execute(
table.changes.insert().values(
transaction_id=transaction.id,
id=row_id,
datetime=utcnow(),
action=action,
change={k: v for k, v in data.items() if k not in {'id'}},
),
)
return prepare(context, action, model, backend, {'id': str(row_id)})
def pull(ctx, source, model, push, export):
context = ctx.obj['context']
store = context.get('store')
commands.prepare(context, store.internal)
commands.prepare(context, store)
dataset = store.manifests['default'].objects['dataset'][source]
with context.enter():
context.bind('transaction',
store.backends['default'].transaction,
write=push)
result = commands.pull(context, dataset, models=model)
result = commands.push(context, store, result) if push else result
if export is None and push is False:
export = 'stdout'
if export:
formats = {
'csv': 'csv',
'json': 'json',
'jsonl': 'jsonl',
'ascii': 'ascii',
}
path = None
if export == 'stdout':
fmt = 'ascii'
elif export.startswith('stdout:'):
fmt = export.split(':', 1)[1]
else:
path = pathlib.Path(export)
fmt = export.suffix.strip('.')
if fmt not in formats:
raise click.UsageError(f"unknown export file type {fmt!r}")
chunks = store.export(result, fmt)
if path is None:
for chunk in chunks:
print(chunk, end='')
else:
with export.open('wb') as f:
for chunk in chunks:
f.write(chunk)
def prepare(context: Context, backend: PostgreSQL, manifest: Manifest):
if manifest.name not in backend.tables:
backend.tables[manifest.name] = {}
# Prepare backend for models.
for model in manifest.objects['model'].values():
if model.backend.name == backend.name:
prepare(context, backend, model)
# Prepare backend for datasets.
for dataset in manifest.objects.get('dataset', {}).values():
for model in dataset.objects.values():
if model.backend.name == backend.name:
prepare(context, backend, model)
def getall(context: Context, model: Model, backend: PostgreSQL, **kwargs):
authorize(context, 'getall', model)
connection = context.get('transaction').connection
table = backend.tables[model.manifest.name][model.name].main
result = connection.execute(sa.select([table]))
for row in result:
yield prepare(context, 'getall', model, backend, row)
def getall(context: Context, model: Model, backend: Mongo, **kwargs):
authorize(context, 'getall', model)
transaction = context.get('transaction')
# Yield all available entries.
model_collection = backend.db[model.get_type_value()]
for row in model_collection.find({}):
yield prepare(context, 'getall', model, backend, row)
def get(context: Context, model: Model, backend: Mongo, id: str):
authorize(context, 'getone', model)
transaction = context.get('transaction')
model_collection = backend.db[model.get_type_value()]
row = model_collection.find_one({"_id": ObjectId(id)})
return prepare(context, 'getone', model, backend, row)
def push(context: Context, model: Model, backend: Mongo, data: dict, *,
action: str):
authorize(context, action, model, data=data)
# load and check if data is a valid for it's model
data = load(context, model, data)
check(context, model, data)
data = prepare(context, model, data)
# Push data to Mongo backend, this can be an insert, update or delete. If
# `id` is not given, it is an insert if `id` is given, it is an update.
#
# Deletes are not yet implemented, but for deletes `data` must equal to
# `{'id': 1, _delete: True}`.
#
# Also this must return inserted/updated/deleted id.
#
# Also this command must write information to changelog after change is
# done.
transaction = context.get('transaction')
model_collection = backend.db[model.get_type_value()]
# Make a copy of data, because `pymongo` changes the reference `data`
# object on `insert_one()` call.
#
# We want to have our data intact from whatever specific mongo metadata
# MongoDB may add to our object.
raw_data = copy.deepcopy(data)
# FIXME: before creating revision check if there's not collision clash
revision_id = get_new_id('revision id')
raw_data['revision'] = revision_id
if 'id' in data:
result = model_collection.update_one({'_id': ObjectId(raw_data['id'])},
{'$set': raw_data})
assert result.matched_count == 1 and result.modified_count == 1
data_id = data['id']
else:
data_id = model_collection.insert_one(raw_data).inserted_id
# parse `ObjectId` to string and add it to our object
raw_data['id'] = str(data_id)
return prepare(context, action, model, backend, raw_data)
def prepare(context: Context, type: Object, backend: Backend,
value: dict) -> dict:
# prepare objects and it's properties for datastore
new_loaded_obj = {}
for k, v in type.properties.items():
# only load value keys which are available in schema
# FIXME: If k is not in value, then value should be NA. Do not skip a value
if k in value:
new_loaded_obj[k] = prepare(context, v.type, backend, value[k])
return new_loaded_obj
async def homepage(request: Request):
global context
context.set(
'auth.request',
get_auth_request({
'method': request.method,
'url': str(request.url),
'body': None,
'headers': request.headers,
}))
context.set('auth.token', get_auth_token(context))
config = context.get('config')
UrlParams = config.components['urlparams']['component']
params = prepare(context, UrlParams(), Version(), request)
return await create_http_response(context, params, request)
def prepare(context: Context, backend: PostgreSQL, model: Model):
columns = []
for prop in model.properties.values():
column = prepare(context, backend, prop)
if isinstance(column, list):
columns.extend(column)
elif column is not None:
columns.append(column)
# Create main table.
main_table_name = get_table_name(backend, model.manifest.name, model.name, MAIN_TABLE)
main_table = sa.Table(
main_table_name, backend.schema,
sa.Column('_transaction_id', sa.Integer, sa.ForeignKey('transaction.id')),
*columns,
)
# Create changes table.
changes_table_name = get_table_name(backend, model.manifest.name, model.name, CHANGES_TABLE)
changes_table = get_changes_table(backend, changes_table_name, sa.Integer)
backend.tables[model.manifest.name][model.name] = ModelTables(main_table, changes_table)
def prepare(context: Context, project: Project):
for model in project.objects.values():
prepare(context, model)
def prepare(context: Context, store: Store):
for manifest in store.manifests.values():
prepare(context, manifest)
def prepare(context: Context, type: Array, backend: Backend,
value: list) -> list:
# prepare array and it's items for datastore
return [prepare(context, type.items.type, backend, v) for v in value]
def prepare(context: Context, manifest: Manifest):
store = context.get('store')
for backend in store.backends.values():
prepare(context, backend, manifest)
def prepare(context: Context, backend: PostgreSQL, prop: Property):
return prepare(context, backend, prop.type)
format='%(levelname)s: %(message)s',
)
c = Config()
c.read()
load_commands(c.get('commands', 'modules', cast=list))
Context = c.get('components', 'core', 'context', cast=importstr)
context = Context()
config = context.set('config', components.Config())
store = context.set('store', Store())
load(context, config, c)
check(context, config)
load(context, store, c)
check(context, store)
wait(context, store, c)
prepare(context, store.internal)
prepare(context, store)
context.set('auth.server', AuthorizationServer(context))
context.set('auth.resource_protector',
ResourceProtector(context, BearerTokenValidator))
set_context(context)
app.debug = config.debug
def get(context: Context, model: Model, backend: PostgreSQL, id: str):
authorize(context, 'getone', model)
connection = context.get('transaction').connection
table = backend.tables[model.manifest.name][model.name].main
result = backend.get(connection, table, table.c.id == int(id))
return prepare(context, 'getone', model, backend, result)
def prepare(context: Context, dataset: Dataset):
for model in dataset.objects.values():
prepare(context, model)
