async def retry_logic(context, request):
"""
First time sets 'title' and produce a conflict when the title is edited.
Then this request should be retried automatically and finish successfuly
"""
ob = await context.async_get("bar")
if ob.title is None:
# Modify field 'title' and commit change
ob.title = "A beautiful title"
tm = get_tm()
txn = get_transaction()
txn.register(ob)
await tm.commit()
# Simulate a conflict error to test retry logic
await tm.begin()
ob.title = "edit title"
ob.__serial__ = 3242432 # should raise conflict error when tm.commit()
txn.register(ob)
elif ob.title == "A beautiful title":
ob.title = "retry logic works"
txn = get_transaction()
txn.register(ob)
else:
raise Exception("Something is not working as expected")
async def get_registry(self, refresh=False):
if refresh and self.object_settings is not None:
txn = get_transaction()
await txn.refresh(self.object_settings)
if self.object_settings is None:
txn = get_transaction()
is_active = txn.status in (Status.ACTIVE, Status.COMMITTING)
if is_active:
self.object_settings = await self._get_registry_or_create()
else:
async with transaction():
self.object_settings = await self._get_registry_or_create()
return self.object_settings
def build_query(
self,
container: IContainer,
query: ParsedQueryInfo,
select_fields: typing.List[str],
distinct: typing.Optional[bool] = False,
) -> typing.Tuple[str, typing.List[typing.Any]]:
if query["sort_on"] is None:
# always need a sort otherwise paging never works
order_by_index = get_pg_index("uuid")
else:
order_by_index = get_pg_index(query["sort_on"]) or BasicJsonIndex(
query["sort_on"])
sql_arguments = []
sql_wheres = []
arg_index = 1
for idx, select in enumerate(query["selects"]):
select_fields.append(select.format(arg=arg_index))
sql_arguments.append(query["selects_arguments"][idx])
arg_index += 1
where_arg_index = 0
for where in query["wheres"]:
if isinstance(where, tuple):
operator, sub_wheres = where
sub_result = []
for sub_where in sub_wheres:
sub_result.append(
sub_where.format(arg=arg_index + where_arg_index))
sql_arguments.append(
query["wheres_arguments"][where_arg_index])
where_arg_index += 1
sql_wheres.append("(" + operator.join(sub_result) + ")")
else:
sql_wheres.append(where.format(arg=arg_index +
where_arg_index))
sql_arguments.append(
query["wheres_arguments"][where_arg_index])
where_arg_index += 1
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
sql_wheres.extend(self.get_default_where_clauses(container))
sql = """select {} {}
from {}
where {}
{}
limit {} offset {}""".format(
"distinct" if distinct else "",
",".join(select_fields),
sqlq(txn.storage.objects_table_name),
" AND ".join(sql_wheres),
"" if distinct else order_by_index.order_by(query["sort_dir"]),
sqlq(query["size"]),
sqlq(query["_from"]),
)
return sql, sql_arguments
async def aggregation(self, container: IContainer, query: ParsedQueryInfo):
select_fields = [
"json->'" + sqlq(field) + "' as " + sqlq(field)
for field in query["metadata"] or []
] # noqa
sql, arguments = self.build_query(container, query, select_fields,
True)
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
conn = await txn.get_connection()
results = []
logger.debug(f"Running search:\n{sql}\n{arguments}")
async with txn.lock:
records = await conn.fetch(sql, *arguments)
for record in records:
results.append([
json.loads(record[field]) for field in query["metadata"] or []
])
total = len(results)
if total >= query["size"] or query["_from"] != 0:
sql, arguments = self.build_count_query(container, query)
logger.debug(f"Running search:\n{sql}\n{arguments}")
async with txn.lock:
records = await conn.fetch(sql, *arguments)
total = records[0]["count"]
return {"items": results, "items_total": total}
async def read_runner(container, strategy):
request = get_current_request()
txn = get_transaction(request)
tm = get_tm(request)
id_ = uuid.uuid4().hex
await tm.abort(txn=txn)
txn = await tm.begin(request=request)
ob = await create_content_in_container(container, 'Item', id_)
await tm.commit(txn=txn)
tm._storage._transaction_strategy = strategy
print(f'Test content read with {strategy} strategy')
start = time.time()
for _ in range(ITERATIONS):
txn = await tm.begin(request=request)
assert await txn.get(ob._p_oid) is not None
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds')
print(f'Test large content read with {strategy} strategy')
start = time.time()
txn = await tm.begin(request=request)
for _ in range(ITERATIONS):
assert await txn.get(ob._p_oid) is not None
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def search(self, container: IContainer,
query: ParsedQueryInfo): # type: ignore
sql, arguments = self.build_query(container, query,
['id', 'zoid', 'json'])
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
conn = await txn.get_connection()
results = []
try:
context_url = get_object_url(container)
except RequestNotFound:
context_url = get_content_path(container)
logger.debug(f'Running search:\n{sql}\n{arguments}')
for record in await conn.fetch(sql, *arguments):
data = json.loads(record['json'])
result = self.load_meatdata(query, data)
result['@name'] = record['id']
result['@uid'] = record['zoid']
result['@id'] = data['@absolute_url'] = context_url + data['path']
results.append(result)
# also do count...
total = len(results)
if total >= query['size'] or query['_from'] != 0:
sql, arguments = self.build_count_query(container, query)
logger.debug(f'Running search:\n{sql}\n{arguments}')
records = await conn.fetch(sql, *arguments)
total = records[0]['count']
return {'member': results, 'items_count': total}
async def aggregation(self, container: IContainer, query: ParsedQueryInfo):
select_fields = [
'json->\'' + sqlq(field) + '\' as ' + sqlq(field)
for field in query['metadata'] or []
] # noqa
sql, arguments = self.build_query(container, query, select_fields,
True)
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
conn = await txn.get_connection()
results = []
logger.debug(f'Running search:\n{sql}\n{arguments}')
for record in await conn.fetch(sql, *arguments):
results.append([
json.loads(record[field]) for field in query['metadata'] or []
])
total = len(results)
if total >= query['size'] or query['_from'] != 0:
sql, arguments = self.build_count_query(container, query)
logger.debug(f'Running search:\n{sql}\n{arguments}')
records = await conn.fetch(sql, *arguments)
total = records[0]['count']
return {'member': results, 'items_count': total}
async def read_runner(container, strategy):
txn = get_transaction()
tm = get_tm()
id_ = uuid.uuid4().hex
await tm.abort(txn=txn)
txn = await tm.begin()
ob = await create_content_in_container(container, "Item", id_)
await tm.commit(txn=txn)
tm._storage._transaction_strategy = strategy
print(f"Test content read with {strategy} strategy")
start = time.time()
for _ in range(ITERATIONS):
txn = await tm.begin()
assert await txn.get(ob.__uuid__) is not None
await tm.commit(txn=txn)
end = time.time()
print(f"Done with {ITERATIONS} in {end - start} seconds")
print(f"Test large content read with {strategy} strategy")
start = time.time()
txn = await tm.begin()
for _ in range(ITERATIONS):
assert await txn.get(ob.__uuid__) is not None
await tm.commit(txn=txn)
end = time.time()
print(f"Done with {ITERATIONS} in {end - start} seconds\n")
async def es_migrate(path,
root,
request,
reindex_security=False,
mapping_only=False,
full=False,
force=False):
try:
ob, end_path = await traverse(request, root,
path.lstrip('/').split('/'))
if len(end_path) != 0:
raise Exception('Could not found object')
search = getUtility(ICatalogUtility)
migrator = Migrator(search,
ob,
reindex_security=reindex_security,
full=full,
force=force,
mapping_only=mapping_only,
request=request,
log_details=True)
await migrator.run_migration()
finally:
txn = get_transaction(request)
if txn is not None:
tm = get_tm(request)
await tm.abort(txn=txn)
async def write_runner(container, strategy):
txn = get_transaction()
tm = get_tm()
await tm.abort(txn=txn)
tm._storage._transaction_strategy = strategy
print(f'Test content create with {strategy} strategy')
start = time.time()
for _ in range(ITERATIONS):
txn = await tm.begin()
id_ = uuid.uuid4().hex
await create_content_in_container(container, 'Item', id_)
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds')
print(f'Test large number create with {strategy} strategy')
start = time.time()
txn = await tm.begin()
for _ in range(ITERATIONS):
id_ = uuid.uuid4().hex
await create_content_in_container(container, 'Item', id_)
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds\n')
def build_count_query(
self,
context,
query: ParsedQueryInfo,
unrestricted: bool = False,
) -> typing.Tuple[str, typing.List[typing.Any]]:
sql_arguments = []
sql_wheres = []
select_fields = ["count(*)"]
arg_index = 1
for idx, where in enumerate(query["wheres"]):
sql_wheres.append(where.format(arg=arg_index))
sql_arguments.append(query["wheres_arguments"][idx])
arg_index += 1
sql_wheres.extend(
self.get_default_where_clauses(context, unrestricted=unrestricted))
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
sql = """select {}
from {}
where {}""".format(",".join(select_fields),
sqlq(txn.storage.objects_table_name),
" AND ".join(sql_wheres))
return sql, sql_arguments
async def read_runner(container, strategy):
request = get_current_request()
txn = get_transaction(request)
tm = get_tm(request)
id_ = uuid.uuid4().hex
await tm.abort(txn=txn)
txn = await tm.begin(request=request)
ob = await create_content_in_container(container, 'Item', id_)
await tm.commit(txn=txn)
tm._storage._transaction_strategy = strategy
print(f'Test content read with {strategy} strategy')
start = time.time()
for _ in range(ITERATIONS):
txn = await tm.begin(request=request)
assert await txn.get(ob._p_oid) is not None
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds')
print(f'Test large content read with {strategy} strategy')
start = time.time()
txn = await tm.begin(request=request)
for _ in range(ITERATIONS):
assert await txn.get(ob._p_oid) is not None
await tm.commit(txn=txn)
end = time.time()
print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def get_object_by_uid(uid: str, txn=None) -> IBaseObject:
'''
Get an object from an uid
:param uid: Object id of object you need to retreive
:param txn: Database transaction object. Will get current
transaction is not provided
'''
if txn is None:
from guillotina.transactions import get_transaction
txn = get_transaction()
result = txn._manager._hard_cache.get(uid, None)
if result is None:
result = await txn._get(uid)
if result['parent_id'] == TRASHED_ID:
raise KeyError(uid)
obj = app_settings['object_reader'](result)
obj.__txn__ = txn
if result['parent_id']:
parent = await get_object_by_uid(result['parent_id'], txn)
if parent is not None:
obj.__parent__ = parent
else:
raise KeyError(result['parent_id'])
return obj
async def get_object_by_oid(oid: str, txn=None) -> typing.Optional[IResource]:
'''
Get an object from an oid
:param oid: Object id of object you need to retreive
:param txn: Database transaction object. Will get current
transaction is not provided
'''
if txn is None:
from guillotina.transactions import get_transaction
txn = get_transaction()
result = txn._manager._hard_cache.get(oid, None)
if result is None:
try:
result = await txn._get(oid)
except KeyError:
return None
if result['parent_id'] == TRASHED_ID:
return None
obj = reader(result)
obj._p_jar = txn
if result['parent_id']:
obj.__parent__ = await get_object_by_oid(result['parent_id'], txn)
return obj
async def items(context, request):
try:
page_size = int(request.query["page_size"])
except Exception:
page_size = 20
try:
page = int(request.query["page"])
except Exception:
page = 1
txn = get_transaction()
include = omit = []
if request.query.get("include"):
include = request.query.get("include").split(",")
if request.query.get("omit"):
omit = request.query.get("omit").split(",")
results = []
for key in await txn.get_page_of_keys(context.__uuid__, page=page, page_size=page_size):
ob = await context.async_get(key)
serializer = get_multi_adapter((ob, request), IResourceSerializeToJson)
try:
results.append(await serializer(include=include, omit=omit))
except TypeError:
results.append(await serializer())
return {"items": results, "total": await context.async_len(), "page": page, "page_size": page_size}
def _get_transaction(self) -> ITransaction:
txn = get_transaction()
if txn is not None:
return txn
if self.__txn__ is not None:
return self.__txn__
raise TransactionNotFound()
async def create_content_in_container(parent: Folder,
type_: str,
id_: str,
request: IRequest = None,
check_security=True,
**kw) -> Resource:
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
:param parent: where to create content inside of
:param type_: content type to create
:param id_: id to give content in parent object
:param request: <optional>
:param check_security: be able to disable security checks
"""
factory = get_cached_factory(type_)
if check_security and factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = query_utility(IPermission,
name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not IInteraction(request).check_permission(permission.id, parent):
raise NoPermissionToAdd(str(parent), type_)
constrains = IConstrainTypes(parent, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(parent), type_)
# We create the object with at least the ID
obj = factory(id=id_, parent=parent)
for key, value in kw.items():
if key == 'id':
# the factory sets id
continue
setattr(obj, key, value)
txn = getattr(parent, '_p_jar', None) or get_transaction()
if txn is None or not txn.storage.supports_unique_constraints:
# need to manually check unique constraints
if await parent.async_contains(obj.id):
raise ConflictIdOnContainer(f'Duplicate ID: {parent} -> {obj.id}')
obj.__new_marker__ = True
await notify(BeforeObjectAddedEvent(obj, parent, id_))
await parent.async_set(obj.id, obj)
return obj
def get_storage():
txn = get_transaction()
storage = txn.manager._storage
if not IPostgresStorage.providedBy(storage):
# would already get big warning log about this
logger.debug("Storage does not support link integrity")
return None
return storage
def _p_jar(self):
try:
txn = get_transaction()
if txn is None:
txn = self.get_transaction_manager()._last_txn
return txn
except AttributeError:
return self.get_transaction_manager()._last_txn
async def async_get_oldest(self):
txn = get_transaction()
conn = await txn.get_connection()
result = await conn.fetch(GET_OLDEST_CHILDREN, self._p_oid)
if result is None or len(result) == 0:
return None
return await txn.get_child(self, result[0]['id'])
def _p_jar(self):
try:
txn = get_transaction()
if txn is None:
txn = self.get_transaction_manager()._last_txn
return txn
except AttributeError:
return self.get_transaction_manager()._last_txn
async def initialize_catalog(self, site):
txn = get_transaction()
conn = await txn.get_connection()
if conn is not None:
for name, index in schema.get_indexes().items():
await conn.execute('''DROP INDEX IF EXISTS {}'''.format(
index.idx_name))
await conn.execute(index.index_sql)
def __init__(self, blob, mode, transaction=None):
self.blob = blob
self.mode = mode
if transaction is None:
transaction = get_transaction()
if transaction is None:
raise Exception("Can not find transaction to work on blob with")
self.transaction = transaction
def add_job_after_commit(self, func: typing.Callable[[], typing.Coroutine], args=None, kwargs=None):
txn = get_transaction()
if txn is not None:
txn.add_after_commit_hook(
self._add_job_after_commit, args=[func], kws={"args": args, "kwargs": kwargs}
)
else:
raise TransactionNotFound("Could not find transaction to run job with")
async def finish_migration(self):
registry = await self.get_registry()
next_version = registry["el_next_index_version"]
assert next_version is not None
txn = get_transaction()
await txn.refresh(registry)
registry["el_index_version"] = next_version
registry["el_next_index_version"] = None
registry.register()
async def move(context, request):
try:
data = await request.json()
except Exception:
data = {}
destination = data.get('destination')
if destination is None:
destination_ob = context.__parent__
else:
try:
destination_ob = await navigate_to(request.container, destination)
except KeyError:
destination_ob = None
if destination_ob is None:
raise PreconditionFailed(context, 'Could not find destination object')
old_id = context.id
if 'new_id' in data:
new_id = data['new_id']
context.id = context.__name__ = new_id
else:
new_id = context.id
security = IInteraction(request)
if not security.check_permission('guillotina.AddContent', destination_ob):
raise PreconditionFailed(
context, 'You do not have permission to add content to the '
'destination object')
if await destination_ob.async_contains(new_id):
raise PreconditionFailed(
context, f'Destination already has object with the id {new_id}')
original_parent = context.__parent__
txn = get_transaction(request)
cache_keys = txn._cache.get_cache_keys(context, 'deleted')
data['id'] = new_id
await notify(
BeforeObjectMovedEvent(context, original_parent, old_id, destination_ob,
new_id, payload=data))
context.__parent__ = destination_ob
context._p_register()
await notify(
ObjectMovedEvent(context, original_parent, old_id, destination_ob,
new_id, payload=data))
cache_keys += txn._cache.get_cache_keys(context, 'added')
await txn._cache.delete_all(cache_keys)
absolute_url = query_multi_adapter((context, request), IAbsoluteURL)
return {
'@url': absolute_url()
}
def build_query(
self,
container: IContainer,
query: ParsedQueryInfo,
select_fields: typing.List[str],
distinct: typing.Optional[bool] = False
) -> typing.Tuple[str, typing.List[typing.Any]]:
if query['sort_on'] is None:
# always need a sort otherwise paging never works
order_by_index = get_pg_index('uuid')
else:
order_by_index = get_pg_index(query['sort_on']) or BasicJsonIndex(
query['sort_on'])
sql_arguments = []
sql_wheres = []
arg_index = 1
for idx, select in enumerate(query['selects']):
select_fields.append(select.format(arg=arg_index))
sql_arguments.append(query['selects_arguments'][idx])
arg_index += 1
where_arg_index = 0
for where in query['wheres']:
if isinstance(where, tuple):
operator, sub_wheres = where
sub_result = []
for sub_where in sub_wheres:
sub_result.append(
sub_where.format(arg=arg_index + where_arg_index))
sql_arguments.append(
query['wheres_arguments'][where_arg_index])
where_arg_index += 1
sql_wheres.append('(' + operator.join(sub_result) + ')')
else:
sql_wheres.append(where.format(arg=arg_index +
where_arg_index))
sql_arguments.append(
query['wheres_arguments'][where_arg_index])
where_arg_index += 1
txn = get_transaction()
if txn is None:
raise TransactionNotFound()
sql_wheres.extend(self.get_default_where_clauses(container))
sql = '''select {} {}
from {}
where {}
{}
limit {} offset {}'''.format(
'distinct' if distinct else '', ','.join(select_fields),
sqlq(txn.storage.objects_table_name), ' AND '.join(sql_wheres),
'' if distinct else order_by_index.order_by(query['sort_dir']),
sqlq(query['size']), sqlq(query['_from']))
return sql, sql_arguments
async def get_registry(self, refresh=False):
if (refresh and hasattr(self.request, 'container_settings') and
REGISTRY_DATA_KEY in self._get_annotations()):
txn = get_transaction(self.request)
await txn.refresh(self.request.container_settings)
if hasattr(self.request, 'container_settings'):
return self.request.container_settings
annotations_container = IAnnotations(self.container)
self.request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY) # noqa
return self.request.container_settings
def register(self, prefer_local=False):
if not prefer_local:
txn = get_transaction()
if txn is not None:
txn.register(self)
return
if self.__txn__ is not None:
self.__txn__.register(self)
return
raise TransactionNotFound()
def _get_current_tid(self):
# make sure to get current committed tid or we may be one-behind
# for what was actually used to commit to db
tid = None
try:
txn = get_transaction()
if txn:
tid = txn._tid
except RequestNotFound:
pass
return tid
def add_job_after_commit(self, func: typing.Callable[[], typing.Coroutine],
request=None, args=None, kwargs=None):
txn = get_transaction(request)
txn.add_after_commit_hook(
self._add_job_after_commit,
args=[func],
kws={
'request': request,
'args': args,
'kwargs': kwargs
})
async def initialize(self, app=None):
# loop
self.app = app
while True:
got_obj = False
try:
view = await self._queue.get()
got_obj = True
txn = get_transaction(view.request)
tm = get_tm(view.request)
if txn is None or (txn.status in (
Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and
txn._db_conn is None):
txn = await tm.begin(view.request)
else:
# still finishing current transaction, this connection
# will be cut off, so we need to wait until we no longer
# have an active transaction on the reqeust...
await self.add(view)
await asyncio.sleep(1)
continue
try:
aiotask_context.set('request', view.request)
view_result = await view()
if isinstance(view_result, ErrorResponse):
await tm.commit(txn=txn)
elif isinstance(view_result, UnauthorizedResponse):
await tm.abort(txn=txn)
else:
await tm.commit(txn=txn)
except Unauthorized:
await tm.abort(txn=txn)
except Exception as e:
logger.error(
"Exception on writing execution",
exc_info=e)
await tm.abort(txn=txn)
except (KeyboardInterrupt, MemoryError, SystemExit,
asyncio.CancelledError, GeneratorExit, RuntimeError):
self._exceptions = True
raise
except Exception as e: # noqa
self._exceptions = True
logger.error('Worker call failed', exc_info=e)
finally:
aiotask_context.set('request', None)
if got_obj:
try:
view.request.execute_futures()
except AttributeError:
pass
self._queue.task_done()
async def get_registry(self, refresh=False):
if (refresh and self.object_settings is not None):
txn = get_transaction(self.request)
await txn.refresh(self.object_settings)
annotations_container = IAnnotations(self.context)
self.object_settings = await annotations_container.async_get('default')
if self.object_settings is None:
# need to create annotation...
self.object_settings = AnnotationData()
await annotations_container.async_set('default',
self.object_settings)
return self.object_settings
def _record(query, duration):
# log each query on the transaction object...
try:
txn = get_transaction()
if txn:
if not hasattr(txn, '_queries'):
txn._queries = {}
if query not in txn._queries:
txn._queries[query] = [0, 0.0]
txn._queries[query][0] += 1
txn._queries[query][1] += duration
except AttributeError:
pass
async def initialize(self, app=None):
# loop
self.app = app
while True:
got_obj = False
try:
view = await self.queue.get()
got_obj = True
txn = get_transaction(view.request)
tm = get_tm(view.request)
if txn is None or (txn.status in (
Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and
txn._db_conn is None):
txn = await tm.begin(view.request)
else:
# still finishing current transaction, this connection
# will be cut off, so we need to wait until we no longer
# have an active transaction on the reqeust...
await self.add(view)
await asyncio.sleep(1)
continue
try:
aiotask_context.set('request', view.request)
await view()
await tm.commit(txn=txn)
except Exception as e:
logger.error(
"Exception on writing execution",
exc_info=e)
await tm.abort(txn=txn)
except (RuntimeError, SystemExit, GeneratorExit, KeyboardInterrupt,
asyncio.CancelledError, KeyboardInterrupt):
# dive, these errors mean we're exit(ing)
self._exceptions = True
return
except Exception as e: # noqa
self._exceptions = True
logger.error('Worker call failed', exc_info=e)
finally:
try:
aiotask_context.set('request', None)
except (RuntimeError, ValueError):
pass
if got_obj:
try:
view.request.execute_futures()
except AttributeError:
pass
self.queue.task_done()
def _record(query, duration):
# log each query on the transaction object...
try:
from guillotina.transactions import get_transaction
txn = get_transaction()
if txn:
if not hasattr(txn, '_queries'):
txn._queries = {}
if query not in txn._queries:
txn._queries[query] = [0, 0.0]
txn._queries[query][0] += 1
txn._queries[query][1] += duration
except AttributeError:
pass
