async def retry_logic(context, request): """ First time sets 'title' and produce a conflict when the title is edited. Then this request should be retried automatically and finish successfuly """ ob = await context.async_get("bar") if ob.title is None: # Modify field 'title' and commit change ob.title = "A beautiful title" tm = get_tm() txn = get_transaction() txn.register(ob) await tm.commit() # Simulate a conflict error to test retry logic await tm.begin() ob.title = "edit title" ob.__serial__ = 3242432 # should raise conflict error when tm.commit() txn.register(ob) elif ob.title == "A beautiful title": ob.title = "retry logic works" txn = get_transaction() txn.register(ob) else: raise Exception("Something is not working as expected")
async def get_registry(self, refresh=False): if refresh and self.object_settings is not None: txn = get_transaction() await txn.refresh(self.object_settings) if self.object_settings is None: txn = get_transaction() is_active = txn.status in (Status.ACTIVE, Status.COMMITTING) if is_active: self.object_settings = await self._get_registry_or_create() else: async with transaction(): self.object_settings = await self._get_registry_or_create() return self.object_settings
def build_query( self, container: IContainer, query: ParsedQueryInfo, select_fields: typing.List[str], distinct: typing.Optional[bool] = False, ) -> typing.Tuple[str, typing.List[typing.Any]]: if query["sort_on"] is None: # always need a sort otherwise paging never works order_by_index = get_pg_index("uuid") else: order_by_index = get_pg_index(query["sort_on"]) or BasicJsonIndex( query["sort_on"]) sql_arguments = [] sql_wheres = [] arg_index = 1 for idx, select in enumerate(query["selects"]): select_fields.append(select.format(arg=arg_index)) sql_arguments.append(query["selects_arguments"][idx]) arg_index += 1 where_arg_index = 0 for where in query["wheres"]: if isinstance(where, tuple): operator, sub_wheres = where sub_result = [] for sub_where in sub_wheres: sub_result.append( sub_where.format(arg=arg_index + where_arg_index)) sql_arguments.append( query["wheres_arguments"][where_arg_index]) where_arg_index += 1 sql_wheres.append("(" + operator.join(sub_result) + ")") else: sql_wheres.append(where.format(arg=arg_index + where_arg_index)) sql_arguments.append( query["wheres_arguments"][where_arg_index]) where_arg_index += 1 txn = get_transaction() if txn is None: raise TransactionNotFound() sql_wheres.extend(self.get_default_where_clauses(container)) sql = """select {} {} from {} where {} {} limit {} offset {}""".format( "distinct" if distinct else "", ",".join(select_fields), sqlq(txn.storage.objects_table_name), " AND ".join(sql_wheres), "" if distinct else order_by_index.order_by(query["sort_dir"]), sqlq(query["size"]), sqlq(query["_from"]), ) return sql, sql_arguments
async def aggregation(self, container: IContainer, query: ParsedQueryInfo): select_fields = [ "json->'" + sqlq(field) + "' as " + sqlq(field) for field in query["metadata"] or [] ] # noqa sql, arguments = self.build_query(container, query, select_fields, True) txn = get_transaction() if txn is None: raise TransactionNotFound() conn = await txn.get_connection() results = [] logger.debug(f"Running search:\n{sql}\n{arguments}") async with txn.lock: records = await conn.fetch(sql, *arguments) for record in records: results.append([ json.loads(record[field]) for field in query["metadata"] or [] ]) total = len(results) if total >= query["size"] or query["_from"] != 0: sql, arguments = self.build_count_query(container, query) logger.debug(f"Running search:\n{sql}\n{arguments}") async with txn.lock: records = await conn.fetch(sql, *arguments) total = records[0]["count"] return {"items": results, "items_total": total}
async def read_runner(container, strategy): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin(request=request) ob = await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content read with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin(request=request) assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large content read with {strategy} strategy') start = time.time() txn = await tm.begin(request=request) for _ in range(ITERATIONS): assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def search(self, container: IContainer, query: ParsedQueryInfo): # type: ignore sql, arguments = self.build_query(container, query, ['id', 'zoid', 'json']) txn = get_transaction() if txn is None: raise TransactionNotFound() conn = await txn.get_connection() results = [] try: context_url = get_object_url(container) except RequestNotFound: context_url = get_content_path(container) logger.debug(f'Running search:\n{sql}\n{arguments}') for record in await conn.fetch(sql, *arguments): data = json.loads(record['json']) result = self.load_meatdata(query, data) result['@name'] = record['id'] result['@uid'] = record['zoid'] result['@id'] = data['@absolute_url'] = context_url + data['path'] results.append(result) # also do count... total = len(results) if total >= query['size'] or query['_from'] != 0: sql, arguments = self.build_count_query(container, query) logger.debug(f'Running search:\n{sql}\n{arguments}') records = await conn.fetch(sql, *arguments) total = records[0]['count'] return {'member': results, 'items_count': total}
async def aggregation(self, container: IContainer, query: ParsedQueryInfo): select_fields = [ 'json->\'' + sqlq(field) + '\' as ' + sqlq(field) for field in query['metadata'] or [] ] # noqa sql, arguments = self.build_query(container, query, select_fields, True) txn = get_transaction() if txn is None: raise TransactionNotFound() conn = await txn.get_connection() results = [] logger.debug(f'Running search:\n{sql}\n{arguments}') for record in await conn.fetch(sql, *arguments): results.append([ json.loads(record[field]) for field in query['metadata'] or [] ]) total = len(results) if total >= query['size'] or query['_from'] != 0: sql, arguments = self.build_count_query(container, query) logger.debug(f'Running search:\n{sql}\n{arguments}') records = await conn.fetch(sql, *arguments) total = records[0]['count'] return {'member': results, 'items_count': total}
async def read_runner(container, strategy): txn = get_transaction() tm = get_tm() id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin() ob = await create_content_in_container(container, "Item", id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f"Test content read with {strategy} strategy") start = time.time() for _ in range(ITERATIONS): txn = await tm.begin() assert await txn.get(ob.__uuid__) is not None await tm.commit(txn=txn) end = time.time() print(f"Done with {ITERATIONS} in {end - start} seconds") print(f"Test large content read with {strategy} strategy") start = time.time() txn = await tm.begin() for _ in range(ITERATIONS): assert await txn.get(ob.__uuid__) is not None await tm.commit(txn=txn) end = time.time() print(f"Done with {ITERATIONS} in {end - start} seconds\n")
async def es_migrate(path, root, request, reindex_security=False, mapping_only=False, full=False, force=False): try: ob, end_path = await traverse(request, root, path.lstrip('/').split('/')) if len(end_path) != 0: raise Exception('Could not found object') search = getUtility(ICatalogUtility) migrator = Migrator(search, ob, reindex_security=reindex_security, full=full, force=force, mapping_only=mapping_only, request=request, log_details=True) await migrator.run_migration() finally: txn = get_transaction(request) if txn is not None: tm = get_tm(request) await tm.abort(txn=txn)
async def write_runner(container, strategy): txn = get_transaction() tm = get_tm() await tm.abort(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content create with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin() id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large number create with {strategy} strategy') start = time.time() txn = await tm.begin() for _ in range(ITERATIONS): id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
def build_count_query( self, context, query: ParsedQueryInfo, unrestricted: bool = False, ) -> typing.Tuple[str, typing.List[typing.Any]]: sql_arguments = [] sql_wheres = [] select_fields = ["count(*)"] arg_index = 1 for idx, where in enumerate(query["wheres"]): sql_wheres.append(where.format(arg=arg_index)) sql_arguments.append(query["wheres_arguments"][idx]) arg_index += 1 sql_wheres.extend( self.get_default_where_clauses(context, unrestricted=unrestricted)) txn = get_transaction() if txn is None: raise TransactionNotFound() sql = """select {} from {} where {}""".format(",".join(select_fields), sqlq(txn.storage.objects_table_name), " AND ".join(sql_wheres)) return sql, sql_arguments
async def read_runner(container, strategy): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin(request=request) ob = await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content read with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin(request=request) assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large content read with {strategy} strategy') start = time.time() txn = await tm.begin(request=request) for _ in range(ITERATIONS): assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def get_object_by_uid(uid: str, txn=None) -> IBaseObject: ''' Get an object from an uid :param uid: Object id of object you need to retreive :param txn: Database transaction object. Will get current transaction is not provided ''' if txn is None: from guillotina.transactions import get_transaction txn = get_transaction() result = txn._manager._hard_cache.get(uid, None) if result is None: result = await txn._get(uid) if result['parent_id'] == TRASHED_ID: raise KeyError(uid) obj = app_settings['object_reader'](result) obj.__txn__ = txn if result['parent_id']: parent = await get_object_by_uid(result['parent_id'], txn) if parent is not None: obj.__parent__ = parent else: raise KeyError(result['parent_id']) return obj
async def get_object_by_oid(oid: str, txn=None) -> typing.Optional[IResource]: ''' Get an object from an oid :param oid: Object id of object you need to retreive :param txn: Database transaction object. Will get current transaction is not provided ''' if txn is None: from guillotina.transactions import get_transaction txn = get_transaction() result = txn._manager._hard_cache.get(oid, None) if result is None: try: result = await txn._get(oid) except KeyError: return None if result['parent_id'] == TRASHED_ID: return None obj = reader(result) obj._p_jar = txn if result['parent_id']: obj.__parent__ = await get_object_by_oid(result['parent_id'], txn) return obj
async def items(context, request): try: page_size = int(request.query["page_size"]) except Exception: page_size = 20 try: page = int(request.query["page"]) except Exception: page = 1 txn = get_transaction() include = omit = [] if request.query.get("include"): include = request.query.get("include").split(",") if request.query.get("omit"): omit = request.query.get("omit").split(",") results = [] for key in await txn.get_page_of_keys(context.__uuid__, page=page, page_size=page_size): ob = await context.async_get(key) serializer = get_multi_adapter((ob, request), IResourceSerializeToJson) try: results.append(await serializer(include=include, omit=omit)) except TypeError: results.append(await serializer()) return {"items": results, "total": await context.async_len(), "page": page, "page_size": page_size}
def _get_transaction(self) -> ITransaction: txn = get_transaction() if txn is not None: return txn if self.__txn__ is not None: return self.__txn__ raise TransactionNotFound()
async def create_content_in_container(parent: Folder, type_: str, id_: str, request: IRequest = None, check_security=True, **kw) -> Resource: """Utility to create a content. This method is the one to use to create content. id_ can be None :param parent: where to create content inside of :param type_: content type to create :param id_: id to give content in parent object :param request: <optional> :param check_security: be able to disable security checks """ factory = get_cached_factory(type_) if check_security and factory.add_permission: if factory.add_permission in PERMISSIONS_CACHE: permission = PERMISSIONS_CACHE[factory.add_permission] else: permission = query_utility(IPermission, name=factory.add_permission) PERMISSIONS_CACHE[factory.add_permission] = permission if request is None: request = get_current_request() if permission is not None and \ not IInteraction(request).check_permission(permission.id, parent): raise NoPermissionToAdd(str(parent), type_) constrains = IConstrainTypes(parent, None) if constrains is not None: if not constrains.is_type_allowed(type_): raise NotAllowedContentType(str(parent), type_) # We create the object with at least the ID obj = factory(id=id_, parent=parent) for key, value in kw.items(): if key == 'id': # the factory sets id continue setattr(obj, key, value) txn = getattr(parent, '_p_jar', None) or get_transaction() if txn is None or not txn.storage.supports_unique_constraints: # need to manually check unique constraints if await parent.async_contains(obj.id): raise ConflictIdOnContainer(f'Duplicate ID: {parent} -> {obj.id}') obj.__new_marker__ = True await notify(BeforeObjectAddedEvent(obj, parent, id_)) await parent.async_set(obj.id, obj) return obj
def get_storage(): txn = get_transaction() storage = txn.manager._storage if not IPostgresStorage.providedBy(storage): # would already get big warning log about this logger.debug("Storage does not support link integrity") return None return storage
def _p_jar(self): try: txn = get_transaction() if txn is None: txn = self.get_transaction_manager()._last_txn return txn except AttributeError: return self.get_transaction_manager()._last_txn
async def async_get_oldest(self): txn = get_transaction() conn = await txn.get_connection() result = await conn.fetch(GET_OLDEST_CHILDREN, self._p_oid) if result is None or len(result) == 0: return None return await txn.get_child(self, result[0]['id'])
def _p_jar(self): try: txn = get_transaction() if txn is None: txn = self.get_transaction_manager()._last_txn return txn except AttributeError: return self.get_transaction_manager()._last_txn
async def initialize_catalog(self, site): txn = get_transaction() conn = await txn.get_connection() if conn is not None: for name, index in schema.get_indexes().items(): await conn.execute('''DROP INDEX IF EXISTS {}'''.format( index.idx_name)) await conn.execute(index.index_sql)
def __init__(self, blob, mode, transaction=None): self.blob = blob self.mode = mode if transaction is None: transaction = get_transaction() if transaction is None: raise Exception("Can not find transaction to work on blob with") self.transaction = transaction
def add_job_after_commit(self, func: typing.Callable[[], typing.Coroutine], args=None, kwargs=None): txn = get_transaction() if txn is not None: txn.add_after_commit_hook( self._add_job_after_commit, args=[func], kws={"args": args, "kwargs": kwargs} ) else: raise TransactionNotFound("Could not find transaction to run job with")
async def finish_migration(self): registry = await self.get_registry() next_version = registry["el_next_index_version"] assert next_version is not None txn = get_transaction() await txn.refresh(registry) registry["el_index_version"] = next_version registry["el_next_index_version"] = None registry.register()
async def move(context, request): try: data = await request.json() except Exception: data = {} destination = data.get('destination') if destination is None: destination_ob = context.__parent__ else: try: destination_ob = await navigate_to(request.container, destination) except KeyError: destination_ob = None if destination_ob is None: raise PreconditionFailed(context, 'Could not find destination object') old_id = context.id if 'new_id' in data: new_id = data['new_id'] context.id = context.__name__ = new_id else: new_id = context.id security = IInteraction(request) if not security.check_permission('guillotina.AddContent', destination_ob): raise PreconditionFailed( context, 'You do not have permission to add content to the ' 'destination object') if await destination_ob.async_contains(new_id): raise PreconditionFailed( context, f'Destination already has object with the id {new_id}') original_parent = context.__parent__ txn = get_transaction(request) cache_keys = txn._cache.get_cache_keys(context, 'deleted') data['id'] = new_id await notify( BeforeObjectMovedEvent(context, original_parent, old_id, destination_ob, new_id, payload=data)) context.__parent__ = destination_ob context._p_register() await notify( ObjectMovedEvent(context, original_parent, old_id, destination_ob, new_id, payload=data)) cache_keys += txn._cache.get_cache_keys(context, 'added') await txn._cache.delete_all(cache_keys) absolute_url = query_multi_adapter((context, request), IAbsoluteURL) return { '@url': absolute_url() }
def build_query( self, container: IContainer, query: ParsedQueryInfo, select_fields: typing.List[str], distinct: typing.Optional[bool] = False ) -> typing.Tuple[str, typing.List[typing.Any]]: if query['sort_on'] is None: # always need a sort otherwise paging never works order_by_index = get_pg_index('uuid') else: order_by_index = get_pg_index(query['sort_on']) or BasicJsonIndex( query['sort_on']) sql_arguments = [] sql_wheres = [] arg_index = 1 for idx, select in enumerate(query['selects']): select_fields.append(select.format(arg=arg_index)) sql_arguments.append(query['selects_arguments'][idx]) arg_index += 1 where_arg_index = 0 for where in query['wheres']: if isinstance(where, tuple): operator, sub_wheres = where sub_result = [] for sub_where in sub_wheres: sub_result.append( sub_where.format(arg=arg_index + where_arg_index)) sql_arguments.append( query['wheres_arguments'][where_arg_index]) where_arg_index += 1 sql_wheres.append('(' + operator.join(sub_result) + ')') else: sql_wheres.append(where.format(arg=arg_index + where_arg_index)) sql_arguments.append( query['wheres_arguments'][where_arg_index]) where_arg_index += 1 txn = get_transaction() if txn is None: raise TransactionNotFound() sql_wheres.extend(self.get_default_where_clauses(container)) sql = '''select {} {} from {} where {} {} limit {} offset {}'''.format( 'distinct' if distinct else '', ','.join(select_fields), sqlq(txn.storage.objects_table_name), ' AND '.join(sql_wheres), '' if distinct else order_by_index.order_by(query['sort_dir']), sqlq(query['size']), sqlq(query['_from'])) return sql, sql_arguments
async def get_registry(self, refresh=False): if (refresh and hasattr(self.request, 'container_settings') and REGISTRY_DATA_KEY in self._get_annotations()): txn = get_transaction(self.request) await txn.refresh(self.request.container_settings) if hasattr(self.request, 'container_settings'): return self.request.container_settings annotations_container = IAnnotations(self.container) self.request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY) # noqa return self.request.container_settings
def register(self, prefer_local=False): if not prefer_local: txn = get_transaction() if txn is not None: txn.register(self) return if self.__txn__ is not None: self.__txn__.register(self) return raise TransactionNotFound()
def _get_current_tid(self): # make sure to get current committed tid or we may be one-behind # for what was actually used to commit to db tid = None try: txn = get_transaction() if txn: tid = txn._tid except RequestNotFound: pass return tid
def add_job_after_commit(self, func: typing.Callable[[], typing.Coroutine], request=None, args=None, kwargs=None): txn = get_transaction(request) txn.add_after_commit_hook( self._add_job_after_commit, args=[func], kws={ 'request': request, 'args': args, 'kwargs': kwargs })
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: view = await self._queue.get() got_obj = True txn = get_transaction(view.request) tm = get_tm(view.request) if txn is None or (txn.status in ( Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and txn._db_conn is None): txn = await tm.begin(view.request) else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.add(view) await asyncio.sleep(1) continue try: aiotask_context.set('request', view.request) view_result = await view() if isinstance(view_result, ErrorResponse): await tm.commit(txn=txn) elif isinstance(view_result, UnauthorizedResponse): await tm.abort(txn=txn) else: await tm.commit(txn=txn) except Unauthorized: await tm.abort(txn=txn) except Exception as e: logger.error( "Exception on writing execution", exc_info=e) await tm.abort(txn=txn) except (KeyboardInterrupt, MemoryError, SystemExit, asyncio.CancelledError, GeneratorExit, RuntimeError): self._exceptions = True raise except Exception as e: # noqa self._exceptions = True logger.error('Worker call failed', exc_info=e) finally: aiotask_context.set('request', None) if got_obj: try: view.request.execute_futures() except AttributeError: pass self._queue.task_done()
async def get_registry(self, refresh=False): if (refresh and self.object_settings is not None): txn = get_transaction(self.request) await txn.refresh(self.object_settings) annotations_container = IAnnotations(self.context) self.object_settings = await annotations_container.async_get('default') if self.object_settings is None: # need to create annotation... self.object_settings = AnnotationData() await annotations_container.async_set('default', self.object_settings) return self.object_settings
def _record(query, duration): # log each query on the transaction object... try: txn = get_transaction() if txn: if not hasattr(txn, '_queries'): txn._queries = {} if query not in txn._queries: txn._queries[query] = [0, 0.0] txn._queries[query][0] += 1 txn._queries[query][1] += duration except AttributeError: pass
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: view = await self.queue.get() got_obj = True txn = get_transaction(view.request) tm = get_tm(view.request) if txn is None or (txn.status in ( Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and txn._db_conn is None): txn = await tm.begin(view.request) else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.add(view) await asyncio.sleep(1) continue try: aiotask_context.set('request', view.request) await view() await tm.commit(txn=txn) except Exception as e: logger.error( "Exception on writing execution", exc_info=e) await tm.abort(txn=txn) except (RuntimeError, SystemExit, GeneratorExit, KeyboardInterrupt, asyncio.CancelledError, KeyboardInterrupt): # dive, these errors mean we're exit(ing) self._exceptions = True return except Exception as e: # noqa self._exceptions = True logger.error('Worker call failed', exc_info=e) finally: try: aiotask_context.set('request', None) except (RuntimeError, ValueError): pass if got_obj: try: view.request.execute_futures() except AttributeError: pass self.queue.task_done()
def _record(query, duration): # log each query on the transaction object... try: from guillotina.transactions import get_transaction txn = get_transaction() if txn: if not hasattr(txn, '_queries'): txn._queries = {} if query not in txn._queries: txn._queries[query] = [0, 0.0] txn._queries[query][0] += 1 txn._queries[query][1] += duration except AttributeError: pass