async def __call__(self): tm = get_tm(self.request) await tm.abort(self.request) ws = web.WebSocketResponse() await ws.prepare(self.request) async for msg in ws: if msg.type == aiohttp.WSMsgType.text: try: message = ujson.loads(msg.data) except ValueError: logger.warning('Invalid websocket payload, ignored: {}'.format( msg.data)) continue if message['op'] == 'close': await ws.close() elif message['op'].lower() == 'get': txn = await tm.begin(request=self.request) try: await self.handle_ws_request(ws, message) except Exception: logger.error('Exception on ws', exc_info=True) finally: # only currently support GET requests which are *never* # supposed to be commits await tm.abort(txn=txn) elif msg.type == aiohttp.WSMsgType.error: logger.debug('ws connection closed with exception {0:s}' .format(ws.exception())) logger.debug('websocket connection closed') return {}
async def read_runner(container, strategy): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin(request=request) ob = await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content read with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin(request=request) assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large content read with {strategy} strategy') start = time.time() txn = await tm.begin(request=request) for _ in range(ITERATIONS): assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def write_runner(container, strategy): txn = get_transaction() tm = get_tm() await tm.abort(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content create with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin() id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large number create with {strategy} strategy') start = time.time() txn = await tm.begin() for _ in range(ITERATIONS): id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def ws_notificate(context, request): ws = web.WebSocketResponse() utility = get_utility(INotificationSender) utility.register_ws(ws) print(request.rel_url) tm = get_tm() await tm.abort() await ws.prepare(request) try: async for msg in ws: if msg.tp == aiohttp.WSMsgType.text: #la nostra socket non riceve nessun messaggio, li invia solo pass elif msg.tp == aiohttp.WSMsgType.error: logger.debug( 'ws connection closed with exception {0:s}'.format(ws.exception())) except (RuntimeError, asyncio.CancelledError): pass finally: logger.debug('websocket connection closed') utility.unregister_ws(ws) return {}
async def read_runner(container, strategy): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin(request=request) ob = await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f'Test content read with {strategy} strategy') start = time.time() for _ in range(ITERATIONS): txn = await tm.begin(request=request) assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds') print(f'Test large content read with {strategy} strategy') start = time.time() txn = await tm.begin(request=request) for _ in range(ITERATIONS): assert await txn.get(ob._p_oid) is not None await tm.commit(txn=txn) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds\n')
async def __call__(self): tm = get_tm() await tm.abort() ws = self.request.get_ws() await ws.prepare() async for msg in ws: try: message = msg.json except WebSocketJsonDecodeError: # We only care about json messages logger.warning( "Invalid websocket payload, ignored: {}".format(msg)) continue if message["op"].lower() == "close": break elif message["op"].lower() == "get": txn = await tm.begin() try: await self.handle_ws_request(ws, message) except Exception: logger.error("Exception on ws", exc_info=True) finally: # only currently support GET requests which are *never* # supposed to be commits await tm.abort(txn=txn) logger.debug("websocket connection closed") await ws.close()
async def __call__(self): tm = get_tm(self.request) await tm.abort(self.request) ws = web.WebSocketResponse() await ws.prepare(self.request) async for msg in ws: if msg.tp == aiohttp.WSMsgType.text: message = ujson.loads(msg.data) if message['op'] == 'close': await ws.close() elif message['op'] == 'GET': txn = await tm.begin(request=self.request) try: await self.handle_ws_request(ws, message) except Exception: await ws.close() raise finally: # only currently support GET requests which are *never* # supposed to be commits await tm.abort(txn=txn) else: await ws.close() elif msg.tp == aiohttp.WSMsgType.error: logger.debug( 'ws connection closed with exception {0:s}'.format( ws.exception())) logger.debug('websocket connection closed') return {}
async def __call__(self): tm = get_tm() await tm.abort() ws = web.WebSocketResponse() await ws.prepare(self.request) async for msg in ws: if msg.type == aiohttp.WSMsgType.text: try: message = ujson.loads(msg.data) except ValueError: logger.warning( 'Invalid websocket payload, ignored: {}'.format( msg.data)) continue if message['op'] == 'close': await ws.close() elif message['op'].lower() == 'get': txn = await tm.begin() try: await self.handle_ws_request(ws, message) except Exception: logger.error('Exception on ws', exc_info=True) finally: # only currently support GET requests which are *never* # supposed to be commits await tm.abort(txn=txn) elif msg.type == aiohttp.WSMsgType.error: logger.debug( 'ws connection closed with exception {0:s}'.format( ws.exception())) logger.debug('websocket connection closed') return {}
async def retry_logic(context, request): """ First time sets 'title' and produce a conflict when the title is edited. Then this request should be retried automatically and finish successfuly """ ob = await context.async_get("bar") if ob.title is None: # Modify field 'title' and commit change ob.title = "A beautiful title" tm = get_tm() txn = get_transaction() txn.register(ob) await tm.commit() # Simulate a conflict error to test retry logic await tm.begin() ob.title = "edit title" ob.__serial__ = 3242432 # should raise conflict error when tm.commit() txn.register(ob) elif ob.title == "A beautiful title": ob.title = "retry logic works" txn = get_transaction() txn.register(ob) else: raise Exception("Something is not working as expected")
async def es_migrate(path, root, request, reindex_security=False, mapping_only=False, full=False, force=False): try: ob, end_path = await traverse(request, root, path.lstrip('/').split('/')) if len(end_path) != 0: raise Exception('Could not found object') search = getUtility(ICatalogUtility) migrator = Migrator(search, ob, reindex_security=reindex_security, full=full, force=force, mapping_only=mapping_only, request=request, log_details=True) await migrator.run_migration() finally: txn = get_transaction(request) if txn is not None: tm = get_tm(request) await tm.abort(txn=txn)
async def read_runner(container, strategy): txn = get_transaction() tm = get_tm() id_ = uuid.uuid4().hex await tm.abort(txn=txn) txn = await tm.begin() ob = await create_content_in_container(container, "Item", id_) await tm.commit(txn=txn) tm._storage._transaction_strategy = strategy print(f"Test content read with {strategy} strategy") start = time.time() for _ in range(ITERATIONS): txn = await tm.begin() assert await txn.get(ob.__uuid__) is not None await tm.commit(txn=txn) end = time.time() print(f"Done with {ITERATIONS} in {end - start} seconds") print(f"Test large content read with {strategy} strategy") start = time.time() txn = await tm.begin() for _ in range(ITERATIONS): assert await txn.get(ob.__uuid__) is not None await tm.commit(txn=txn) end = time.time() print(f"Done with {ITERATIONS} in {end - start} seconds\n")
async def __call__(self): self.data = {} tm = get_tm() await tm.abort() self.ws = self.request.get_ws() try: await self.ws.prepare() except ConnectionResetError: return {} try: self.pubsub = get_utility(IPubSubUtility) self.channel_name = "ws-field-edit-{}".format( self.context.__uuid__) # subscribe to redis channel for this context asyncio.ensure_future( self.pubsub.subscribe(self.channel_name, self.request.uid, self.subscriber_callback)) self.configure_auto_save() async for msg in self.ws: try: message = msg.json except Exception: # We only care about json messages logger.warning( "Invalid websocket payload, ignored: {}".format(msg)) continue if message["op"].lower() == "close": break try: await self.handle_message(message) except Exception: logger.error("Exception on ws", exc_info=True) except asyncio.CancelledError: logger.debug("browser closed") pass finally: try: await self.pubsub.unsubscribe(self.channel_name, self.request.uid) await self.ws.close() # make sure to close socket except Exception: pass logger.debug("websocket connection closed") return {}
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: view = await self._queue.get() got_obj = True txn = get_transaction(view.request) tm = get_tm(view.request) if txn is None or (txn.status in ( Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and txn._db_conn is None): txn = await tm.begin(view.request) else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.add(view) await asyncio.sleep(1) continue try: aiotask_context.set('request', view.request) view_result = await view() if isinstance(view_result, ErrorResponse): await tm.commit(txn=txn) elif isinstance(view_result, UnauthorizedResponse): await tm.abort(txn=txn) else: await tm.commit(txn=txn) except Unauthorized: await tm.abort(txn=txn) except Exception as e: logger.error( "Exception on writing execution", exc_info=e) await tm.abort(txn=txn) except (KeyboardInterrupt, MemoryError, SystemExit, asyncio.CancelledError, GeneratorExit, RuntimeError): self._exceptions = True raise except Exception as e: # noqa self._exceptions = True logger.error('Worker call failed', exc_info=e) finally: aiotask_context.set('request', None) if got_obj: try: view.request.execute_futures() except AttributeError: pass self._queue.task_done()
async def get_value(self, field_name): self.request._db_write_enabled = False tm = get_tm(self.request) txn = await tm.begin(self.request) context, field = await self.get_field(field_name) if IAsyncBehavior.implementedBy(context.__class__): await context.load() await txn.refresh(context.data) else: await txn.refresh(context) val = getattr(context, field.__name__) await tm.abort(txn=txn) return val
async def run_create(container): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) print(f'Test content create') start = time.time() for _ in range(ITERATIONS): id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) await tm.begin(request=request) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds')
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: view = await self.queue.get() got_obj = True txn = get_transaction(view.request) tm = get_tm(view.request) if txn is None or (txn.status in ( Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and txn._db_conn is None): txn = await tm.begin(view.request) else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.add(view) await asyncio.sleep(1) continue try: aiotask_context.set('request', view.request) await view() await tm.commit(txn=txn) except Exception as e: logger.error( "Exception on writing execution", exc_info=e) await tm.abort(txn=txn) except (RuntimeError, SystemExit, GeneratorExit, KeyboardInterrupt, asyncio.CancelledError, KeyboardInterrupt): # dive, these errors mean we're exit(ing) self._exceptions = True return except Exception as e: # noqa self._exceptions = True logger.error('Worker call failed', exc_info=e) finally: try: aiotask_context.set('request', None) except (RuntimeError, ValueError): pass if got_obj: try: view.request.execute_futures() except AttributeError: pass self.queue.task_done()
async def test_deserialize_cloud_file(dummy_request): from guillotina.test_package import IFileContent, FileContent with get_tm() as tm, await tm.begin() as txn, dummy_request: obj = create_content(FileContent) obj.__txn__ = txn obj.file = None await get_adapter( IFileContent['file'].bind(obj), IJSONToValue, args=[ 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', obj ]) assert isinstance(obj.file, DBFile) assert obj.file.size == 42
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: func, tvars = await self.queue.get() got_obj = True load_task_vars(tvars) txn = get_transaction() tm = get_tm() if txn is None or ( txn.status in (Status.ABORTED, Status.COMMITTED, Status.CONFLICT) and txn._db_conn is None ): txn = await tm.begin() else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.add((func, tvars)) await asyncio.sleep(1) continue try: await func() await tm.commit(txn=txn) except Exception as e: logger.error("Exception on writing execution", exc_info=e) await tm.abort(txn=txn) except ( RuntimeError, SystemExit, GeneratorExit, KeyboardInterrupt, asyncio.CancelledError, KeyboardInterrupt, ): # dive, these errors mean we're exit(ing) self._exceptions = True return except Exception as e: # noqa self._exceptions = True logger.error("Worker call failed", exc_info=e) finally: if got_obj: execute.execute_futures() self.queue.task_done()
async def ws_conversate(context, request): ws = request.get_ws() await ws.prepare() utility = get_utility(IMessageSender) utility.register_ws(ws, request) tm = get_tm() await tm.abort() try: async for msg in ws: # ws does not receive any messages, just sends pass finally: logger.debug('websocket connection closed') utility.unregister_ws(ws) return {}
async def __call__(self): self.data = {} self.ws = web.WebSocketResponse() tm = get_tm(self.request) await tm.abort(self.request) try: await self.ws.prepare(self.request) except ConnectionResetError: return {} try: self.pubsub = get_utility(IPubSubUtility) self.channel_name = "ws-field-edit-{}".format( self.context.__uuid__) # subscribe to redis channel for this context asyncio.ensure_future( self.pubsub.subscribe(self.channel_name, self.request.uid, self.subscriber_callback)) self.configure_auto_save() async for msg in self.ws: if msg.type == aiohttp.WSMsgType.text: await self.handle_message(msg) elif msg.type == aiohttp.WSMsgType.error: logger.debug( "resource ws connection closed with exception {0:s}". format(self.ws.exception())) except asyncio.CancelledError: logger.debug("browser closed") pass finally: try: await self.pubsub.unsubscribe(self.channel_name, self.request.uid) await self.ws.close() # make sure to close socket except Exception: pass logger.debug("websocket connection closed") return {}
async def run_modify(container): request = get_current_request() txn = get_transaction(request) tm = get_tm(request) id_ = uuid.uuid4().hex await create_content_in_container(container, 'Item', id_) await tm.commit(txn=txn) await tm.begin(request=request) print(f'Test content modify') start = time.time() for idx in range(ITERATIONS): ob = await container.async_get(id_) ob.title = str(idx) ob._p_register() await tm.commit(txn=txn) await tm.begin(request=request) end = time.time() print(f'Done with {ITERATIONS} in {end - start} seconds')
async def after_commit(success, txn, added, modified, deleted): if not success: return rules = list(get_rules()) need_update = added + modified reverse_update_matches = [] for m in chain(*[ compiler.need_reverse_update(obj, rules) for obj in (added + modified + deleted) ]): if m not in reverse_update_matches: reverse_update_matches.append(m) tm = get_tm() txn = await tm.begin() txn.guillotina_declperm_marker = True done = {} try: for obj in need_update: obj = await get_object_by_oid(obj._p_oid, txn) await compiler.apply_perms(txn, obj, rules) done[obj._p_oid] = obj for match in reverse_update_matches: async for res in compiler.get_resources_matching(txn, match): if res["zoid"] in done: continue obj = await get_object_by_oid(res["zoid"], txn) await compiler.apply_perms(txn, obj, rules) done[obj._p_oid] = obj except Exception: log.exception("error applying permissions") await tm.abort(txn=txn) else: log.debug("applied permissions successfully") await tm.commit(txn=txn)
async def initialize(self, app=None): self._queue = asyncio.Queue() while True: try: ob, request, event = await self._queue.get() txn = get_transaction(request) tm = get_tm(request) if txn is None or txn.status in (Status.ABORTED, Status.COMMITTED): txn = await tm.begin(request) else: # still finishing current transaction, this connection # will be cut off, so we need to wait until we no longer # have an active transaction on the reqeust... await self.push(ob, request, event) await asyncio.sleep(1) continue try: registry = self.request.container_settings config = registry.for_interface(IContentRuleSettings) if config is None: continue rules = [r for r in config['rules'].values()] executor = Executor(ob, request, event, rules) matching_rules = await executor.get_matching_rules() await executor.execute_actions(matching_rules) await tm.abort(txn=txn) except Exception as e: logger.error("Exception executing content rules", exc_info=e) await tm.abort(txn=txn) except Exception: logger.warn('Error processing queue', exc_info=True) await asyncio.sleep(1) finally: try: self._queue.task_done() except ValueError: pass
async def ws_conversate(context, request): ws = web.WebSocketResponse() utility = getUtility(IMessageSender) utility.register_ws(ws, request) tm = get_tm(request) await tm.abort(request) await ws.prepare(request) async for msg in ws: if msg.tp == aiohttp.WSMsgType.text: # ws does not receive any messages, just sends pass elif msg.tp == aiohttp.WSMsgType.error: logger.debug('ws connection closed with exception {0:s}'.format( ws.exception())) logger.debug('websocket connection closed') utility.unregister_ws(ws) return {}
async def es_reindex(path, root, request, reindex_security=False): try: ob, end_path = await traverse(request, root, path.lstrip('/').split('/')) assert len(end_path) == 0 search = getUtility(ICatalogUtility) full = True if reindex_security: full = False reindexer = Reindexer(search, ob, reindex_security=reindex_security, full=full, request=request, log_details=True) await reindexer.reindex(ob) finally: txn = get_transaction(request) if txn is not None: tm = get_tm(request) await tm.abort(txn=txn)
async def save(self): self.request._db_write_enabled = True tm = get_tm(self.request) txn = await tm.begin(self.request) for key, value in self.data.items(): context, field = await self.get_field(key) if IAsyncBehavior.implementedBy(context.__class__): # it's a behavior we're editing... await context.load() await txn.refresh(context.data) else: await txn.refresh(context) setattr(context, field.__name__, value) if IAsyncBehavior.implementedBy(context.__class__): # it's a behavior we're editing... context.data.register() else: context.register() await tm.commit(txn=txn)
async def handle(self, message): payload = message.get("payload") or {} if not isinstance(payload, str): payload = orjson.dumps(payload) headers = dict(self.request.headers) headers.update(message.get("headers") or {}) request = await self.clone_request(message["method"], message["endpoint"], payload, headers) try: task_vars.request.set(request) errored: bool = True try: result = await self._handle(request, message) errored = False except ErrorResponse as err: result = self._gen_result(err) except Exception as exc: logger.warning("Error executing batch item", exc_info=True) # Attempt to get error response from exception error_resp = query_adapter( exc, IErrorResponseException, kwargs={ "error": "ServiceError", "eid": uuid.uuid4().hex }, ) if error_resp is None: # If that didn't work, default to generic error response error_resp = generate_error_response(exc, request) result = self._gen_result(error_resp) finally: if errored and self.eager_commit: tm = get_tm() await tm.abort() return result finally: task_vars.request.set(self.request)
async def initialize(self, app=None): # loop self.app = app while True: got_obj = False try: priority, view = await self._queue.get() got_obj = True txn = get_transaction(view.request) if txn is None: tm = get_tm(view.request) txn = tm.begin(view.request) try: view_result = await view() if isinstance(view_result, ErrorResponse): await txn.commit() elif isinstance(view_result, UnauthorizedResponse): await txn.abort() else: await txn.commit() except Unauthorized: await txn.abort() except Exception as e: logger.error("Exception on writing execution", exc_info=e) await txn.abort() except KeyboardInterrupt or MemoryError or SystemExit or asyncio.CancelledError: self._exceptions = True raise except Exception as e: # noqa self._exceptions = True logger.error('Worker call failed', exc_info=e) finally: if SHARED_CONNECTION is False and hasattr( view.request, 'conn'): view.request.conn.close() if got_obj: self._queue.task_done()
async def apply_object(path, user_data, root, request, function, commit=False, args=None, kwargs=None): ''' Required options in task data are: - path: base path to start using apply on - function: what function to apply ob on - commit: should we commit after finishing, defaults to false - user_data: {id, roles, groups} - args: [] - kwargs: {} ''' if args is None: args = [] if kwargs is None: kwargs = {} path = path try: login_user(request, user_data) ob, end_path = await traverse(request, root, path.lstrip('/').split('/')) assert len(end_path) == 0 setup_request(request, user_data) function = resolve_dotted_name(function) return await apply_coroutine(function, ob, *args, **kwargs) finally: txn = get_transaction(request) if txn is not None: tm = get_tm(request) if commit: await tm.commit(txn=txn) else: await tm.abort(txn=txn)
async def __aenter__(self): tm = get_tm() self.txn = await tm.begin() self.root = await tm.get_root() return self.root
async def _handle(self, request, message): tm = get_tm() txn = get_transaction() if txn.status in (Status.ABORTED, Status.COMMITTED, Status.CONFLICT): # start txn txn = await tm.begin() method = app_settings["http_methods"][message["method"].upper()] endpoint = urlparse(message["endpoint"]).path path = tuple(p for p in endpoint.split("/") if p) obj, tail = await traverse(request, task_vars.container.get(), path) if tail and len(tail) > 0: # convert match lookups view_name = routes.path_to_view_name(tail) # remove query params from view name view_name = view_name.split("?")[0] elif not tail: view_name = "" else: raise permission = get_utility(IPermission, name="guillotina.AccessContent") security = get_security_policy() allowed = security.check_permission(permission.id, obj) if not allowed: return { "success": False, "body": { "reason": "Not allowed" }, "status": 401 } try: view = query_multi_adapter((obj, request), method, name=view_name) except AttributeError: view = None try: view.__route__.matches(request, tail or []) except (KeyError, IndexError, AttributeError): view = None if view is None: return { "success": False, "body": { "reason": "Not found" }, "status": 404 } ViewClass = view.__class__ view_permission = get_view_permission(ViewClass) if not security.check_permission(view_permission, view): return { "success": False, "body": { "reason": "No view access" }, "status": 401, } if hasattr(view, "prepare"): view = (await view.prepare()) or view view_result = await view() if self.eager_commit: await tm.commit() return self._gen_result(view_result)
async def abort_txn(ctx): tm = get_tm() await tm.abort()