def get_tasks_summary(tasks, endpoint, complete=False, **kwargs): """Returns a dict that represents a summary of a tasks response :param tasks: items to be included in the response :param endpoint: endpoint from the request :param complete: whether to include the full representation of the tasks :param kwargs: additional (hashable) params to be included in the message :return: dict with the summary and the list of task representations """ tasks = tasks or [] if not isinstance(tasks, (list, tuple)): tasks = [tasks] # Get the information dict of each task tasks = filter(None, tasks) tasks = map(lambda t: get_task_info(t, complete=complete), tasks) zeo = get_post_zeo() complete_info = complete and " (complete)" or "" logger.info("::{}: {} tasks{} [{}]".format(endpoint, len(tasks), complete_info, zeo)) info = kwargs or {} info.update({ "count": len(tasks), "items": tasks, "url": japi.url_for("senaite.queue.{}".format(endpoint)), "zeo": zeo, }) return info
def _post(self, endpoint, resource=None, payload=None, timeout=10): """Sends a POST request to SENAITE's Queue Server Raises an exception if the response status is not HTTP 2xx or timeout :param endpoint: the endpoint to POST against :param resource: (Optional) resource from the endpoint to POST against :param payload: (Optional) hashable payload for the POST """ server_url = api.get_server_url() parts = "/".join(filter(None, [endpoint, resource])) url = "{}/@@API/senaite/v1/queue_server/{}".format(server_url, parts) logger.info("** POST: {}".format(url)) # HTTP Queue Authentication to be added in the request auth = QueueAuth(capi.get_current_user().id) # Additional information to the payload request = capi.get_request() if payload is None: payload = {} payload.update({"__zeo": request.get("SERVER_URL")}) # This might rise exceptions (e.g. TimeoutException) response = self._req.post(url, json=payload, auth=auth, timeout=timeout) # Check the request is successful. Raise exception otherwise response.raise_for_status() # Return the result return response.json()
def guard(self, action): """Returns False if the sample is queued or contains queued analyses """ # Check if this current request life-cycle is handled by a consumer request = capi.get_request() queue_task_uid = request.get("queue_tuid", "") if capi.is_uid(queue_task_uid): ctx_id = capi.get_id(self.context) logger.info("Skip guard for {}: {}".format(ctx_id, action)) return True # Don't do anything if senaite.queue is not enabled if not api.is_queue_enabled(): return True # Check if the sample is queued if api.is_queued(self.context, status=["queued"]): return False # Check whether the sample contains queued analyses for brain in self.context.getAnalyses(): if api.is_queued(brain, status=["queued"]): return False return True
def remove_legacy_storage(portal): """Removes the legacy storage and removes marker IQueued from objects """ logger.info("Removing legacy storage ...") # Main legacy tool for queue management of tasks legacy_storage_tool_id = "senaite.queue.main.storage" legacy_storage_tasks_id = "senaite.queue.main.storage.tasks" setup = _api.get_setup() annotations = IAnnotations(setup) # Walk through the tasks and remove IQueued marker interface tasks_storage = annotations.get(legacy_storage_tasks_id) or {} # Queued tasks map(remove_queued_task, tasks_storage.get("tasks", [])) # Failed tasks map(remove_queued_task, tasks_storage.get("failed", [])) # Current task remove_queued_task(tasks_storage.get("current")) # Last processed task remove_queued_task(tasks_storage.get("processed")) # Flush main storage annotation if annotations.get(legacy_storage_tool_id) is not None: del annotations[legacy_storage_tool_id] if annotations.get(legacy_storage_tasks_id) is not None: del annotations[legacy_storage_tasks_id] logger.info("Removing legacy storage [DONE]")
def post_install(portal_setup): """Runs after the last import step of the *default* profile This handler is registered as a *post_handler* in the generic setup profile :param portal_setup: SetupTool """ logger.info("{} install handler [BEGIN]".format(PRODUCT_NAME.upper())) context = portal_setup._getImportContext(PROFILE_ID) # noqa portal = context.getSite() # noqa logger.info("{} install handler [DONE]".format(PRODUCT_NAME.upper()))
def remove_queue_dispatcher_utility(portal): logger.info("Removing IQueueDispatcher utility ...") sm = portal.getSiteManager() sm.unregisterUtility(provided=IQueueDispatcher) util = sm.queryUtility(IQueueDispatcher) if util: del util sm.utilities.unsubscribe((), IQueueDispatcher) del sm.utilities.__dict__['_provided'][IQueueDispatcher] logger.info("Removed IQueueDispatcher utility ...")
def get(context, request, task_uid): # noqa """Returns a JSON representation of the task with the specified task uid """ # Get the task task = get_task(task_uid) # Digest the task and return zeo = get_post_zeo() task_uid = get_task_uid(task, default="none") logger.info("::server.get: {} [{}]".format(task_uid, zeo)) return get_task_info(task, complete=True)
def process(context, request, task_uid=None): # noqa """Processes the task passed-in """ # disable CSRF req.disable_csrf_protection() # Maybe the task uid has been sent via POST task_uid = task_uid or req.get_json().get("task_uid") # Get the task task = get_task(task_uid) if task.username != capi.get_current_user().id: # 403 Authenticated, but user does not have access to the resource _fail(403) # Process t0 = time.time() task_context = task.get_context() if not task_context: _fail(500, "Task's context is not available") # Get the adapter able to process this specific type of task adapter = queryAdapter(task_context, IQueuedTaskAdapter, name=task.name) if not adapter: _fail(501, "No adapter found for {}".format(task.name)) logger.info("Processing task {}: '{}' for '{}' ({}) ...".format( task.task_short_uid, task.name, capi.get_id(task_context), task.context_uid)) # Inject the queue_consumer marker to the request so guards skip checks # against the queue request = capi.get_request() request.set("queue_tuid", task_uid) # If the task refers to a worksheet, inject (ws_id) in params to make # sure guards (assign, un-assign) return True if IWorksheet.providedBy(task_context): request.set("ws_uid", capi.get_uid(task_context)) # Process the task adapter.process(task) # Sleep a bit for minimum effect against userland threads # Better to have a transaction conflict here than in userland min_seconds = task.get("min_seconds", 3) while time.time() - t0 < min_seconds: time.sleep(0.5) msg = "Processed: {}".format(task.task_short_uid) return get_message_summary(msg, "consumer.process")
def setup_pas_plugin(place): logger.info("Setting up Queue's PAS plugin ...") pas = place.acl_users if PAS_PLUGIN_ID not in pas.objectIds(): plugin = QueueAuthPlugin(title="SENAITE Queue PAS plugin") plugin.id = PAS_PLUGIN_ID pas._setObject(PAS_PLUGIN_ID, plugin) # noqa logger.info("Created {} in acl_users".format(PAS_PLUGIN_ID)) plugin = getattr(pas, PAS_PLUGIN_ID) if not isinstance(plugin, QueueAuthPlugin): raise ValueError( "PAS plugin {} is not a QueueAuthPlugin".format(PAS_PLUGIN_ID)) # Activate all supported interfaces for this plugin activatePluginInterfaces(place, PAS_PLUGIN_ID) # Make our plugin the first one for some interfaces top_interfaces = ["IExtractionPlugin", "IAuthenticationPlugin"] plugins = pas.plugins for info in pas.plugins.listPluginTypeInfo(): interface_name = info["id"] if interface_name in top_interfaces: iface = plugins._getInterfaceFromName(interface_name) # noqa for obj in plugins.listPlugins(iface): plugins.movePluginsUp(iface, [PAS_PLUGIN_ID]) logger.info("Moved {} to top of {}".format( PAS_PLUGIN_ID, interface_name)) logger.info("Setting up Queue's PAS plugin [DONE]")
def setup_handler(context): """Generic setup handler """ if context.readDataFile('senaite.queue.install.txt') is None: return logger.info("setup handler [BEGIN]".format(PRODUCT_NAME.upper())) portal = context.getSite() # noqa # Install the PAS Plugin to allow authenticating tasks as their creators, # both in senaite's site and in zope's root to grant access to zope's users setup_pas_plugin(portal.getPhysicalRoot()) setup_pas_plugin(portal) logger.info("{} setup handler [DONE]".format(PRODUCT_NAME.upper()))
def get_message_summary(message, endpoint, **kwargs): """Returns a dict that represents a summary of a message response :param message: message to be included in the response :param endpoint: endpoint from the request :param kwargs: additional (hashable) params to be included in the message :return: dict with the summary of the response """ zeo = get_post_zeo() logger.info("::{}: {} [{}]".format(endpoint, message, zeo)) info = kwargs or {} info.update({ "message": message, "url": japi.url_for("senaite.queue.{}".format(endpoint)), "zeo": zeo, }) return info
def pop(context, request): # noqa """Pops the next task from the queue, if any. Popped task is no longer available in the queued tasks pool, but added in the running tasks pool """ # Get the consumer ID consumer_id = req.get_json().get("consumer_id") if not is_consumer_id(consumer_id): _fail(428, "No valid consumer id") # Pop the task from the queue task = qapi.get_queue().pop(consumer_id) # Return the task info task_uid = get_task_uid(task, default="<empty>") logger.info("::server.pop: {} [{}]".format(task_uid, consumer_id)) return get_task_info(task, complete=True)
def reset_settings(portal): """Reset the settings from registry to match with defaults """ logger.info("Reset Queue settings ...") default_settings = { "default": 10, "max_retries": 3, "min_seconds_task": 3, "max_seconds_unlock": 120, } for key, val in default_settings.items(): registry_key = "senaite.queue.{}".format(key) plone_api.portal.set_registry_record(registry_key, val) logger.info("Reset Queue settings [DONE]")
def pre_install(portal_setup): """Runs before the first import step of the *default* profile This handler is registered as a *pre_handler* in the generic setup profile :param portal_setup: SetupTool """ logger.info("{} pre-install handler [BEGIN]".format(PRODUCT_NAME.upper())) context = portal_setup._getImportContext(PROFILE_ID) portal = context.getSite() # noqa # Only install senaite.lims once! qi = portal.portal_quickinstaller if not qi.isProductInstalled("senaite.lims"): profile_name = "profile-senaite.lims:default" portal_setup.runAllImportStepsFromProfile(profile_name) logger.info("{} pre-install handler [DONE]".format(PRODUCT_NAME.upper()))
def uninstall_pas_plugin(place): """Uninstalls the Queue's PAS Plugin """ pas = place.acl_users if PAS_PLUGIN_ID not in pas.objectIds(): return plugin = getattr(pas, PAS_PLUGIN_ID) if not isinstance(plugin, QueueAuthPlugin): logger.warning( "PAS plugin {} is not a QueueAuthPlugin".format(PAS_PLUGIN_ID)) return pas._delObject(PAS_PLUGIN_ID) # noqa access to protected logger.info( "Removed QueueAuthPlugin {} from acl_users".format(PAS_PLUGIN_ID))
def post_uninstall(portal_setup): """Runs after the last import step of the *uninstall* profile This handler is registered as a *post_handler* in the generic setup profile :param portal_setup: SetupTool """ logger.info("{} uninstall handler [BEGIN]".format(PRODUCT_NAME.upper())) # https://docs.plone.org/develop/addons/components/genericsetup.html#custom-installer-code-setuphandlers-py context = portal_setup._getImportContext(UNINSTALL_PROFILE_ID) # noqa portal = context.getSite() # noqa # Uninstall Queue's PAS plugin, from both Zope's acl_users and site's uninstall_pas_plugin(portal.getPhysicalRoot()) uninstall_pas_plugin(portal) logger.info("{} uninstall handler [DONE]".format(PRODUCT_NAME.upper()))
def upgrade(tool): portal = tool.aq_inner.aq_parent setup = portal.portal_setup ut = UpgradeUtils(portal) ver_from = ut.getInstalledVersion(PRODUCT_NAME) if ut.isOlderVersion(PRODUCT_NAME, version): logger.info("Skipping upgrade of {0}: {1} > {2}".format( PRODUCT_NAME, ver_from, version)) return True logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from, version)) # -------- ADD YOUR STUFF BELOW -------- # https://github.com/senaite/senaite.queue/pull/3 setup.runImportStepFromProfile(PROFILE_ID, "plone.app.registry") setup.runImportStepFromProfile(PROFILE_ID, "actions") # Remove queue dispatcher utility, that is no longer used setup.runImportStepFromProfile(PROFILE_ID, "componentregistry") remove_queue_dispatcher_utility(portal) logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version)) return True
def upgrade(tool): portal = tool.aq_inner.aq_parent setup = portal.portal_setup ut = UpgradeUtils(portal) ver_from = ut.getInstalledVersion(PRODUCT_NAME) if ut.isOlderVersion(PRODUCT_NAME, version): logger.info("Skipping upgrade of {0}: {1} > {2}".format( PRODUCT_NAME, ver_from, version)) return True logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from, version)) # Re-import the registry profile to add new settings in control panel setup.runImportStepFromProfile(PROFILE_ID, "plone.app.registry") # Reset control panel settings to defaults reset_settings(portal) # Port old storage mechanism remove_legacy_storage(portal) # Install the PAS Plugin to allow authenticating tasks as their creators setup_pas_plugin(portal) # Create and store the key to use for auth reset_auth_key(portal) logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version)) return True
def _add(self, task): # Only QueueTask type is supported if not is_task(task): raise ValueError("{} is not supported".format(repr(task))) # Don't add to the queue if the task is already in there if task in self._tasks: logger.warn("Task {} ({}) in the queue already".format( task.name, task.task_short_uid)) return None # Do not add the task if unique and task for same context and name if task.get("unique", False): query = {"context_uid": task.context_uid, "name": task.name} if self.search(query): logger.debug("Task {} for {} in the queue already".format( task.name, task.context_path)) return None # Update task status and append to the list of tasks task.update({"status": "queued"}) self._tasks.append(task) # Sort by priority + created reverse # We multiply the priority for 300 sec. (5 minutes) and then we sum the # result to the time the task was created. This way, we ensure tasks # priority at the same time we guarantee older, with low priority # tasks don't fall through the cracks. # TODO: Make this 300 sec. configurable? self._tasks.sort(key=lambda t: (t.created + (300 * t.priority))) # Update the since time if self._since_time < 0 or self._since_time > task.created: self._since_time = task.created logger.info("Added task {} ({}): {}".format(task.name, task.task_short_uid, task.context_path)) return task
def get_list_summary(items, endpoint, **kwargs): """Returns a dict that represents a summary of a list response :param items: items to be included in the response :param endpoint: endpoint from the request :param kwargs: additional (hashable) params to be included in the message :return: dict with the summary and the list of items """ items = items or [] if not isinstance(items, (list, tuple)): items = [items] # Remove empties items = filter(None, items) zeo = get_post_zeo() logger.info("::{}: {} items [{}]".format(endpoint, len(items), zeo)) info = kwargs or {} info.update({ "count": len(items), "items": items, "url": japi.url_for("senaite.queue.{}".format(endpoint)), "zeo": zeo, }) return info
def upgrade(tool): portal = tool.aq_inner.aq_parent setup = portal.portal_setup ut = UpgradeUtils(portal) ver_from = ut.getInstalledVersion(PRODUCT_NAME) if ut.isOlderVersion(PRODUCT_NAME, version): logger.info("Skipping upgrade of {0}: {1} > {2}".format( PRODUCT_NAME, ver_from, version)) return True logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from, version)) logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version)) return True
def consume_task(): """Consumes a task from the queue, if any """ if not is_installed(): return info("Queue is not installed") host = _api.get_request().get("SERVER_URL") if not is_valid_zeo_host(host): return error("zeo host not set or not valid: {} [SKIP]".format(host)) consumer_thread = get_consumer_thread() if consumer_thread: # There is a consumer working already name = consumer_thread.getName() return info("Consumer running: {} [SKIP]".format(name)) logger.info("Queue client: {}".format(host)) # Server's queue URL server = api.get_server_url() # Check the status of the queue status = api.get_queue_status() if status not in ["resuming", "ready"]: return warn("Server is {} ({}) [SKIP]".format(status, server)) if api.is_queue_server(): message = [ "Server = Consumer: {}".format(server), "*******************************************************", "Client configured as both queue server and consumer.", "This is not suitable for productive environments!", "Change the Queue Server URL in SENAITE's control panel", "or setup another zeo client as queue consumer.", "Current URL: {}".format(server), "*******************************************************" ] logger.warn("\n".join(message)) # Pop next task to process consumer_id = host try: task = api.get_queue().pop(consumer_id) if not task: return info("Queue is empty or process undergoing [SKIP]") except Exception as e: return error("Cannot pop. {}: {}".format(type(e).__name__, str(e))) auth_key = _api.get_registry_record("senaite.queue.auth_key") kwargs = { "task_uid": task.task_uid, "task_username": task.username, "consumer_id": consumer_id, "base_url": _api.get_url(_api.get_portal()), "server_url": api.get_server_url(), "user_id": _api.get_current_user().id, "max_seconds": get_max_seconds(), "auth_key": auth_key, } name = "{}{}".format(CONSUMER_THREAD_PREFIX, int(time.time())) t = threading.Thread(name=name, target=process_task, kwargs=kwargs) t.start() return info("Consumer running: {} [SKIP]".format(CONSUMER_THREAD_PREFIX))