def get_tasks_summary(tasks, endpoint, complete=False, **kwargs):
"""Returns a dict that represents a summary of a tasks response
:param tasks: items to be included in the response
:param endpoint: endpoint from the request
:param complete: whether to include the full representation of the tasks
:param kwargs: additional (hashable) params to be included in the message
:return: dict with the summary and the list of task representations
"""
tasks = tasks or []
if not isinstance(tasks, (list, tuple)):
tasks = [tasks]
# Get the information dict of each task
tasks = filter(None, tasks)
tasks = map(lambda t: get_task_info(t, complete=complete), tasks)
zeo = get_post_zeo()
complete_info = complete and " (complete)" or ""
logger.info("::{}: {} tasks{} [{}]".format(endpoint, len(tasks),
complete_info, zeo))
info = kwargs or {}
info.update({
"count": len(tasks),
"items": tasks,
"url": japi.url_for("senaite.queue.{}".format(endpoint)),
"zeo": zeo,
})
return info
def _post(self, endpoint, resource=None, payload=None, timeout=10):
"""Sends a POST request to SENAITE's Queue Server
Raises an exception if the response status is not HTTP 2xx or timeout
:param endpoint: the endpoint to POST against
:param resource: (Optional) resource from the endpoint to POST against
:param payload: (Optional) hashable payload for the POST
"""
server_url = api.get_server_url()
parts = "/".join(filter(None, [endpoint, resource]))
url = "{}/@@API/senaite/v1/queue_server/{}".format(server_url, parts)
logger.info("** POST: {}".format(url))
# HTTP Queue Authentication to be added in the request
auth = QueueAuth(capi.get_current_user().id)
# Additional information to the payload
request = capi.get_request()
if payload is None:
payload = {}
payload.update({"__zeo": request.get("SERVER_URL")})
# This might rise exceptions (e.g. TimeoutException)
response = self._req.post(url,
json=payload,
auth=auth,
timeout=timeout)
# Check the request is successful. Raise exception otherwise
response.raise_for_status()
# Return the result
return response.json()
def guard(self, action):
"""Returns False if the sample is queued or contains queued analyses
"""
# Check if this current request life-cycle is handled by a consumer
request = capi.get_request()
queue_task_uid = request.get("queue_tuid", "")
if capi.is_uid(queue_task_uid):
ctx_id = capi.get_id(self.context)
logger.info("Skip guard for {}: {}".format(ctx_id, action))
return True
# Don't do anything if senaite.queue is not enabled
if not api.is_queue_enabled():
return True
# Check if the sample is queued
if api.is_queued(self.context, status=["queued"]):
return False
# Check whether the sample contains queued analyses
for brain in self.context.getAnalyses():
if api.is_queued(brain, status=["queued"]):
return False
return True
def remove_legacy_storage(portal):
"""Removes the legacy storage and removes marker IQueued from objects
"""
logger.info("Removing legacy storage ...")
# Main legacy tool for queue management of tasks
legacy_storage_tool_id = "senaite.queue.main.storage"
legacy_storage_tasks_id = "senaite.queue.main.storage.tasks"
setup = _api.get_setup()
annotations = IAnnotations(setup)
# Walk through the tasks and remove IQueued marker interface
tasks_storage = annotations.get(legacy_storage_tasks_id) or {}
# Queued tasks
map(remove_queued_task, tasks_storage.get("tasks", []))
# Failed tasks
map(remove_queued_task, tasks_storage.get("failed", []))
# Current task
remove_queued_task(tasks_storage.get("current"))
# Last processed task
remove_queued_task(tasks_storage.get("processed"))
# Flush main storage annotation
if annotations.get(legacy_storage_tool_id) is not None:
del annotations[legacy_storage_tool_id]
if annotations.get(legacy_storage_tasks_id) is not None:
del annotations[legacy_storage_tasks_id]
logger.info("Removing legacy storage [DONE]")
def post_install(portal_setup):
"""Runs after the last import step of the *default* profile
This handler is registered as a *post_handler* in the generic setup profile
:param portal_setup: SetupTool
"""
logger.info("{} install handler [BEGIN]".format(PRODUCT_NAME.upper()))
context = portal_setup._getImportContext(PROFILE_ID) # noqa
portal = context.getSite() # noqa
logger.info("{} install handler [DONE]".format(PRODUCT_NAME.upper()))
def remove_queue_dispatcher_utility(portal):
logger.info("Removing IQueueDispatcher utility ...")
sm = portal.getSiteManager()
sm.unregisterUtility(provided=IQueueDispatcher)
util = sm.queryUtility(IQueueDispatcher)
if util:
del util
sm.utilities.unsubscribe((), IQueueDispatcher)
del sm.utilities.__dict__['_provided'][IQueueDispatcher]
logger.info("Removed IQueueDispatcher utility ...")
def get(context, request, task_uid): # noqa
"""Returns a JSON representation of the task with the specified task uid
"""
# Get the task
task = get_task(task_uid)
# Digest the task and return
zeo = get_post_zeo()
task_uid = get_task_uid(task, default="none")
logger.info("::server.get: {} [{}]".format(task_uid, zeo))
return get_task_info(task, complete=True)
def process(context, request, task_uid=None): # noqa
"""Processes the task passed-in
"""
# disable CSRF
req.disable_csrf_protection()
# Maybe the task uid has been sent via POST
task_uid = task_uid or req.get_json().get("task_uid")
# Get the task
task = get_task(task_uid)
if task.username != capi.get_current_user().id:
# 403 Authenticated, but user does not have access to the resource
_fail(403)
# Process
t0 = time.time()
task_context = task.get_context()
if not task_context:
_fail(500, "Task's context is not available")
# Get the adapter able to process this specific type of task
adapter = queryAdapter(task_context, IQueuedTaskAdapter, name=task.name)
if not adapter:
_fail(501, "No adapter found for {}".format(task.name))
logger.info("Processing task {}: '{}' for '{}' ({}) ...".format(
task.task_short_uid, task.name, capi.get_id(task_context),
task.context_uid))
# Inject the queue_consumer marker to the request so guards skip checks
# against the queue
request = capi.get_request()
request.set("queue_tuid", task_uid)
# If the task refers to a worksheet, inject (ws_id) in params to make
# sure guards (assign, un-assign) return True
if IWorksheet.providedBy(task_context):
request.set("ws_uid", capi.get_uid(task_context))
# Process the task
adapter.process(task)
# Sleep a bit for minimum effect against userland threads
# Better to have a transaction conflict here than in userland
min_seconds = task.get("min_seconds", 3)
while time.time() - t0 < min_seconds:
time.sleep(0.5)
msg = "Processed: {}".format(task.task_short_uid)
return get_message_summary(msg, "consumer.process")
def setup_pas_plugin(place):
logger.info("Setting up Queue's PAS plugin ...")
pas = place.acl_users
if PAS_PLUGIN_ID not in pas.objectIds():
plugin = QueueAuthPlugin(title="SENAITE Queue PAS plugin")
plugin.id = PAS_PLUGIN_ID
pas._setObject(PAS_PLUGIN_ID, plugin) # noqa
logger.info("Created {} in acl_users".format(PAS_PLUGIN_ID))
plugin = getattr(pas, PAS_PLUGIN_ID)
if not isinstance(plugin, QueueAuthPlugin):
raise ValueError(
"PAS plugin {} is not a QueueAuthPlugin".format(PAS_PLUGIN_ID))
# Activate all supported interfaces for this plugin
activatePluginInterfaces(place, PAS_PLUGIN_ID)
# Make our plugin the first one for some interfaces
top_interfaces = ["IExtractionPlugin", "IAuthenticationPlugin"]
plugins = pas.plugins
for info in pas.plugins.listPluginTypeInfo():
interface_name = info["id"]
if interface_name in top_interfaces:
iface = plugins._getInterfaceFromName(interface_name) # noqa
for obj in plugins.listPlugins(iface):
plugins.movePluginsUp(iface, [PAS_PLUGIN_ID])
logger.info("Moved {} to top of {}".format(
PAS_PLUGIN_ID, interface_name))
logger.info("Setting up Queue's PAS plugin [DONE]")
def setup_handler(context):
"""Generic setup handler
"""
if context.readDataFile('senaite.queue.install.txt') is None:
return
logger.info("setup handler [BEGIN]".format(PRODUCT_NAME.upper()))
portal = context.getSite() # noqa
# Install the PAS Plugin to allow authenticating tasks as their creators,
# both in senaite's site and in zope's root to grant access to zope's users
setup_pas_plugin(portal.getPhysicalRoot())
setup_pas_plugin(portal)
logger.info("{} setup handler [DONE]".format(PRODUCT_NAME.upper()))
def get_message_summary(message, endpoint, **kwargs):
"""Returns a dict that represents a summary of a message response
:param message: message to be included in the response
:param endpoint: endpoint from the request
:param kwargs: additional (hashable) params to be included in the message
:return: dict with the summary of the response
"""
zeo = get_post_zeo()
logger.info("::{}: {} [{}]".format(endpoint, message, zeo))
info = kwargs or {}
info.update({
"message": message,
"url": japi.url_for("senaite.queue.{}".format(endpoint)),
"zeo": zeo,
})
return info
def pop(context, request): # noqa
"""Pops the next task from the queue, if any. Popped task is no longer
available in the queued tasks pool, but added in the running tasks pool
"""
# Get the consumer ID
consumer_id = req.get_json().get("consumer_id")
if not is_consumer_id(consumer_id):
_fail(428, "No valid consumer id")
# Pop the task from the queue
task = qapi.get_queue().pop(consumer_id)
# Return the task info
task_uid = get_task_uid(task, default="<empty>")
logger.info("::server.pop: {} [{}]".format(task_uid, consumer_id))
return get_task_info(task, complete=True)
def reset_settings(portal):
"""Reset the settings from registry to match with defaults
"""
logger.info("Reset Queue settings ...")
default_settings = {
"default": 10,
"max_retries": 3,
"min_seconds_task": 3,
"max_seconds_unlock": 120,
}
for key, val in default_settings.items():
registry_key = "senaite.queue.{}".format(key)
plone_api.portal.set_registry_record(registry_key, val)
logger.info("Reset Queue settings [DONE]")
def pre_install(portal_setup):
"""Runs before the first import step of the *default* profile
This handler is registered as a *pre_handler* in the generic setup profile
:param portal_setup: SetupTool
"""
logger.info("{} pre-install handler [BEGIN]".format(PRODUCT_NAME.upper()))
context = portal_setup._getImportContext(PROFILE_ID)
portal = context.getSite() # noqa
# Only install senaite.lims once!
qi = portal.portal_quickinstaller
if not qi.isProductInstalled("senaite.lims"):
profile_name = "profile-senaite.lims:default"
portal_setup.runAllImportStepsFromProfile(profile_name)
logger.info("{} pre-install handler [DONE]".format(PRODUCT_NAME.upper()))
def uninstall_pas_plugin(place):
"""Uninstalls the Queue's PAS Plugin
"""
pas = place.acl_users
if PAS_PLUGIN_ID not in pas.objectIds():
return
plugin = getattr(pas, PAS_PLUGIN_ID)
if not isinstance(plugin, QueueAuthPlugin):
logger.warning(
"PAS plugin {} is not a QueueAuthPlugin".format(PAS_PLUGIN_ID))
return
pas._delObject(PAS_PLUGIN_ID) # noqa access to protected
logger.info(
"Removed QueueAuthPlugin {} from acl_users".format(PAS_PLUGIN_ID))
def post_uninstall(portal_setup):
"""Runs after the last import step of the *uninstall* profile
This handler is registered as a *post_handler* in the generic setup profile
:param portal_setup: SetupTool
"""
logger.info("{} uninstall handler [BEGIN]".format(PRODUCT_NAME.upper()))
# https://docs.plone.org/develop/addons/components/genericsetup.html#custom-installer-code-setuphandlers-py
context = portal_setup._getImportContext(UNINSTALL_PROFILE_ID) # noqa
portal = context.getSite() # noqa
# Uninstall Queue's PAS plugin, from both Zope's acl_users and site's
uninstall_pas_plugin(portal.getPhysicalRoot())
uninstall_pas_plugin(portal)
logger.info("{} uninstall handler [DONE]".format(PRODUCT_NAME.upper()))
def upgrade(tool):
portal = tool.aq_inner.aq_parent
setup = portal.portal_setup
ut = UpgradeUtils(portal)
ver_from = ut.getInstalledVersion(PRODUCT_NAME)
if ut.isOlderVersion(PRODUCT_NAME, version):
logger.info("Skipping upgrade of {0}: {1} > {2}".format(
PRODUCT_NAME, ver_from, version))
return True
logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from,
version))
# -------- ADD YOUR STUFF BELOW --------
# https://github.com/senaite/senaite.queue/pull/3
setup.runImportStepFromProfile(PROFILE_ID, "plone.app.registry")
setup.runImportStepFromProfile(PROFILE_ID, "actions")
# Remove queue dispatcher utility, that is no longer used
setup.runImportStepFromProfile(PROFILE_ID, "componentregistry")
remove_queue_dispatcher_utility(portal)
logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version))
return True
def upgrade(tool):
portal = tool.aq_inner.aq_parent
setup = portal.portal_setup
ut = UpgradeUtils(portal)
ver_from = ut.getInstalledVersion(PRODUCT_NAME)
if ut.isOlderVersion(PRODUCT_NAME, version):
logger.info("Skipping upgrade of {0}: {1} > {2}".format(
PRODUCT_NAME, ver_from, version))
return True
logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from, version))
# Re-import the registry profile to add new settings in control panel
setup.runImportStepFromProfile(PROFILE_ID, "plone.app.registry")
# Reset control panel settings to defaults
reset_settings(portal)
# Port old storage mechanism
remove_legacy_storage(portal)
# Install the PAS Plugin to allow authenticating tasks as their creators
setup_pas_plugin(portal)
# Create and store the key to use for auth
reset_auth_key(portal)
logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version))
return True
def _add(self, task):
# Only QueueTask type is supported
if not is_task(task):
raise ValueError("{} is not supported".format(repr(task)))
# Don't add to the queue if the task is already in there
if task in self._tasks:
logger.warn("Task {} ({}) in the queue already".format(
task.name, task.task_short_uid))
return None
# Do not add the task if unique and task for same context and name
if task.get("unique", False):
query = {"context_uid": task.context_uid, "name": task.name}
if self.search(query):
logger.debug("Task {} for {} in the queue already".format(
task.name, task.context_path))
return None
# Update task status and append to the list of tasks
task.update({"status": "queued"})
self._tasks.append(task)
# Sort by priority + created reverse
# We multiply the priority for 300 sec. (5 minutes) and then we sum the
# result to the time the task was created. This way, we ensure tasks
# priority at the same time we guarantee older, with low priority
# tasks don't fall through the cracks.
# TODO: Make this 300 sec. configurable?
self._tasks.sort(key=lambda t: (t.created + (300 * t.priority)))
# Update the since time
if self._since_time < 0 or self._since_time > task.created:
self._since_time = task.created
logger.info("Added task {} ({}): {}".format(task.name,
task.task_short_uid,
task.context_path))
return task
def get_list_summary(items, endpoint, **kwargs):
"""Returns a dict that represents a summary of a list response
:param items: items to be included in the response
:param endpoint: endpoint from the request
:param kwargs: additional (hashable) params to be included in the message
:return: dict with the summary and the list of items
"""
items = items or []
if not isinstance(items, (list, tuple)):
items = [items]
# Remove empties
items = filter(None, items)
zeo = get_post_zeo()
logger.info("::{}: {} items [{}]".format(endpoint, len(items), zeo))
info = kwargs or {}
info.update({
"count": len(items),
"items": items,
"url": japi.url_for("senaite.queue.{}".format(endpoint)),
"zeo": zeo,
})
return info
def upgrade(tool):
portal = tool.aq_inner.aq_parent
setup = portal.portal_setup
ut = UpgradeUtils(portal)
ver_from = ut.getInstalledVersion(PRODUCT_NAME)
if ut.isOlderVersion(PRODUCT_NAME, version):
logger.info("Skipping upgrade of {0}: {1} > {2}".format(
PRODUCT_NAME, ver_from, version))
return True
logger.info("Upgrading {0}: {1} -> {2}".format(PRODUCT_NAME, ver_from,
version))
logger.info("{0} upgraded to version {1}".format(PRODUCT_NAME, version))
return True
def consume_task():
"""Consumes a task from the queue, if any
"""
if not is_installed():
return info("Queue is not installed")
host = _api.get_request().get("SERVER_URL")
if not is_valid_zeo_host(host):
return error("zeo host not set or not valid: {} [SKIP]".format(host))
consumer_thread = get_consumer_thread()
if consumer_thread:
# There is a consumer working already
name = consumer_thread.getName()
return info("Consumer running: {} [SKIP]".format(name))
logger.info("Queue client: {}".format(host))
# Server's queue URL
server = api.get_server_url()
# Check the status of the queue
status = api.get_queue_status()
if status not in ["resuming", "ready"]:
return warn("Server is {} ({}) [SKIP]".format(status, server))
if api.is_queue_server():
message = [
"Server = Consumer: {}".format(server),
"*******************************************************",
"Client configured as both queue server and consumer.",
"This is not suitable for productive environments!",
"Change the Queue Server URL in SENAITE's control panel",
"or setup another zeo client as queue consumer.",
"Current URL: {}".format(server),
"*******************************************************"
]
logger.warn("\n".join(message))
# Pop next task to process
consumer_id = host
try:
task = api.get_queue().pop(consumer_id)
if not task:
return info("Queue is empty or process undergoing [SKIP]")
except Exception as e:
return error("Cannot pop. {}: {}".format(type(e).__name__, str(e)))
auth_key = _api.get_registry_record("senaite.queue.auth_key")
kwargs = {
"task_uid": task.task_uid,
"task_username": task.username,
"consumer_id": consumer_id,
"base_url": _api.get_url(_api.get_portal()),
"server_url": api.get_server_url(),
"user_id": _api.get_current_user().id,
"max_seconds": get_max_seconds(),
"auth_key": auth_key,
}
name = "{}{}".format(CONSUMER_THREAD_PREFIX, int(time.time()))
t = threading.Thread(name=name, target=process_task, kwargs=kwargs)
t.start()
return info("Consumer running: {} [SKIP]".format(CONSUMER_THREAD_PREFIX))