def tasks(context, request, status=None): # noqa
"""Returns a JSON representation of the tasks from the queue
"""
# Maybe the status has been sent via POST
request_data = req.get_json()
status = status or request_data.get("status", [])
since = request_data.get("since", 0)
# Get the tasks
items = qapi.get_queue().get_tasks(status)
# Skip ghosts unless explicitly asked
if "ghost" not in status:
items = filter(lambda t: not t.get("ghost"), items)
# Skip older
items = filter(lambda t: t.created > since, items)
# Convert to the dict representation
complete = request_data.get("complete") or False
summary = get_tasks_summary(list(items), "server.tasks", complete=complete)
# Update the summary with the created time of oldest task
summary.update({"since_time": qapi.get_queue().get_since_time()})
return summary
def delete(context, request): # noqa
"""Removes the task from the queue
"""
# Get the task uid
task_uid = req.get_json().get("task_uid")
# Get the task
task = get_task(task_uid)
qapi.get_queue().delete(task.task_uid)
# Return the process summary
msg = "Task deleted: {}".format(task_uid)
task_info = {"task": get_task_info(task)}
return get_message_summary(msg, "server.delete", **task_info)
def process(browser, task_uid):
"""Simulates the processing of the task
"""
request = _api.get_request()
site_url = _api.get_url(_api.get_portal())
url = "{}/@@API/senaite/v1/queue_consumer/process".format(site_url)
payload = {"task_uid": task_uid}
browser.post(url, parse.urlencode(payload, doseq=True))
# We loose the globalrequest each time we do a post with browser
globalrequest.setRequest(request)
# Mark the task as done
api.get_queue().done(task_uid)
transaction.commit()
return browser.contents
def get_num_pending(self):
if not api.is_queue_enabled():
return 0
# We are only interested in tasks with uids
queue = api.get_queue()
uids = map(lambda t: t.get("uids"), queue.get_tasks_for(self.context))
uids = filter(None, list(itertools.chain.from_iterable(uids)))
return len(set(uids))
def __call__(self, action, uids):
"""Removes the selected tasks and redirects to the previous URL
"""
queue = qapi.get_queue()
map(queue.delete, uids)
url = api.get_url(api.get_portal())
url = "{}/queue_tasks".format(url)
return self.redirect(url)
def timeout(context, request): # noqa
"""The task timed out
"""
# Get the task uid
request_data = req.get_json()
task_uid = request_data.get("task_uid")
# Get the task
task = get_task(task_uid)
if task.status not in [
"running",
]:
_fail(412, "Task is not running")
# Notify the queue
qapi.get_queue().timeout(task)
# Return the process summary
task_info = {"task": get_task_info(task)}
return get_message_summary(task_uid, "server.timeout", **task_info)
def process(self, task):
"""Process the task from the queue
"""
# If there are too many objects to process, split them in chunks to
# prevent the task to take too much time to complete
chunks = get_chunks(task["uids"], 50)
# Process the first chunk
map(self.reindex_security, chunks[0])
# Add remaining objects to the queue
if chunks[1]:
request = _api.get_request()
context = task.get_context()
kwargs = {
"uids": chunks[1],
"priority": task.priority,
}
new_task = QueueTask(task.name, request, context, **kwargs)
api.get_queue().add(new_task)
def uids(context, request, status=None): # noqa
"""Returns a JSON representation of the uids from queued objects
"""
# Maybe the status has been sent via POST
request_data = req.get_json()
status = status or request_data.get("status")
# Get the uids from queued objects
items = qapi.get_queue().get_uids(status)
# Convert to the dict representation
return get_list_summary(items, "server.uids")
def done(context, request): # noqa
"""Acknowledge the task has been successfully processed. Task is removed
from the running tasks pool and returned
"""
# Get the task uid
task_uid = req.get_json().get("task_uid")
# Get the task
task = get_task(task_uid)
if task.status not in [
"running",
]:
_fail(412, "Task is not running")
# Notify the queue
qapi.get_queue().done(task)
# Return the process summary
msg = "Task done: {}".format(task_uid)
task_info = {"task": get_task_info(task)}
return get_message_summary(msg, "server.done", **task_info)
def get_task(task_uid):
"""Resolves the task for the given task uid
"""
if not api.is_uid(task_uid) or task_uid == "0":
# 400 Bad Request, wrong task uid
_fail(412, "Task uid empty or no valid format")
task = qapi.get_queue().get_task(task_uid)
if not task:
_fail(404, "Task {}".format(task_uid))
return task
def __call__(self, action, uids):
"""Re-queues the selected tasks and redirects to the previous URL
"""
queue = qapi.get_queue()
for uid in uids:
task = queue.get_task(uid)
task.retries = get_max_retries()
queue.delete(uid)
queue.add(task)
url = api.get_url(api.get_portal())
url = "{}/queue_tasks".format(url)
return self.redirect(url)
def fail(context, request): # noqa
"""Acknowledge the task has NOT been successfully processed. Task is
moved from running tasks to failed or re-queued and returned
"""
# Get the task uid
request_data = req.get_json()
task_uid = request_data.get("task_uid")
error_message = request_data.get("error_message")
# Get the task
task = get_task(task_uid)
if task.status not in [
"running",
]:
_fail(412, "Task is not running")
# Notify the queue
qapi.get_queue().fail(task, error_message=error_message)
# Return the process summary
msg = "Task failed: {}".format(task_uid)
task_info = {"task": get_task_info(task)}
return get_message_summary(msg, "server.fail", **task_info)
def get_task(task_uid):
"""Resolves the task for the given task uid
"""
if not capi.is_uid(task_uid) or task_uid == "0":
# 400 Bad Request, wrong task uid
_fail(412, "Task uid empty or no valid format")
task = api.get_queue().get_task(task_uid)
if not task:
_fail(404, "Task {}".format(task_uid))
if not capi.is_uid(task.context_uid):
_fail(500, "Task's context uid is not valid")
return task
def pop(context, request): # noqa
"""Pops the next task from the queue, if any. Popped task is no longer
available in the queued tasks pool, but added in the running tasks pool
"""
# Get the consumer ID
consumer_id = req.get_json().get("consumer_id")
if not is_consumer_id(consumer_id):
_fail(428, "No valid consumer id")
# Pop the task from the queue
task = qapi.get_queue().pop(consumer_id)
# Return the task info
task_uid = get_task_uid(task, default="<empty>")
logger.info("::server.pop: {} [{}]".format(task_uid, consumer_id))
return get_task_info(task, complete=True)
def search(context, request): # noqa
"""Performs a search
"""
# Get the search criteria
query = req.get_request_data()
if isinstance(query, (list, tuple)):
query = query[0]
elif not isinstance(query, dict):
query = {}
uid = query.get("uid")
name = query.get("name")
complete = query.get("complete", False)
# Get the tasks from the utility
items = qapi.get_queue().get_tasks_for(uid, name=name)
return get_tasks_summary(items, "server.search", complete=complete)
def add(context, request): # noqa
"""Adds a new task to the queue server
"""
# Extract the task(s) from the request
raw_tasks = req.get_request_data()
# Convert raw task(s) to QueueTask object(s)
items = map(to_task, raw_tasks)
valid = map(is_task, items)
if not all(valid):
_fail(406, "No valid task(s)")
# Add the task(s) to the queue
map(qapi.get_queue().add, items)
# Return the process summary
return get_tasks_summary(items, "server.add", complete=False)
def requeue(context, request, task_uid=None): # noqa
"""Requeue the task. Task is moved from either failed or running pool to
the queued tasks pool and returned
"""
# Maybe the task uid has been sent via POST
task_uid = task_uid or req.get_json().get("task_uid")
# Get the task
task = get_task(task_uid)
# Remove, restore max number of retries and re-add the task
task.retries = get_max_retries()
queue = qapi.get_queue()
queue.delete(task_uid)
queue.add(task)
# Return the process summary
msg = "Task re-queued: {}".format(task_uid)
task_info = {"task": get_task_info(task)}
return get_message_summary(msg, "server.requeue", **task_info)
def diff(context, request):
request_data = req.get_json()
status = request_data.get("status", [])
client_uids = request_data.get("uids", [])
client_uids = filter(api.is_uid, client_uids)
# Get the tasks
items = qapi.get_queue().get_tasks(status)
# Keep track of the uids the client has to remove
server_uids = map(lambda t: t.task_uid, items)
stale_uids = filter(lambda uid: uid not in server_uids, client_uids)
def keep(task):
# Skip ghosts unless explicitly asked
if "ghost" not in status:
if task.get("ghost"):
return False
# Always include tasks that are running (client might have this task
# already, but in queued status)
if task.status in "running":
return True
# Skip the task if client has it
return task.task_uid not in client_uids
# Keep the tasks that matter
items = filter(keep, items)
# Convert to the dict representation
complete = request_data.get("complete") or False
summary = get_tasks_summary(list(items), "server.diff", complete=complete)
# Update the summary with the uids the client has to remove
summary.update({
"stale": stale_uids,
# TODO Implement unknowns
"unknown": []
})
return summary
def handle_submit(self):
"""Handle form submission for the assignment of a WorksheetTemplate
"""
wst_uid = self.request.form.get("getWorksheetTemplate")
if not wst_uid:
return False
# Do not allow the assignment of a worksheet template when queued
if api.is_queued(self.context):
return False
# Current context is the worksheet
worksheet = self.context
layout = worksheet.getLayout()
# XXX For what is this used?
self.request["context_uid"] = capi.get_uid(worksheet)
# Apply the worksheet template to this worksheet
wst = capi.get_object_by_uid(wst_uid)
worksheet.applyWorksheetTemplate(wst)
# Are there tasks queued for this Worksheet?
if api.is_queue_enabled():
queue = api.get_queue()
tasks = queue.get_tasks_for(worksheet)
if tasks:
return True
# Maybe the queue has not been used (disabled?)
new_layout = worksheet.getLayout()
if len(new_layout) != len(layout):
# Layout has changed. Assume the analyses were added
return True
return False
def folderitems(self):
states_map = {
"running": "state-published",
"failed": "state-retracted",
"queued": "state-active",
"ghost": "state-unassigned",
}
# flag for manual sorting
self.manual_sort_on = self.get_sort_on()
# Get the items
status = ["running", "queued"]
if self.review_state.get("id") == "failed":
status = ["failed"]
elif self.review_state.get("id") == "all":
status = ["running", "queued", "failed", "ghost"]
items = map(self.make_item, qapi.get_queue().get_tasks(status=status))
# Infere the priorities
site_url = api.get_url(api.get_portal())
api_url = "{}/@@API/v1/@@API/senaite/v1/queue_server".format(site_url)
idx = 1
for item in items:
if item["status"] not in ["queued", "running"]:
priority = 0
else:
priority = idx
idx += 1
created = datetime.fromtimestamp(int(item["created"])).isoformat()
context_link = get_link(item["context_path"], item["context_path"])
task_link = "{}/{}".format(api_url, item["uid"])
params = {"class": "text-monospace"}
task_link = get_link(task_link, item["task_short_uid"], **params)
status_msg = _(item["status"])
css_class = states_map.get(item["status"])
if item.get("ghost"):
css_class = "{} {}".format(css_class, states_map["ghost"])
status_msg = "{} ({})".format(status_msg, _("ghost"))
item.update({
"priority": str(priority).zfill(4),
"state_class": css_class,
"replace": {
"status": status_msg,
"context_path": context_link,
"task_short_uid": task_link,
"created": created,
}
})
# Sort the items
sort_on = self.manual_sort_on in self.columns.keys() or "priority"
reverse = self.get_sort_order() == "ascending"
items = sorted(items, key=itemgetter(sort_on), reverse=reverse)
# Pagination
self.total = len(items)
limit_from = self.get_limit_from()
if limit_from and len(items) > limit_from:
return items[limit_from:self.pagesize + limit_from]
return items[:self.pagesize]
def consume_task():
"""Consumes a task from the queue, if any
"""
if not is_installed():
return info("Queue is not installed")
host = _api.get_request().get("SERVER_URL")
if not is_valid_zeo_host(host):
return error("zeo host not set or not valid: {} [SKIP]".format(host))
consumer_thread = get_consumer_thread()
if consumer_thread:
# There is a consumer working already
name = consumer_thread.getName()
return info("Consumer running: {} [SKIP]".format(name))
logger.info("Queue client: {}".format(host))
# Server's queue URL
server = api.get_server_url()
# Check the status of the queue
status = api.get_queue_status()
if status not in ["resuming", "ready"]:
return warn("Server is {} ({}) [SKIP]".format(status, server))
if api.is_queue_server():
message = [
"Server = Consumer: {}".format(server),
"*******************************************************",
"Client configured as both queue server and consumer.",
"This is not suitable for productive environments!",
"Change the Queue Server URL in SENAITE's control panel",
"or setup another zeo client as queue consumer.",
"Current URL: {}".format(server),
"*******************************************************"
]
logger.warn("\n".join(message))
# Pop next task to process
consumer_id = host
try:
task = api.get_queue().pop(consumer_id)
if not task:
return info("Queue is empty or process undergoing [SKIP]")
except Exception as e:
return error("Cannot pop. {}: {}".format(type(e).__name__, str(e)))
auth_key = _api.get_registry_record("senaite.queue.auth_key")
kwargs = {
"task_uid": task.task_uid,
"task_username": task.username,
"consumer_id": consumer_id,
"base_url": _api.get_url(_api.get_portal()),
"server_url": api.get_server_url(),
"user_id": _api.get_current_user().id,
"max_seconds": get_max_seconds(),
"auth_key": auth_key,
}
name = "{}{}".format(CONSUMER_THREAD_PREFIX, int(time.time()))
t = threading.Thread(name=name, target=process_task, kwargs=kwargs)
t.start()
return info("Consumer running: {} [SKIP]".format(CONSUMER_THREAD_PREFIX))
