def tasks(context, request, status=None): # noqa """Returns a JSON representation of the tasks from the queue """ # Maybe the status has been sent via POST request_data = req.get_json() status = status or request_data.get("status", []) since = request_data.get("since", 0) # Get the tasks items = qapi.get_queue().get_tasks(status) # Skip ghosts unless explicitly asked if "ghost" not in status: items = filter(lambda t: not t.get("ghost"), items) # Skip older items = filter(lambda t: t.created > since, items) # Convert to the dict representation complete = request_data.get("complete") or False summary = get_tasks_summary(list(items), "server.tasks", complete=complete) # Update the summary with the created time of oldest task summary.update({"since_time": qapi.get_queue().get_since_time()}) return summary
def delete(context, request): # noqa """Removes the task from the queue """ # Get the task uid task_uid = req.get_json().get("task_uid") # Get the task task = get_task(task_uid) qapi.get_queue().delete(task.task_uid) # Return the process summary msg = "Task deleted: {}".format(task_uid) task_info = {"task": get_task_info(task)} return get_message_summary(msg, "server.delete", **task_info)
def process(browser, task_uid): """Simulates the processing of the task """ request = _api.get_request() site_url = _api.get_url(_api.get_portal()) url = "{}/@@API/senaite/v1/queue_consumer/process".format(site_url) payload = {"task_uid": task_uid} browser.post(url, parse.urlencode(payload, doseq=True)) # We loose the globalrequest each time we do a post with browser globalrequest.setRequest(request) # Mark the task as done api.get_queue().done(task_uid) transaction.commit() return browser.contents
def get_num_pending(self): if not api.is_queue_enabled(): return 0 # We are only interested in tasks with uids queue = api.get_queue() uids = map(lambda t: t.get("uids"), queue.get_tasks_for(self.context)) uids = filter(None, list(itertools.chain.from_iterable(uids))) return len(set(uids))
def __call__(self, action, uids): """Removes the selected tasks and redirects to the previous URL """ queue = qapi.get_queue() map(queue.delete, uids) url = api.get_url(api.get_portal()) url = "{}/queue_tasks".format(url) return self.redirect(url)
def timeout(context, request): # noqa """The task timed out """ # Get the task uid request_data = req.get_json() task_uid = request_data.get("task_uid") # Get the task task = get_task(task_uid) if task.status not in [ "running", ]: _fail(412, "Task is not running") # Notify the queue qapi.get_queue().timeout(task) # Return the process summary task_info = {"task": get_task_info(task)} return get_message_summary(task_uid, "server.timeout", **task_info)
def process(self, task): """Process the task from the queue """ # If there are too many objects to process, split them in chunks to # prevent the task to take too much time to complete chunks = get_chunks(task["uids"], 50) # Process the first chunk map(self.reindex_security, chunks[0]) # Add remaining objects to the queue if chunks[1]: request = _api.get_request() context = task.get_context() kwargs = { "uids": chunks[1], "priority": task.priority, } new_task = QueueTask(task.name, request, context, **kwargs) api.get_queue().add(new_task)
def uids(context, request, status=None): # noqa """Returns a JSON representation of the uids from queued objects """ # Maybe the status has been sent via POST request_data = req.get_json() status = status or request_data.get("status") # Get the uids from queued objects items = qapi.get_queue().get_uids(status) # Convert to the dict representation return get_list_summary(items, "server.uids")
def done(context, request): # noqa """Acknowledge the task has been successfully processed. Task is removed from the running tasks pool and returned """ # Get the task uid task_uid = req.get_json().get("task_uid") # Get the task task = get_task(task_uid) if task.status not in [ "running", ]: _fail(412, "Task is not running") # Notify the queue qapi.get_queue().done(task) # Return the process summary msg = "Task done: {}".format(task_uid) task_info = {"task": get_task_info(task)} return get_message_summary(msg, "server.done", **task_info)
def get_task(task_uid): """Resolves the task for the given task uid """ if not api.is_uid(task_uid) or task_uid == "0": # 400 Bad Request, wrong task uid _fail(412, "Task uid empty or no valid format") task = qapi.get_queue().get_task(task_uid) if not task: _fail(404, "Task {}".format(task_uid)) return task
def __call__(self, action, uids): """Re-queues the selected tasks and redirects to the previous URL """ queue = qapi.get_queue() for uid in uids: task = queue.get_task(uid) task.retries = get_max_retries() queue.delete(uid) queue.add(task) url = api.get_url(api.get_portal()) url = "{}/queue_tasks".format(url) return self.redirect(url)
def fail(context, request): # noqa """Acknowledge the task has NOT been successfully processed. Task is moved from running tasks to failed or re-queued and returned """ # Get the task uid request_data = req.get_json() task_uid = request_data.get("task_uid") error_message = request_data.get("error_message") # Get the task task = get_task(task_uid) if task.status not in [ "running", ]: _fail(412, "Task is not running") # Notify the queue qapi.get_queue().fail(task, error_message=error_message) # Return the process summary msg = "Task failed: {}".format(task_uid) task_info = {"task": get_task_info(task)} return get_message_summary(msg, "server.fail", **task_info)
def get_task(task_uid): """Resolves the task for the given task uid """ if not capi.is_uid(task_uid) or task_uid == "0": # 400 Bad Request, wrong task uid _fail(412, "Task uid empty or no valid format") task = api.get_queue().get_task(task_uid) if not task: _fail(404, "Task {}".format(task_uid)) if not capi.is_uid(task.context_uid): _fail(500, "Task's context uid is not valid") return task
def pop(context, request): # noqa """Pops the next task from the queue, if any. Popped task is no longer available in the queued tasks pool, but added in the running tasks pool """ # Get the consumer ID consumer_id = req.get_json().get("consumer_id") if not is_consumer_id(consumer_id): _fail(428, "No valid consumer id") # Pop the task from the queue task = qapi.get_queue().pop(consumer_id) # Return the task info task_uid = get_task_uid(task, default="<empty>") logger.info("::server.pop: {} [{}]".format(task_uid, consumer_id)) return get_task_info(task, complete=True)
def search(context, request): # noqa """Performs a search """ # Get the search criteria query = req.get_request_data() if isinstance(query, (list, tuple)): query = query[0] elif not isinstance(query, dict): query = {} uid = query.get("uid") name = query.get("name") complete = query.get("complete", False) # Get the tasks from the utility items = qapi.get_queue().get_tasks_for(uid, name=name) return get_tasks_summary(items, "server.search", complete=complete)
def add(context, request): # noqa """Adds a new task to the queue server """ # Extract the task(s) from the request raw_tasks = req.get_request_data() # Convert raw task(s) to QueueTask object(s) items = map(to_task, raw_tasks) valid = map(is_task, items) if not all(valid): _fail(406, "No valid task(s)") # Add the task(s) to the queue map(qapi.get_queue().add, items) # Return the process summary return get_tasks_summary(items, "server.add", complete=False)
def requeue(context, request, task_uid=None): # noqa """Requeue the task. Task is moved from either failed or running pool to the queued tasks pool and returned """ # Maybe the task uid has been sent via POST task_uid = task_uid or req.get_json().get("task_uid") # Get the task task = get_task(task_uid) # Remove, restore max number of retries and re-add the task task.retries = get_max_retries() queue = qapi.get_queue() queue.delete(task_uid) queue.add(task) # Return the process summary msg = "Task re-queued: {}".format(task_uid) task_info = {"task": get_task_info(task)} return get_message_summary(msg, "server.requeue", **task_info)
def diff(context, request): request_data = req.get_json() status = request_data.get("status", []) client_uids = request_data.get("uids", []) client_uids = filter(api.is_uid, client_uids) # Get the tasks items = qapi.get_queue().get_tasks(status) # Keep track of the uids the client has to remove server_uids = map(lambda t: t.task_uid, items) stale_uids = filter(lambda uid: uid not in server_uids, client_uids) def keep(task): # Skip ghosts unless explicitly asked if "ghost" not in status: if task.get("ghost"): return False # Always include tasks that are running (client might have this task # already, but in queued status) if task.status in "running": return True # Skip the task if client has it return task.task_uid not in client_uids # Keep the tasks that matter items = filter(keep, items) # Convert to the dict representation complete = request_data.get("complete") or False summary = get_tasks_summary(list(items), "server.diff", complete=complete) # Update the summary with the uids the client has to remove summary.update({ "stale": stale_uids, # TODO Implement unknowns "unknown": [] }) return summary
def handle_submit(self): """Handle form submission for the assignment of a WorksheetTemplate """ wst_uid = self.request.form.get("getWorksheetTemplate") if not wst_uid: return False # Do not allow the assignment of a worksheet template when queued if api.is_queued(self.context): return False # Current context is the worksheet worksheet = self.context layout = worksheet.getLayout() # XXX For what is this used? self.request["context_uid"] = capi.get_uid(worksheet) # Apply the worksheet template to this worksheet wst = capi.get_object_by_uid(wst_uid) worksheet.applyWorksheetTemplate(wst) # Are there tasks queued for this Worksheet? if api.is_queue_enabled(): queue = api.get_queue() tasks = queue.get_tasks_for(worksheet) if tasks: return True # Maybe the queue has not been used (disabled?) new_layout = worksheet.getLayout() if len(new_layout) != len(layout): # Layout has changed. Assume the analyses were added return True return False
def folderitems(self): states_map = { "running": "state-published", "failed": "state-retracted", "queued": "state-active", "ghost": "state-unassigned", } # flag for manual sorting self.manual_sort_on = self.get_sort_on() # Get the items status = ["running", "queued"] if self.review_state.get("id") == "failed": status = ["failed"] elif self.review_state.get("id") == "all": status = ["running", "queued", "failed", "ghost"] items = map(self.make_item, qapi.get_queue().get_tasks(status=status)) # Infere the priorities site_url = api.get_url(api.get_portal()) api_url = "{}/@@API/v1/@@API/senaite/v1/queue_server".format(site_url) idx = 1 for item in items: if item["status"] not in ["queued", "running"]: priority = 0 else: priority = idx idx += 1 created = datetime.fromtimestamp(int(item["created"])).isoformat() context_link = get_link(item["context_path"], item["context_path"]) task_link = "{}/{}".format(api_url, item["uid"]) params = {"class": "text-monospace"} task_link = get_link(task_link, item["task_short_uid"], **params) status_msg = _(item["status"]) css_class = states_map.get(item["status"]) if item.get("ghost"): css_class = "{} {}".format(css_class, states_map["ghost"]) status_msg = "{} ({})".format(status_msg, _("ghost")) item.update({ "priority": str(priority).zfill(4), "state_class": css_class, "replace": { "status": status_msg, "context_path": context_link, "task_short_uid": task_link, "created": created, } }) # Sort the items sort_on = self.manual_sort_on in self.columns.keys() or "priority" reverse = self.get_sort_order() == "ascending" items = sorted(items, key=itemgetter(sort_on), reverse=reverse) # Pagination self.total = len(items) limit_from = self.get_limit_from() if limit_from and len(items) > limit_from: return items[limit_from:self.pagesize + limit_from] return items[:self.pagesize]
def consume_task(): """Consumes a task from the queue, if any """ if not is_installed(): return info("Queue is not installed") host = _api.get_request().get("SERVER_URL") if not is_valid_zeo_host(host): return error("zeo host not set or not valid: {} [SKIP]".format(host)) consumer_thread = get_consumer_thread() if consumer_thread: # There is a consumer working already name = consumer_thread.getName() return info("Consumer running: {} [SKIP]".format(name)) logger.info("Queue client: {}".format(host)) # Server's queue URL server = api.get_server_url() # Check the status of the queue status = api.get_queue_status() if status not in ["resuming", "ready"]: return warn("Server is {} ({}) [SKIP]".format(status, server)) if api.is_queue_server(): message = [ "Server = Consumer: {}".format(server), "*******************************************************", "Client configured as both queue server and consumer.", "This is not suitable for productive environments!", "Change the Queue Server URL in SENAITE's control panel", "or setup another zeo client as queue consumer.", "Current URL: {}".format(server), "*******************************************************" ] logger.warn("\n".join(message)) # Pop next task to process consumer_id = host try: task = api.get_queue().pop(consumer_id) if not task: return info("Queue is empty or process undergoing [SKIP]") except Exception as e: return error("Cannot pop. {}: {}".format(type(e).__name__, str(e))) auth_key = _api.get_registry_record("senaite.queue.auth_key") kwargs = { "task_uid": task.task_uid, "task_username": task.username, "consumer_id": consumer_id, "base_url": _api.get_url(_api.get_portal()), "server_url": api.get_server_url(), "user_id": _api.get_current_user().id, "max_seconds": get_max_seconds(), "auth_key": auth_key, } name = "{}{}".format(CONSUMER_THREAD_PREFIX, int(time.time())) t = threading.Thread(name=name, target=process_task, kwargs=kwargs) t.start() return info("Consumer running: {} [SKIP]".format(CONSUMER_THREAD_PREFIX))