def main(): parser = argparse.ArgumentParser(description='Runs a task') cfg = config.get_config(parser=parser, config_type="run") cfg["is_cli"] = True set_current_config(cfg) log.info(cfg) if len(cfg["taskargs"]) == 1: params = json.loads(cfg["taskargs"][0]) else: params = {} # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"} for group in utils.group_iter(cfg["taskargs"], n=2): if len(group) != 2: print "Number of arguments wasn't even" sys.exit(1) params[group[0]] = group[1] if cfg["async"]: ret = queue.send_task(cfg["taskpath"], params, sync=False, queue=cfg["queue"]) print ret else: worker_class = load_class_by_path(cfg["worker_class"]) job = worker_class.job_class(None) job.data = { "path": cfg["taskpath"], "params": params, "queue": cfg["queue"] } job.datestarted = datetime.datetime.utcnow() set_current_job(job) ret = job.perform() print json.dumps(ret)
def main(): parser = argparse.ArgumentParser(description='Runs a task') cfg = config.get_config(parser=parser, config_type="run", sources=("file", "env", "args")) cfg["is_cli"] = True set_current_config(cfg) if len(cfg["taskargs"]) == 1: params = json.loads(cfg["taskargs"][0]) # pylint: disable=no-member else: params = {} # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"} for group in utils.group_iter(cfg["taskargs"], n=2): if len(group) != 2: print("Number of arguments wasn't even") sys.exit(1) params[group[0]] = group[1] if cfg["queue"]: ret = queue_job(cfg["taskpath"], params, queue=cfg["queue"]) print(ret) else: worker_class = load_class_by_path(cfg["worker_class"]) job = worker_class.job_class(None) job.set_data({ "path": cfg["taskpath"], "params": params, "queue": cfg["queue"] }) job.datestarted = datetime.datetime.utcnow() set_current_job(job) ret = job.perform() print(json_stdlib.dumps(ret, cls=MongoJSONEncoder)) # pylint: disable=no-member
def perform_action(self, action, query, destination_queue): stats = {"requeued": 0, "cancelled": 0} if action == "cancel": default_job_timeout = get_current_config()["default_job_timeout"] # Finding the ttl here to expire is a bit hard because we may have mixed paths # and hence mixed ttls. # If we are cancelling by path, get this ttl if query.get("path"): result_ttl = get_task_cfg(query["path"]).get( "result_ttl", default_job_timeout) # If not, get the maxmimum ttl of all tasks. else: tasks_defs = get_current_config().get("tasks", {}) tasks_ttls = [ cfg.get("result_ttl", 0) for cfg in tasks_defs.values() ] result_ttl = max([default_job_timeout] + tasks_ttls) now = datetime.datetime.utcnow() ret = self.collection.update(query, { "$set": { "status": "cancel", "dateexpires": now + datetime.timedelta(seconds=result_ttl), "dateupdated": now } }, multi=True) stats["cancelled"] = ret["n"] # Special case when emptying just by queue name: empty it directly! # In this case we could also loose some jobs that were queued after # the MongoDB update. They will be "lost" and requeued later like the other case # after the Redis BLPOP if query.keys() == ["queue"]: Queue(query["queue"]).empty() elif action in ("requeue", "requeue_retry"): # Requeue task by groups of maximum 1k items (if all in the same # queue) cursor = self.collection.find(query, projection=["_id", "queue"]) # We must freeze the list because queries below would change it. # This could not fit in memory, research adding {"stats": {"$ne": # "queued"}} in the query fetched_jobs = list(cursor) for jobs in group_iter(fetched_jobs, n=1000): jobs_by_queue = defaultdict(list) for job in jobs: jobs_by_queue[job["queue"]].append(job["_id"]) stats["requeued"] += 1 for queue in jobs_by_queue: updates = { "status": "queued", "dateupdated": datetime.datetime.utcnow() } if destination_queue is not None: updates["queue"] = destination_queue if action == "requeue": updates["retry_count"] = 0 self.collection.update( {"_id": { "$in": jobs_by_queue[queue] }}, {"$set": updates}, multi=True) # Between these two lines, jobs can become "lost" too. Queue(destination_queue or queue).enqueue_job_ids( [str(x) for x in jobs_by_queue[queue]]) print stats return stats
def perform_action(self, action, query, destination_queue): stats = { "requeued": 0, "cancelled": 0 } if action == "cancel": default_job_timeout = get_current_config()["default_job_timeout"] # Finding the ttl here to expire is a bit hard because we may have mixed paths # and hence mixed ttls. # If we are cancelling by path, get this ttl if query.get("path"): result_ttl = get_task_cfg(query["path"]).get("result_ttl", default_job_timeout) # If not, get the maxmimum ttl of all tasks. else: tasks_defs = get_current_config().get("tasks", {}) tasks_ttls = [cfg.get("result_ttl", 0) for cfg in tasks_defs.values()] result_ttl = max([default_job_timeout] + tasks_ttls) now = datetime.datetime.utcnow() ret = self.collection.update(query, {"$set": { "status": "cancel", "dateexpires": now + datetime.timedelta(seconds=result_ttl), "dateupdated": now }}, multi=True) stats["cancelled"] = ret["n"] # Special case when emptying just by queue name: empty it directly! # In this case we could also loose some jobs that were queued after # the MongoDB update. They will be "lost" and requeued later like the other case # after the Redis BLPOP if query.keys() == ["queue"]: Queue(query["queue"]).empty() elif action in ("requeue", "requeue_retry"): # Requeue task by groups of maximum 1k items (if all in the same # queue) cursor = self.collection.find(query, projection=["_id", "queue"]) # We must freeze the list because queries below would change it. # This could not fit in memory, research adding {"stats": {"$ne": # "queued"}} in the query fetched_jobs = list(cursor) for jobs in group_iter(fetched_jobs, n=1000): jobs_by_queue = defaultdict(list) for job in jobs: jobs_by_queue[job["queue"]].append(job["_id"]) stats["requeued"] += 1 for queue in jobs_by_queue: updates = { "status": "queued", "dateupdated": datetime.datetime.utcnow() } if destination_queue is not None: updates["queue"] = destination_queue if action == "requeue": updates["retry_count"] = 0 self.collection.update({ "_id": {"$in": jobs_by_queue[queue]} }, {"$set": updates}, multi=True) # Between these two lines, jobs can become "lost" too. Queue(destination_queue or queue, add_to_known_queues=True).enqueue_job_ids( [str(x) for x in jobs_by_queue[queue]]) print stats return stats
def perform_action(self, action, query, destination_queue): stats = {"requeued": 0, "cancelled": 0} if action == "cancel": default_job_timeout = get_current_config()["default_job_timeout"] # Finding the ttl here to expire is a bit hard because we may have mixed paths # and hence mixed ttls. # If we are cancelling by path, get this ttl if query.get("path"): result_ttl = get_task_cfg(query["path"]).get( "result_ttl", default_job_timeout) # If not, get the maxmimum ttl of all tasks. else: tasks_defs = get_current_config().get("tasks", {}) tasks_ttls = [ cfg.get("result_ttl", 0) for cfg in itervalues(tasks_defs) ] result_ttl = max([default_job_timeout] + tasks_ttls) now = datetime.datetime.utcnow() size_by_queues = defaultdict(int) if "queue" not in query: for job in self.collection.find(query, projection={"queue": 1}): size_by_queues[job["queue"]] += 1 ret = self.collection.update(query, { "$set": { "status": "cancel", "dateexpires": now + datetime.timedelta(seconds=result_ttl), "dateupdated": now } }, multi=True) stats["cancelled"] = ret["n"] if "queue" in query: if isinstance(query["queue"], str): size_by_queues[query["queue"]] = ret["n"] set_queues_size(size_by_queues, action="decr") # Special case when emptying just by queue name: empty it directly! # In this case we could also loose some jobs that were queued after # the MongoDB update. They will be "lost" and requeued later like the other case # after the Redis BLPOP if list(query.keys()) == ["queue"] and isinstance( query["queue"], basestring): Queue(query["queue"]).empty() elif action in ("requeue", "requeue_retry"): # Requeue task by groups of maximum 1k items (if all in the same # queue) status_query = query.get("status") if not status_query: query["status"] = {"$ne": "queued"} cursor = self.collection.find(query, projection=["_id", "queue"]) for jobs in group_iter(cursor, n=1000): jobs_by_queue = defaultdict(list) for job in jobs: jobs_by_queue[job["queue"]].append(job["_id"]) stats["requeued"] += 1 for queue in jobs_by_queue: updates = { "status": "queued", "datequeued": datetime.datetime.utcnow(), "dateupdated": datetime.datetime.utcnow() } if destination_queue is not None: updates["queue"] = destination_queue if action == "requeue": updates["retry_count"] = 0 self.collection.update( {"_id": { "$in": jobs_by_queue[queue] }}, {"$set": updates}, multi=True) set_queues_size({ queue: len(jobs) for queue, jobs in jobs_by_queue.items() }) return stats