def send_raw_tasks(self, queue, params_list, start=True, block=True):
if not self.started and start:
self.start()
send_raw_tasks(queue, params_list)
if block:
# Wait for the queue to be empty. Might be error-prone when tasks are in-memory between the 2
q = Queue(queue)
while q.size() > 0 or self.mongodb_jobs.mrq_jobs.find({"status": "started"}).count() > 0:
print "S", q.size(), self.mongodb_jobs.mrq_jobs.find({"status": "started"}).count()
time.sleep(0.1)
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
pretend = bool(params.get("pretend"))
check_mongo = bool(params.get("check_mongo"))
known_queues = Queue.redis_known_queues()
removed_queues = []
queues_from_config = Queue.all_known_from_config()
print "Found %s known queues & %s from config" % (len(known_queues), len(queues_from_config))
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if queue in queues_from_config:
continue
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
size = q.size()
if check_mongo:
size += connections.mongodb_jobs.mrq_jobs.count({"queue": queue})
if size == 0:
removed_queues.append(queue)
print "Removing empty queue '%s' from known queues ..." % queue
if not pretend:
q.remove_from_known_queues()
print "Cleaned %s queues" % len(removed_queues)
return removed_queues
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
pretend = bool(params.get("pretend"))
check_mongo = bool(params.get("check_mongo"))
known_queues = Queue.redis_known_queues()
removed_queues = []
queues_from_config = Queue.all_known_from_config()
print "Found %s known queues & %s from config" % (
len(known_queues), len(queues_from_config))
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if queue in queues_from_config:
continue
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
size = q.size()
if check_mongo:
size += connections.mongodb_jobs.mrq_jobs.count(
{"queue": queue})
if size == 0:
removed_queues.append(queue)
print "Removing empty queue '%s' from known queues ..." % queue
if not pretend:
q.remove_from_known_queues()
print "Cleaned %s queues" % len(removed_queues)
return removed_queues
def run(self, params):
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
all_queues = Queue.all_known()
for queue_name in all_queues:
queue = Queue(queue_name)
queue_size = queue.size()
if queue.is_raw:
continue
log.info("Checking queue %s" % queue_name)
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" %
(queue_name, queue_size))
continue
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items"
% (queue_name, len(queue_jobs_ids)))
continue
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{
"queue": queue_name,
"status": "queued"
},
projection={
"_id": 1
}).sort([["_id", 1]]):
stats["fetched"] += 1
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Found job %s on queue %s. Stopping" %
(job_data["_id"], queue.id))
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=queue_name)
return stats
def run(self, params):
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
# This was only checking in Redis and wasn't resistant to a redis-wide flush.
# Doing Queue.all() is slower but covers more edge cases.
# all_queues = Queue.all_known()
all_queues = Queue.all()
log.info("Checking %s queues" % len(all_queues))
for queue_name in all_queues:
queue = Queue(queue_name)
queue_size = queue.size()
# If the queue is raw, the jobs were only stored in redis so they are lost for good.
if queue.is_raw:
continue
log.info("Checking queue %s" % queue_name)
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue_name, queue_size))
continue
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items" % (queue_name, len(queue_jobs_ids))
)
continue
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"queue": queue_name, "status": "queued"}, projection={"_id": 1}
).sort([["_id", 1]]):
stats["fetched"] += 1
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Found job %s on queue %s. Stopping" % (job_data["_id"], queue.id))
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=queue_name)
return stats
def run(self, params):
max_age = int(params.get("max_age") or (7 * 86400))
known_queues = Queue.redis_known_queues()
# Only clean queues older than N days
time_threshold = time.time() - max_age
for queue, time_last_used in known_queues.iteritems():
if time_last_used < time_threshold:
q = Queue(queue, add_to_known_queues=False)
if q.size() == 0:
q.remove_from_known_queues()
def run(self, params):
collection = connections.mongodb_jobs.mrq_jobs
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
for job_data in collection.find({
"status": "queued"
},
fields={
"_id": 1,
"queue": 1
}).sort([("_id", 1)]):
stats["fetched"] += 1
queue = Queue(job_data["queue"])
queue_size = queue.size()
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" %
(queue, queue_size))
break
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items"
% (queue, len(queue_jobs_ids)))
break
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Stopping because we found job %s in redis" %
job_data["_id"])
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=job_data["queue"])
return stats
def send_raw_tasks(self, queue, params_list, start=True, block=True):
if not self.started and start:
self.start()
queue_raw_jobs(queue, params_list)
if block:
# Wait for the queue to be empty. Might be error-prone when tasks
# are in-memory between the 2
q = Queue(queue)
while q.size() > 0 or self.mongodb_jobs.mrq_jobs.find({"status": "started"}).count() > 0:
# print "S", q.size(),
# self.mongodb_jobs.mrq_jobs.find({"status":
# "started"}).count()
time.sleep(0.1)
def run(self, params):
self.collection = connections.mongodb_jobs.mrq_jobs
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {
"fetched": 0,
"requeued": 0
}
for job_data in self.collection.find({
"status": "queued"
}, fields={"_id": 1, "queue": 1}).sort([("_id", 1)]):
stats["fetched"] += 1
queue = Queue(job_data["queue"])
queue_size = queue.size()
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue, queue_size))
break
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info("Stopping because queue %s actually had more than %s items" % (queue, len(queue_jobs_ids)))
break
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Stopping because we found job %s in redis" % job_data["_id"])
break
# At this point, this job is not on the queue and we're sure the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=job_data["queue"])
return stats
def api_datatables(unit):
# import time
# time.sleep(5)
collection = None
sort = None
skip = int(request.args.get("iDisplayStart", 0))
limit = int(request.args.get("iDisplayLength", 20))
if unit == "queues":
queues = []
for name in Queue.all_known():
queue = Queue(name)
q = {
"name": name,
"size": queue.size(), # Redis size
"is_sorted": queue.is_sorted,
"is_timed": queue.is_timed,
"is_raw": queue.is_raw,
"is_set": queue.is_set
}
if queue.is_sorted:
raw_config = queue.get_config()
q["graph_config"] = raw_config.get(
"dashboard_graph", lambda: {
"start": time.time() - (7 * 24 * 3600),
"stop": time.time() + (7 * 24 * 3600),
"slices": 30
} if queue.is_timed else {
"start": 0,
"stop": 100,
"slices": 30
})()
if q["graph_config"]:
q["graph"] = queue.get_sorted_graph(**q["graph_config"])
if queue.is_timed:
q["jobs_to_dequeue"] = queue.count_jobs_to_dequeue()
queues.append(q)
queues.sort(key=lambda x: -x["size"])
data = {"aaData": queues, "iTotalDisplayRecords": len(queues)}
elif unit == "workers":
fields = None
collection = connections.mongodb_jobs.mrq_workers
sort = [("datestarted", -1)]
query = {}
if request.args.get("id"):
query["_id"] = ObjectId(request.args["id"])
else:
if request.args.get("status"):
statuses = request.args["status"].split("-")
query["status"] = {"$in": statuses}
if request.args.get("ip"):
query["$or"] = [{
"config.local_ip": request.args["ip"]
}, {
"config.external_ip": request.args["ip"]
}]
if request.args.get("queue"):
query["config.queues"] = request.args["queue"]
elif unit == "agents":
fields = None
query = {"status": {"$nin": ["stop"]}}
collection = connections.mongodb_jobs.mrq_agents
sort = [("datestarted", -1)]
if request.args.get("showstopped"):
query = {}
elif unit == "scheduled_jobs":
collection = connections.mongodb_jobs.mrq_scheduled_jobs
fields = None
query = {}
elif unit == "jobs":
fields = None
query = build_api_datatables_query(request)
sort = None # TODO [("_id", 1)]
# We can't search easily params because we store it as decoded JSON in mongo :(
# Add a string index?
# if request.args.get("sSearch"):
# query.update(json.loads(request.args.get("sSearch")))
collection = connections.mongodb_jobs.mrq_jobs
if collection is not None:
cursor = collection.find(query, projection=fields)
if sort:
cursor.sort(sort)
if skip is not None:
cursor.skip(skip)
if limit is not None:
cursor.limit(limit)
data = {
"aaData": list(cursor),
"iTotalDisplayRecords": collection.find(query).count()
}
data["sEcho"] = request.args["sEcho"]
return jsonify(data)
def api_datatables(unit):
# import time
# time.sleep(5)
collection = None
sort = None
skip = int(request.args.get("iDisplayStart", 0))
limit = int(request.args.get("iDisplayLength", 20))
if unit == "queues":
queues = []
for name, jobs in Queue.all_known().items():
queue = Queue(name)
q = {
"name": name,
"jobs": jobs, # MongoDB size
"size": queue.size(), # Redis size
"is_sorted": queue.is_sorted,
"is_timed": queue.is_timed,
"is_raw": queue.is_raw,
"is_set": queue.is_set
}
if queue.is_sorted:
raw_config = cfg.get("raw_queues", {}).get(name, {})
q["graph_config"] = raw_config.get("dashboard_graph", lambda: {
"start": time.time() - (7 * 24 * 3600),
"stop": time.time() + (7 * 24 * 3600),
"slices": 30
} if queue.is_timed else {
"start": 0,
"stop": 100,
"slices": 30
})()
if q["graph_config"]:
q["graph"] = queue.get_sorted_graph(**q["graph_config"])
if queue.is_timed:
q["jobs_to_dequeue"] = queue.count_jobs_to_dequeue()
queues.append(q)
queues.sort(key=lambda x: -(x["jobs"] + x["size"]))
data = {
"aaData": queues,
"iTotalDisplayRecords": len(queues)
}
elif unit == "workers":
fields = None
query = {"status": {"$nin": ["stop"]}}
collection = connections.mongodb_jobs.mrq_workers
sort = [("datestarted", -1)]
if request.args.get("showstopped"):
query = {}
elif unit == "scheduled_jobs":
collection = connections.mongodb_jobs.mrq_scheduled_jobs
fields = None
query = {}
elif unit == "jobs":
fields = None
query = build_api_datatables_query(request)
sort = [("_id", 1)]
# We can't search easily params because we store it as decoded JSON in mongo :(
# Add a string index?
# if request.args.get("sSearch"):
# query.update(json.loads(request.args.get("sSearch")))
collection = connections.mongodb_jobs.mrq_jobs
if collection is not None:
cursor = collection.find(query, projection=fields)
if sort:
cursor.sort(sort)
if skip is not None:
cursor.skip(skip)
if limit is not None:
cursor.limit(limit)
data = {
"aaData": list(cursor),
"iTotalDisplayRecords": collection.find(query).count()
}
data["sEcho"] = request.args["sEcho"]
return jsonify(data)
def api_datatables(unit):
collection = None
sort = None
skip = int(request.args.get("iDisplayStart", 0))
limit = int(request.args.get("iDisplayLength", 20))
with_mongodb_size = bool(request.args.get("with_mongodb_size"))
if unit == "queues":
queues = []
for name in Queue.all_known():
queue = Queue(name)
jobs = None
if with_mongodb_size:
jobs = connections.mongodb_jobs.mrq_jobs.count({
"queue":
name,
"status":
request.args.get("status") or "queued"
})
q = {
"name": name,
"jobs": jobs, # MongoDB size
"size": queue.size(), # Redis size
"is_sorted": queue.is_sorted,
"is_timed": queue.is_timed,
"is_raw": queue.is_raw,
"is_set": queue.is_set
}
if queue.is_sorted:
raw_config = cfg.get("raw_queues", {}).get(name, {})
q["graph_config"] = raw_config.get(
"dashboard_graph", lambda: {
"start": time.time() - (7 * 24 * 3600),
"stop": time.time() + (7 * 24 * 3600),
"slices": 30
} if queue.is_timed else {
"start": 0,
"stop": 100,
"slices": 30
})()
if q["graph_config"]:
q["graph"] = queue.get_sorted_graph(**q["graph_config"])
if queue.is_timed:
q["jobs_to_dequeue"] = queue.count_jobs_to_dequeue()
queues.append(q)
sSortField, sSortDirection = _get_sort_args(request, 'size', 'desc')
queues.sort(key=lambda x: x.get(sSortField, 0),
reverse=sSortDirection == 'desc')
data = {"aaData": queues, "iTotalDisplayRecords": len(queues)}
elif unit == "workers":
fields = None
query = {"status": {"$nin": ["stop"]}}
collection = connections.mongodb_jobs.mrq_workers
sSortField, sSortDirection = _get_sort_args(request, 'datestarted',
'desc')
sort = [(sSortField, -1 if sSortDirection == 'desc' else 1)]
if request.args.get("showstopped"):
query = {}
elif unit == "scheduled_jobs":
collection = connections.mongodb_jobs.mrq_scheduled_jobs
fields = None
query = {}
sSortField, sSortDirection = _get_sort_args(request, 'interval',
'desc')
sort = [(sSortField, -1 if sSortDirection == 'desc' else 1)]
elif unit == "jobs":
fields = None
query = build_api_datatables_query(request)
sSortField, sSortDirection = _get_sort_args(request)
sort = [(sSortField, -1 if sSortDirection == 'desc' else 1)]
time_s = request.args.get("time_s", '')
time_e = request.args.get("time_e", '')
if time_s and not time_e:
print 'datestarted', time_s
query.update({'datestarted': {'$gte': str2datetime(time_s)}})
elif time_e and not time_s:
print 'datestarted', time_e
query.update({'datestarted': {'$lte': str2datetime(time_e)}})
elif time_s and time_e:
print 'datestarted', time_s, time_e
query.update({
'datestarted': {
'$gte': str2datetime(time_s),
'$lte': str2datetime(time_e)
}
})
# We can't search easily params because we store it as decoded JSON in mongo :(
# Add a string index?
# if request.args.get("sSearch"):
# query.update(json.loads(request.args.get("sSearch")))
collection = connections.mongodb_jobs.mrq_jobs
if collection is not None:
cursor = collection.find(query, projection=fields)
if sort:
cursor.sort(sort)
if skip is not None:
cursor.skip(skip)
if limit is not None:
cursor.limit(limit)
data = {
"aaData": list(cursor),
"iTotalDisplayRecords": collection.find(query).count()
}
data["sEcho"] = request.args["sEcho"]
return jsonify(data)
