def run(self, params):
users = [{"user": x} for x in get_distinct_users()]
print "Queue users"
queue_jobs("bluebirdlib.tasks.GetUserTweets", users, queue="tweets")
hts = [{"search": x} for x in HASHTAGS]
print "Queue hashtags"
queue_jobs("bluebirdlib.tasks.getHashTagTweets", hts, queue="tweets")
print "Queue scheduler"
queue_job("bluebirdlib.tasks.Scheduler", {}, queue="tweets")
return 0
def send_tasks(self, path, params_list, block=True, queue=None, accept_statuses=["success"], start=True):
if not self.started and start:
self.start()
job_ids = queue_jobs(path, params_list, queue=queue)
return self.wait_for_tasks_results(job_ids, block=block, accept_statuses=accept_statuses)
def send_tasks(self, path, params_list, block=True, queue=None, accept_statuses=["success"], start=True):
if not self.started and start:
self.start()
job_ids = queue_jobs(path, params_list, queue=queue)
return self.wait_for_tasks_results(job_ids, block=block, accept_statuses=accept_statuses)
def send_tasks(path,
params_list,
queue=None,
sync=False,
batch_size=1000,
subpool_size=None):
if sync:
def subpool_inner_func(params):
try:
return run_task(path, params)
except Exception as exc:
trace = traceback.format_exc()
log.error("Error in send_tasks subpool: %s \n%s" %
(exc, trace))
raise
if subpool_size is None:
return [run_task(path, params) for params in params_list]
else:
return subpool_map(int(subpool_size), subpool_inner_func,
params_list)
return queue_jobs(path, params_list, queue=queue, batch_size=batch_size)
#!/usr/bin/env python
import sys
from mrq.context import setup_context
from mrq.job import queue_jobs, queue_raw_jobs
setup_context()
queue = sys.argv[1]
n = int(sys.argv[2])
if queue == "square":
queue_jobs("tasks.Square", [{"n": 42} for _ in range(n)], queue=queue)
elif queue in ("square_raw", "square_nostorage_raw"):
queue_raw_jobs(queue, [42 for _ in range(n)])