def work(burst=False):
"Queue worker processing statuses."
from summarizer import r, summary
rules = summary.filters.AdblockURLFilter.rules # force
with Connection(r):
worker = Worker([Queue(config.SUMMARIZER_QUEUE)])
worker.work(burst)
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
# keep for later
enqueued_at_date = strip_microseconds(job.enqueued_at)
w = Worker([q])
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.origin, q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEquals(job.enqueued_at, enqueued_at_date)
self.assertIsNotNone(job.exc_info) # should contain exc_info
def run_worker():
from rq import Worker, Connection
with Connection():
q = get_queue()
qs = [q]
w = Worker(qs)
w.work()
def test_self_modification_persistence_with_error(self):
"""Make sure that any meta modification done by
the job itself persists completely through the
queue/worker/job stack -- even if the job errored"""
q = Queue()
# Also make sure that previously existing metadata
# persists properly
job = q.enqueue(modify_self_and_error, meta={'foo': 'bar', 'baz': 42},
args=[{'baz': 10, 'newinfo': 'waka'}])
w = Worker([q])
w.work(burst=True)
# Postconditions
self.assertEqual(q.count, 0)
failed_job_registry = FailedJobRegistry(queue=q)
self.assertTrue(job in failed_job_registry)
self.assertEqual(w.get_current_job_id(), None)
job_check = Job.fetch(job.id)
self.assertEqual(set(job_check.meta.keys()),
set(['foo', 'baz', 'newinfo']))
self.assertEqual(job_check.meta['foo'], 'bar')
self.assertEqual(job_check.meta['baz'], 10)
self.assertEqual(job_check.meta['newinfo'], 'waka')
def test_job_dependency_race_condition(self):
"""Dependencies added while the job gets finished shouldn't get lost."""
# This patches the enqueue_dependents to enqueue a new dependency AFTER
# the original code was executed.
orig_enqueue_dependents = Queue.enqueue_dependents
def new_enqueue_dependents(self, job, *args, **kwargs):
orig_enqueue_dependents(self, job, *args, **kwargs)
if hasattr(Queue, '_add_enqueue') and Queue._add_enqueue is not None and Queue._add_enqueue.id == job.id:
Queue._add_enqueue = None
Queue().enqueue_call(say_hello, depends_on=job)
Queue.enqueue_dependents = new_enqueue_dependents
q = Queue()
w = Worker([q])
with mock.patch.object(Worker, 'execute_job', wraps=w.execute_job) as mocked:
parent_job = q.enqueue(say_hello, result_ttl=0)
Queue._add_enqueue = parent_job
job = q.enqueue_call(say_hello, depends_on=parent_job)
w.work(burst=True)
job = Job.fetch(job.id)
self.assertEqual(job.get_status(), JobStatus.FINISHED)
# The created spy checks two issues:
# * before the fix of #739, 2 of the 3 jobs where executed due
# to the race condition
# * during the development another issue was fixed:
# due to a missing pipeline usage in Queue.enqueue_job, the job
# which was enqueued before the "rollback" was executed twice.
# So before that fix the call count was 4 instead of 3
self.assertEqual(mocked.call_count, 3)
def test_custom_exc_handling(self):
"""Custom exception handling."""
def black_hole(job, *exc_info):
failed_queue = job.connection.get_failed_queue()
failed_queue.pop_job_id(job.id)
return False
q = self.conn.mkqueue()
failed_q = self.conn.get_failed_queue()
# Preconditions
self.assertEqual(failed_q.count, 0)
self.assertEqual(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEqual(q.count, 1)
w = Worker([q], exception_handlers=black_hole, connection=self.conn)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEqual(q.count, 0)
self.assertEqual(failed_q.count, 0)
# Check the job
job.refresh()
self.assertEqual(job.is_failed, True)
def test_work_is_unreadable(self):
"""Unreadable jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# NOTE: We have to fake this enqueueing for this test case.
# What we're simulating here is a call to a function that is not
# importable from the worker process.
job = Job.create(func=div_by_zero, args=(3,))
job.save()
data = self.testconn.hget(job.key, 'data')
invalid_data = data.replace(b'div_by_zero', b'nonexisting')
assert data != invalid_data
self.testconn.hset(job.key, 'data', invalid_data)
# We use the low-level internal function to enqueue any data (bypassing
# validity checks)
q.push_job_id(job.id)
self.assertEquals(q.count, 1)
# All set, we're going to process it
w = Worker([q])
w.work(burst=True) # should silently pass
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
def start_lib_worker(self, worker) :
# Preload libraries
# from newsle.core.models.orm.stat_configs import StatConfigs
# Provide queue names to listen to as arguments to this script,
# similar to rqworker
from rq import Queue, Connection, Worker, use_connection
import redis
if "server" in worker and len( worker["server"] ) > 0 :
server = worker["server"]
host = server.get("host", "127.0.0.1")
port = server.get("port", "6379")
password = server.get("password", None)
redis_conn = redis.StrictRedis(host=host, port=port, db=None,
password=password, unix_socket_path=None)
use_connection(redis_conn)
with Connection():
queues = ["default"]
if "queues" in worker and len( worker["queues"] ) > 0 :
queues = worker["queues"]
qs = map(Queue, queues)
w = Worker(qs)
w.work()
def launch_worker():
with Connection():
w = Worker(
get_available_worker_name()
)
w.work()
def test_custom_exc_handling(self):
"""Custom exception handling."""
def black_hole(job, *exc_info):
# Don't fall through to default behaviour (moving to failed queue)
return False
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
w = Worker([q], exc_handler=black_hole)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 0)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.is_failed, True)
def test_job_times(self):
"""job times are set correctly."""
q = Queue('foo')
w = Worker([q])
before = utcnow()
before = before.replace(microsecond=0)
job = q.enqueue(say_hello)
self.assertIsNotNone(job.enqueued_at)
self.assertIsNone(job.started_at)
self.assertIsNone(job.ended_at)
self.assertEqual(
w.work(burst=True), True,
'Expected at least some work done.'
)
self.assertEqual(job.result, 'Hi there, Stranger!')
after = utcnow()
job.refresh()
self.assertTrue(
before <= job.enqueued_at <= after,
'Not %s <= %s <= %s' % (before, job.enqueued_at, after)
)
self.assertTrue(
before <= job.started_at <= after,
'Not %s <= %s <= %s' % (before, job.started_at, after)
)
self.assertTrue(
before <= job.ended_at <= after,
'Not %s <= %s <= %s' % (before, job.ended_at, after)
)
def test_timeouts(self):
"""Worker kills jobs after timeout."""
sentinel_file = '/tmp/.rq_sentinel'
q = Queue()
w = Worker([q])
# Put it on the queue with a timeout value
res = q.enqueue(create_file_after_timeout,
args=(sentinel_file, 4),
timeout=1)
try:
os.unlink(sentinel_file)
except OSError as e:
if e.errno == 2:
pass
self.assertEquals(os.path.exists(sentinel_file), False)
w.work(burst=True)
self.assertEquals(os.path.exists(sentinel_file), False)
# TODO: Having to do the manual refresh() here is really ugly!
res.refresh()
self.assertIn('JobTimeoutException', as_text(res.exc_info))
def test_cancelled_jobs_arent_executed(self): # noqa
"""Cancelling jobs."""
SENTINEL_FILE = '/tmp/rq-tests.txt'
try:
# Remove the sentinel if it is leftover from a previous test run
os.remove(SENTINEL_FILE)
except OSError as e:
if e.errno != 2:
raise
q = Queue()
job = q.enqueue(create_file, SENTINEL_FILE)
# Here, we cancel the job, so the sentinel file may not be created
assert q.count == 1
job.cancel()
assert q.count == 1
w = Worker([q])
w.work(burst=True)
assert q.count == 0
# Should not have created evidence of execution
self.assertEquals(os.path.exists(SENTINEL_FILE), False)
def test_worker_sets_job_status(self):
"""Ensure that worker correctly sets job status."""
q = Queue()
w = Worker([q])
job = q.enqueue(say_hello)
self.assertEqual(job.get_status(), Status.QUEUED)
self.assertEqual(job.is_queued, True)
self.assertEqual(job.is_finished, False)
self.assertEqual(job.is_failed, False)
w.work(burst=True)
job = Job.fetch(job.id)
self.assertEqual(job.get_status(), Status.FINISHED)
self.assertEqual(job.is_queued, False)
self.assertEqual(job.is_finished, True)
self.assertEqual(job.is_failed, False)
# Failed jobs should set status to "failed"
job = q.enqueue(div_by_zero, args=(1,))
w.work(burst=True)
job = Job.fetch(job.id)
self.assertEqual(job.get_status(), Status.FAILED)
self.assertEqual(job.is_queued, False)
self.assertEqual(job.is_finished, False)
self.assertEqual(job.is_failed, True)
def test_worker_ttl(self):
"""Worker ttl."""
w = Worker([])
w.register_birth() # ugly: our test should only call public APIs
[worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
self.assertIsNotNone(self.testconn.ttl(worker_key))
w.register_death()
def test_worker_registration(self):
"""Ensure worker.key is correctly set in Redis."""
foo_queue = Queue(name='foo')
bar_queue = Queue(name='bar')
worker = Worker([foo_queue, bar_queue])
register(worker)
redis = worker.connection
self.assertTrue(redis.sismember(worker.redis_workers_keys, worker.key))
self.assertEqual(Worker.count(connection=redis), 1)
self.assertTrue(
redis.sismember(WORKERS_BY_QUEUE_KEY % foo_queue.name, worker.key)
)
self.assertEqual(Worker.count(queue=foo_queue), 1)
self.assertTrue(
redis.sismember(WORKERS_BY_QUEUE_KEY % bar_queue.name, worker.key)
)
self.assertEqual(Worker.count(queue=bar_queue), 1)
unregister(worker)
self.assertFalse(redis.sismember(worker.redis_workers_keys, worker.key))
self.assertFalse(
redis.sismember(WORKERS_BY_QUEUE_KEY % foo_queue.name, worker.key)
)
self.assertFalse(
redis.sismember(WORKERS_BY_QUEUE_KEY % bar_queue.name, worker.key)
)
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
q = Queue()
self.assertEqual(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEqual(q.count, 1)
# keep for later
enqueued_at_date = str(job.enqueued_at)
w = Worker([q])
w.work(burst=True) # should silently pass
# Postconditions
self.assertEqual(q.count, 0)
failed_job_registry = FailedJobRegistry(queue=q)
self.assertTrue(job in failed_job_registry)
self.assertEqual(w.get_current_job_id(), None)
# Check the job
job = Job.fetch(job.id)
self.assertEqual(job.origin, q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEqual(str(job.enqueued_at), enqueued_at_date)
self.assertTrue(job.exc_info) # should contain exc_info
def test_suspend_worker_execution(self):
"""Test Pause Worker Execution"""
SENTINEL_FILE = '/tmp/rq-tests.txt'
try:
# Remove the sentinel if it is leftover from a previous test run
os.remove(SENTINEL_FILE)
except OSError as e:
if e.errno != 2:
raise
q = Queue()
q.enqueue(create_file, SENTINEL_FILE)
w = Worker([q])
suspend(self.testconn)
w.work(burst=True)
assert q.count == 1
# Should not have created evidence of execution
self.assertEqual(os.path.exists(SENTINEL_FILE), False)
resume(self.testconn)
w.work(burst=True)
assert q.count == 0
self.assertEqual(os.path.exists(SENTINEL_FILE), True)
def test_work_is_unreadable(self):
"""Unreadable jobs are put on the failed job registry."""
q = Queue()
self.assertEqual(q.count, 0)
# NOTE: We have to fake this enqueueing for this test case.
# What we're simulating here is a call to a function that is not
# importable from the worker process.
job = Job.create(func=div_by_zero, args=(3,), origin=q.name)
job.save()
job_data = job.data
invalid_data = job_data.replace(b'div_by_zero', b'nonexisting')
assert job_data != invalid_data
self.testconn.hset(job.key, 'data', zlib.compress(invalid_data))
# We use the low-level internal function to enqueue any data (bypassing
# validity checks)
q.push_job_id(job.id)
self.assertEqual(q.count, 1)
# All set, we're going to process it
w = Worker([q])
w.work(burst=True) # should silently pass
self.assertEqual(q.count, 0)
failed_job_registry = FailedJobRegistry(queue=q)
self.assertTrue(job in failed_job_registry)
def test_worker_logs_success(self):
"""Worker can log success response of various jobs"""
q = Queue()
w = Worker([q])
# ascii response
job = q.enqueue(say_hello)
w.prepare_job_execution(job)
try:
success = w.perform_job(job, q)
except UnicodeEncodeError:
self.fail('perform_job raised UnicodeEncodeError unexpectedly')
self.assertTrue(success)
# bytestring response
job = q.enqueue(return_bytestring)
w.prepare_job_execution(job)
try:
success = w.perform_job(job, q)
except UnicodeDecodeError:
self.fail('perform_job raised UnicodeDecodeError unexpectedly')
self.assertTrue(success)
# unicode response
job = q.enqueue(return_unicode)
w.prepare_job_execution(job)
try:
success = w.perform_job(job, q)
except UnicodeEncodeError:
self.fail('perform_job raised UnicodeEncodeError unexpectedly')
self.assertTrue(success)
def test_worker_ttl(self):
"""Worker ttl."""
w = Worker([])
w.register_birth()
[worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
self.assertIsNotNone(self.testconn.ttl(worker_key))
w.register_death()
def test_work_via_string_argument(self):
"""Worker processes work fed via string arguments."""
q = Queue("foo")
w = Worker([q])
job = q.enqueue("tests.fixtures.say_hello", name="Frank")
self.assertEquals(w.work(burst=True), True, "Expected at least some work done.")
self.assertEquals(job.result, "Hi there, Frank!")
def __init__(self, *nargs, **kwargs):
processes = kwargs['processes'] if 'processes' in kwargs else 5
self.gevent_pool = gevent.pool.Pool(int(processes))
Worker.__init__(self, *nargs, **kwargs)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN', None)
setup_loghandlers(args)
setup_redis(args)
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def test_worker_ttl(self):
"""Worker ttl."""
w = Worker([], connection=self.conn)
w.register_birth() # ugly: our test should only call public APIs
[worker_key] = self.testconn.smembers(WORKERS_KEY)
self.assertIsNotNone(self.testconn.ttl(worker_key))
w.register_death()
def test_log_job_description_false(self, mock_logger_info):
"""Check that log_job_description False causes job lifespan to not be logged."""
q = Queue()
w = Worker([q], log_job_description=False)
job = q.enqueue(say_hello, args=('Frank',), result_ttl=10)
w.dequeue_job_and_maintain_ttl(10)
self.assertNotIn("Frank", mock_logger_info.call_args[0][2])
def test_all_queues():
"""All queues"""
q1 = Queue('first-queue')
q2 = Queue('second-queue')
q3 = Queue('third-queue')
# Ensure a queue is added only once a job is enqueued
assert not len((yield from Queue.all()))
yield from q1.enqueue(say_hello)
assert len((yield from Queue.all())) == 1
# Ensure this holds true for multiple queues
yield from q2.enqueue(say_hello)
yield from q3.enqueue(say_hello)
names = [q.name for q in (yield from Queue.all())]
assert len((yield from Queue.all())) == 3
# Verify names
assert 'first-queue' in names
assert 'second-queue' in names
assert 'third-queue' in names
# Now empty two queues
with SynchronousConnection():
w = SynchronousWorker([SynchronousQueue('second-queue'),
SynchronousQueue('third-queue')])
w.work(burst=True)
# Queue.all() should still report the empty queues
assert len((yield from Queue.all())) == 3
def test_failed_job_max_tries_1__move_to_dlq():
q = Queue()
failed_q = get_failed_queue()
dlq = Queue('dead_letter_queue')
# we could test with one worker here, but we don't want
# the test to depend on when the Worker performs maint.
# tasks (before or after processing jobs)
w = Worker([q])
rw = RetryWorker([q], retry_config=dict(
max_tries=1, delays=[]))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
job.refresh()
assert q.count == 0
assert failed_q.count == 0
assert dlq.count == 1
assert job.meta['tries'] == 1
def test_work_via_string_argument(self):
"""Worker processes work fed via string arguments."""
q = Queue('foo')
w = Worker([q])
job = q.enqueue('tests.fixtures.say_hello', name='Frank')
self.assertEquals(w.work(burst=True), True,
'Expected at least some work done.')
self.assertEquals(job.result, 'Hi there, Frank!')
def handle(self, *args, **options):
listen = ['high', 'default', 'low']
redis_url = os.getenv('REDIS_URL', 'redis://127.0.0.1:6379')
conn = redis.from_url(redis_url)
with Connection(conn):
worker = Worker(map(Queue, listen))
worker.work()
def work(self):
redis_conn = self.get_redis_client(self.redis_url)
with Connection(connection=redis_conn):
qs = self.queue_names or ['default']
worker = Worker(qs)
worker.work()
])
log.error('worker error in job',
func_name=job.func_name,
args=job.args,
exc_info=(exc_type, exc_value, traceback))
# reset job state and retry the job
if exc_type != PersistentError:
job.set_status(JobStatus.QUEUED)
job.exc_info = None
q.enqueue_job(job)
else:
statsd.increment('worker_persistent_error',
tags=[
'job:%s' % job.func_name,
'error_type:%s' % exc_type,
'error:%s' % exc_value
])
log.error('PersistentError: not retrying', e=exc_value)
if __name__ == '__main__':
with Connection():
queue_names = [q.name]
worker_name = str(uuid4())
w = Worker(queue_names,
name=worker_name,
connection=redis_conn,
exception_handlers=[rq_error_handler])
w.work()
from redis import Redis
from rq import Queue, Worker
redis = Redis(host='redis', port=6379)
queue = Queue('model_prediction', connection=redis)
if __name__ == '__main__':
print('Starting Worker')
worker = Worker([queue], connection=redis, name='model_prediction')
worker.work()
print('Ending Worker')
def get_worker(redis_url, redis_password, queue_name="job_scheduler_queue"):
queue = redis_queue(redis_url, redis_password, queue_name)
return Worker(
queues=[queue],
connection=queue.connection
)
#! /rq/vevn/bin/ python
import os
from rq import Queue, Connection, Worker
from redis import Redis
listen = ['default'] # listen ??
redis_url = 'redis://redis:6379' # path to Redis url
redis_con = Redis.from_url(redis_url) # connection for Redis server
#
# create a worker process to listen for queue tasks.
#
if __name__ == "__main__":
# Provide queue names to listen to as arguments to this script,
with Connection(redis_con): # create connection
worker = Worker(list(map(Queue, listen))) # default queue
worker.work() # worker ??
https://devcenter.heroku.com/articles/python-rq
See "Queuing jobs" section on how to enqueue jobs.
https://devcenter.heroku.com/articles/python-rq#queuing-jobs
The worker does not start by default after deployment. It will be necessary
to start manually such as:
heroku ps:scale worker=N
where N is the number of workers that you want.
"""
import os
import redis
from rq import Connection
from rq import Queue
from rq import Worker
LISTEN = ['high', 'default', 'low']
REDIS_URL = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
CONN = redis.from_url(REDIS_URL)
if __name__ == '__main__':
with Connection(CONN):
worker = Worker(map(Queue, LISTEN))
worker.work()
def worker(queues='default'):
with Connection(redis_connection):
w = Worker(queues)
w.work()
def test_custom_job_class(self):
"""Ensure Worker accepts custom job class."""
q = Queue()
worker = Worker([q], job_class=CustomJob)
self.assertEqual(worker.job_class, CustomJob)
# set up bot logging
bot_logger = bot_log_setup()
if __name__ == '__main__':
# parse cli arguments into cliargs dictionary
cliargs_bot = vars(parse_cli_args())
# Redis queue names
listen = ['diskover_crawl']
print("""\033[31m
___ _ ____ _ _ ____ _ _ ____ ____ ;
|__> | ==== |-:_ [__] \/ |=== |--< ["]
____ ____ ____ _ _ _ ___ ____ ___ /[_]\\
|___ |--< |--| |/\| |___ |==] [__] | ] [ v%s
Redis RQ worker bot for diskover crawler
Crawling all your stuff.
\033[0m""" % (diskover.version))
with Connection(redis_conn):
w = Worker(listen)
if cliargs_bot['burst']:
w.work(burst=True)
else:
w.work()
def run_worker(burst=False):
w = Worker(queues=[os.environ.get("LCC_QUEUE_NAME", "lcc")],
connection=connection)
w.work(burst=burst)
import sys
from redis import Redis
from rq import Queue, Connection, Worker
from social_relay.config import REDIS_HOST, REDIS_PORT, REDIS_DB
if __name__ == '__main__':
# Provide queue names to listen to as arguments to this script,
# similar to rqworker
conn = Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
with Connection(conn):
qs = map(Queue, sys.argv[1:]) or [Queue()]
w = Worker(qs)
w.work()
def runworker():
redis_url = app.config['REDIS_URL']
redis_connection = redis.from_url(redis_url)
with Connection(redis_connection):
worker = Worker(app.config['QUEUES'])
worker.work()
def run_worker():
redis_conn = Redis()
with redis_conn:
w = Worker("al_test", connection=redis_conn)
w.work()
def take_action(self, parsed_args):
super().take_action(parsed_args)
with Connection():
w = Worker(['mail_sender'])
w.work()
def test_custom_exc_handling(self):
"""Custom exception handling."""
def first_handler(job, *exc_info):
job.meta = {'first_handler': True}
job.save_meta()
return True
def second_handler(job, *exc_info):
job.meta.update({'second_handler': True})
job.save_meta()
def black_hole(job, *exc_info):
# Don't fall through to default behaviour (moving to failed queue)
return False
q = Queue()
self.assertEqual(q.count, 0)
job = q.enqueue(div_by_zero)
w = Worker([q], exception_handlers=first_handler)
w.work(burst=True)
# Check the job
job.refresh()
self.assertEqual(job.is_failed, True)
self.assertTrue(job.meta['first_handler'])
job = q.enqueue(div_by_zero)
w = Worker([q], exception_handlers=[first_handler, second_handler])
w.work(burst=True)
# Both custom exception handlers are run
job.refresh()
self.assertEqual(job.is_failed, True)
self.assertTrue(job.meta['first_handler'])
self.assertTrue(job.meta['second_handler'])
job = q.enqueue(div_by_zero)
w = Worker(
[q],
exception_handlers=[first_handler, black_hole, second_handler])
w.work(burst=True)
# second_handler is not run since it's interrupted by black_hole
job.refresh()
self.assertEqual(job.is_failed, True)
self.assertTrue(job.meta['first_handler'])
self.assertEqual(job.meta.get('second_handler'), None)
def test_worker_sets_result_ttl(self):
"""Ensure that Worker properly sets result_ttl for individual jobs."""
q = Queue()
job = q.enqueue(say_hello, args=('Frank',), result_ttl=10)
w = Worker([q])
w.work(burst=True)
self.assertNotEqual(self.testconn._ttl(job.key), 0)
# Job with -1 result_ttl don't expire
job = q.enqueue(say_hello, args=('Frank',), result_ttl=-1)
w = Worker([q])
w.work(burst=True)
self.assertEqual(self.testconn._ttl(job.key), -1)
# Job with result_ttl = 0 gets deleted immediately
job = q.enqueue(say_hello, args=('Frank',), result_ttl=0)
w = Worker([q])
w.work(burst=True)
self.assertEqual(self.testconn.get(job.key), None)
import os
import redis
from rq import Worker, Queue, Connection
#this is the file that spins up a worker to process redis queues.
listen = ['in_twitter_queue']
redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') #heroku only
conn = redis.from_url(redis_url)
if __name__ == '__main__':
with Connection(conn): #sets up connection
worker = Worker(map(Queue, listen))
worker.work() #worker is spun up to listen to queue
from django.contrib.contenttypes.models import ContentType
from django.test import override_settings
from django.urls import reverse
from django.utils.timezone import make_aware
from django_rq.queues import get_connection
from rest_framework import status
from rq import Worker
from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, RackGroup, RackRole, Site
from extras.api.views import ReportViewSet, ScriptViewSet
from extras.models import ConfigContext, CustomField, ExportTemplate, ImageAttachment, Tag
from extras.reports import Report
from extras.scripts import BooleanVar, IntegerVar, Script, StringVar
from utilities.testing import APITestCase, APIViewTestCases
rq_worker_running = Worker.count(get_connection('default'))
class AppTest(APITestCase):
def test_root(self):
url = reverse('extras-api:api-root')
response = self.client.get('{}?format=api'.format(url), **self.header)
self.assertEqual(response.status_code, 200)
class CustomFieldTest(APIViewTestCases.APIViewTestCase):
model = CustomField
brief_fields = ['id', 'name', 'url']
create_data = [
from __future__ import with_statement
import os
import redis
from rq import Worker, Queue, Connection
from config import REDIS_HOST
from config import REDIS_PORT
from config import REDIS_DB
from config import QUEUES_LISTEN
# Get redis url
redis_url = 'redis://' + REDIS_HOST + ':' + str(REDIS_PORT)
# Get redis connection
redis_conn = redis.from_url(redis_url)
# Get redis queue object for each listing
qH = Queue('high', connection=redis_conn)
qN = Queue('normal', connection=redis_conn)
qL = Queue('low', connection=redis_conn)
# Create redis database object
rDB = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
if __name__ == '__main__':
# start a worker
with Connection(redis_conn):
worker = Worker(map(Queue, QUEUES_LISTEN))
worker.work()
def get_workers(): with Connection(get_redis_conn()): workers = Worker.all() return workers
def test_custom_queue_class(self):
"""Ensure Worker accepts custom queue class."""
q = CustomQueue()
worker = Worker([q], queue_class=CustomQueue)
self.assertEqual(worker.queue_class, CustomQueue)
def test_statistics(self):
"""Successful and failed job counts are saved properly"""
q = Queue()
job = q.enqueue(div_by_zero)
w = Worker([q])
w.register_birth()
self.assertEqual(w.failed_job_count, 0)
self.assertEqual(w.successful_job_count, 0)
self.assertEqual(w.total_working_time, 0)
registry = StartedJobRegistry(connection=w.connection)
job.started_at = utcnow()
job.ended_at = job.started_at + timedelta(seconds=0.75)
w.handle_job_failure(job)
w.handle_job_success(job, q, registry)
w.refresh()
self.assertEqual(w.failed_job_count, 1)
self.assertEqual(w.successful_job_count, 1)
self.assertEqual(w.total_working_time,
1500000) # 1.5 seconds in microseconds
w.handle_job_failure(job)
w.handle_job_success(job, q, registry)
w.refresh()
self.assertEqual(w.failed_job_count, 2)
self.assertEqual(w.successful_job_count, 2)
self.assertEqual(w.total_working_time, 3000000)
import os
import redis
from rq import Worker, Queue, Connection
from rq.handlers import move_to_failed_queue
from exception_handler import post_to_db
listen = ['thumbnails']
# Heroku provides the env variable REDISTOGO_URL for Heroku RedisToGo;
# the default redis://redis_worker:6379 points to the local docker redis
redis_url = os.getenv('REDISTOGO_URL', 'redis://redis_worker:6379')
connection = redis.from_url(redis_url)
if __name__ == '__main__':
with Connection(connection):
worker = Worker(map(Queue, listen),
exception_handlers=[post_to_db, move_to_failed_queue])
worker.work()
def test_find_by_key(self):
"""Worker.find_by_key restores queues, state and job_id."""
queues = [Queue('foo'), Queue('bar')]
w = Worker(queues)
w.register_death()
w.register_birth()
w.set_state(WorkerStatus.STARTED)
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.queues, queues)
self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
self.assertEqual(worker._job_id, None)
self.assertTrue(worker.key in Worker.all_keys(worker.connection))
# If worker is gone, its keys should also be removed
worker.connection.delete(worker.key)
Worker.find_by_key(worker.key)
self.assertFalse(worker.key in Worker.all_keys(worker.connection))
def start_worker():
with Connection(conn):
worker = Worker(queues=queue)
worker.work()
def test_worker_all(self):
"""Worker.all() works properly"""
foo_queue = Queue('foo')
bar_queue = Queue('bar')
w1 = Worker([foo_queue, bar_queue], name='w1')
w1.register_birth()
w2 = Worker([foo_queue], name='w2')
w2.register_birth()
self.assertEqual(set(Worker.all(connection=foo_queue.connection)),
set([w1, w2]))
self.assertEqual(set(Worker.all(queue=foo_queue)), set([w1, w2]))
self.assertEqual(set(Worker.all(queue=bar_queue)), set([w1]))
w1.register_death()
w2.register_death()
from __future__ import print_function, absolute_import
import os
import sys
import redis
import pyalerts
import pyalerts.jobs
from pyalerts.jobs import *
from pyalerts.scheduler.queue import conn, listen
from rq import Worker, Queue, Connection, get_current_job
if __name__ == '__main__':
with Connection(conn):
worker = Worker(list(map(Queue, listen)))
worker.work()
def work():
redis_conn = Redis()
parser = Worker(['analyzing'], connection=redis_conn)
parser.work()
def rq_worker():
worker = Worker([task_manager], connection=redis)
worker.work()
def scrape_tree_meta(paths, cliargs, reindex_dict):
worker = get_worker_name()
tree_dirs = []
tree_files = []
if cliargs['qumulo']:
qumulo = True
from diskover_qumulo import qumulo_get_dir_meta, qumulo_get_file_meta
else:
qumulo = False
totalcrawltime = 0
# check if other bots are idle and throw them some jobs (dir paths)
if len(paths) >= cliargs['batchsize']:
workers_idle = 0
workers = Worker.all(connection=redis_conn)
num_workers = len(workers)
for w in workers:
if w._state == "idle":
workers_idle += 1
if workers_idle > num_workers // 2:
workers_idle = True
break
q_len = len(q_crawl)
if q_len == 0 and workers_idle == True:
# take half the paths randomly
shuffle(paths)
n = len(paths) // 2
tosspaths = paths[:n]
paths = paths[n:]
q_crawl.enqueue(scrape_tree_meta,
args=(
tosspaths,
cliargs,
reindex_dict,
))
for path in paths:
starttime = time.time()
root, dirs, files = path
totaldirsize = 0
totaldiritems_subdirs = len(dirs)
totaldiritems_files = 0
# check if stats embeded in data from diskover tree walk client
if type(root) is tuple:
statsembeded = True
else:
statsembeded = False
if qumulo:
if root['path'] != '/':
root_path = root['path'].rstrip(os.path.sep)
else:
root_path = root['path']
dmeta = qumulo_get_dir_meta(worker, root, cliargs, reindex_dict,
redis_conn)
else:
if statsembeded:
root_path = root[0]
dmeta = get_dir_meta(worker,
root,
cliargs,
reindex_dict,
statsembeded=True)
else:
root_path = root
dmeta = get_dir_meta(worker,
root_path,
cliargs,
reindex_dict,
statsembeded=False)
if dmeta == "sametimes":
# fetch meta data for directory and all it's files (doc sources) from index2 since
# directory times haven't changed
dir_source, files_source = get_metadata(root_path, cliargs)
datenow = datetime.utcnow().isoformat()
for file_source in files_source:
# update indexed at time
file_source['indexing_date'] = datenow
# update worker name
file_source['worker_name'] = worker
tree_files.append(('file', file_source))
if dir_source:
# update indexed at time
dir_source['indexing_date'] = datenow
# update worker name
dir_source['worker_name'] = worker
# update crawl time
elapsed = time.time() - starttime
dir_source['crawl_time'] = round(elapsed, 6)
tree_dirs.append(dir_source)
totalcrawltime += elapsed
# get meta off disk since times different in Redis than on disk
elif dmeta:
# check if meta for files embeded
if statsembeded:
for file in files:
fmeta = get_file_meta(worker,
file,
cliargs,
reindex_dict,
statsembeded=True)
if fmeta:
tree_files.append(fmeta)
# add file size to totaldirsize
totaldirsize += fmeta['filesize']
totaldiritems_files += 1
else:
for file in files:
if qumulo:
fmeta = qumulo_get_file_meta(worker, file, cliargs,
reindex_dict)
else:
fmeta = get_file_meta(worker,
os.path.join(root_path, file),
cliargs,
reindex_dict,
statsembeded=False)
if fmeta:
tree_files.append(fmeta)
# add file size to totaldirsize
totaldirsize += fmeta['filesize']
totaldiritems_files += 1
# update crawl time
elapsed = time.time() - starttime
dmeta['crawl_time'] = round(elapsed, 6)
# update directory meta filesize, items
dmeta['filesize'] = totaldirsize
dmeta['items_files'] = totaldiritems_files
dmeta['items_subdirs'] = totaldiritems_subdirs
totaldiritems = totaldiritems_files + totaldiritems_subdirs
dmeta['items'] += totaldiritems
tree_dirs.append(dmeta)
totalcrawltime += elapsed
# check if doc count is more than es chunksize and bulk add to es
if len(tree_dirs) + len(tree_files) >= config['es_chunksize']:
td = tree_dirs[:]
tf = tree_files[:]
es_bulk_add(worker, td, tf, cliargs, totalcrawltime)
del tree_dirs[:]
del tree_files[:]
totalcrawltime = 0
# bulk add to es
if len(tree_dirs) > 0 or len(tree_files) > 0:
es_bulk_add(worker, tree_dirs, tree_files, cliargs, totalcrawltime)
