示例#1
0
def work(burst=False):
    "Queue worker processing statuses."
    from summarizer import r, summary
    rules = summary.filters.AdblockURLFilter.rules # force
    with Connection(r):
        worker = Worker([Queue(config.SUMMARIZER_QUEUE)])
        worker.work(burst)
示例#2
0
文件: test_worker.py 项目: askyer/rq
    def test_work_fails(self):
        """Failing jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        # keep for later
        enqueued_at_date = strip_microseconds(job.enqueued_at)

        w = Worker([q])
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.origin, q.name)

        # Should be the original enqueued_at date, not the date of enqueueing
        # to the failed queue
        self.assertEquals(job.enqueued_at, enqueued_at_date)
        self.assertIsNotNone(job.exc_info)  # should contain exc_info
示例#3
0
def run_worker():
    from rq import Worker, Connection
    with Connection():
        q = get_queue()
        qs = [q]
        w = Worker(qs)
        w.work()
示例#4
0
    def test_self_modification_persistence_with_error(self):
        """Make sure that any meta modification done by
        the job itself persists completely through the
        queue/worker/job stack -- even if the job errored"""
        q = Queue()
        # Also make sure that previously existing metadata
        # persists properly
        job = q.enqueue(modify_self_and_error, meta={'foo': 'bar', 'baz': 42},
                        args=[{'baz': 10, 'newinfo': 'waka'}])

        w = Worker([q])
        w.work(burst=True)

        # Postconditions
        self.assertEqual(q.count, 0)
        failed_job_registry = FailedJobRegistry(queue=q)
        self.assertTrue(job in failed_job_registry)
        self.assertEqual(w.get_current_job_id(), None)

        job_check = Job.fetch(job.id)
        self.assertEqual(set(job_check.meta.keys()),
                         set(['foo', 'baz', 'newinfo']))
        self.assertEqual(job_check.meta['foo'], 'bar')
        self.assertEqual(job_check.meta['baz'], 10)
        self.assertEqual(job_check.meta['newinfo'], 'waka')
示例#5
0
    def test_job_dependency_race_condition(self):
        """Dependencies added while the job gets finished shouldn't get lost."""

        # This patches the enqueue_dependents to enqueue a new dependency AFTER
        # the original code was executed.
        orig_enqueue_dependents = Queue.enqueue_dependents

        def new_enqueue_dependents(self, job, *args, **kwargs):
            orig_enqueue_dependents(self, job, *args, **kwargs)
            if hasattr(Queue, '_add_enqueue') and Queue._add_enqueue is not None and Queue._add_enqueue.id == job.id:
                Queue._add_enqueue = None
                Queue().enqueue_call(say_hello, depends_on=job)

        Queue.enqueue_dependents = new_enqueue_dependents

        q = Queue()
        w = Worker([q])
        with mock.patch.object(Worker, 'execute_job', wraps=w.execute_job) as mocked:
            parent_job = q.enqueue(say_hello, result_ttl=0)
            Queue._add_enqueue = parent_job
            job = q.enqueue_call(say_hello, depends_on=parent_job)
            w.work(burst=True)
            job = Job.fetch(job.id)
            self.assertEqual(job.get_status(), JobStatus.FINISHED)

            # The created spy checks two issues:
            # * before the fix of #739, 2 of the 3 jobs where executed due
            #   to the race condition
            # * during the development another issue was fixed:
            #   due to a missing pipeline usage in Queue.enqueue_job, the job
            #   which was enqueued before the "rollback" was executed twice.
            #   So before that fix the call count was 4 instead of 3
            self.assertEqual(mocked.call_count, 3)
示例#6
0
    def test_custom_exc_handling(self):
        """Custom exception handling."""
        def black_hole(job, *exc_info):
            failed_queue = job.connection.get_failed_queue()
            failed_queue.pop_job_id(job.id)
            return False

        q = self.conn.mkqueue()
        failed_q = self.conn.get_failed_queue()

        # Preconditions
        self.assertEqual(failed_q.count, 0)
        self.assertEqual(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEqual(q.count, 1)

        w = Worker([q], exception_handlers=black_hole, connection=self.conn)
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEqual(q.count, 0)
        self.assertEqual(failed_q.count, 0)

        # Check the job
        job.refresh()
        self.assertEqual(job.is_failed, True)
示例#7
0
文件: test_worker.py 项目: askyer/rq
    def test_work_is_unreadable(self):
        """Unreadable jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # NOTE: We have to fake this enqueueing for this test case.
        # What we're simulating here is a call to a function that is not
        # importable from the worker process.
        job = Job.create(func=div_by_zero, args=(3,))
        job.save()
        data = self.testconn.hget(job.key, 'data')
        invalid_data = data.replace(b'div_by_zero', b'nonexisting')
        assert data != invalid_data
        self.testconn.hset(job.key, 'data', invalid_data)

        # We use the low-level internal function to enqueue any data (bypassing
        # validity checks)
        q.push_job_id(job.id)

        self.assertEquals(q.count, 1)

        # All set, we're going to process it
        w = Worker([q])
        w.work(burst=True)   # should silently pass
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)
示例#8
0
	def start_lib_worker(self, worker) :
		# Preload libraries
		# from newsle.core.models.orm.stat_configs import StatConfigs
		# Provide queue names to listen to as arguments to this script,
		# similar to rqworker
		from rq import Queue, Connection, Worker, use_connection
		import redis
		if "server" in worker and len( worker["server"] ) > 0 :
				
				server = worker["server"]
				host = server.get("host", "127.0.0.1")
				port = server.get("port", "6379")
				password = server.get("password", None)

				redis_conn = redis.StrictRedis(host=host, port=port, db=None,
							   password=password, unix_socket_path=None)
				
				use_connection(redis_conn)

		with Connection():
			queues = ["default"]
			if "queues" in worker and len( worker["queues"] ) > 0 :
				queues = worker["queues"]

			qs = map(Queue, queues)

			w = Worker(qs)
			w.work()
示例#9
0
文件: Worker.py 项目: adyasha/dune
def launch_worker():
    with Connection():
        w = Worker(
            get_available_worker_name()
        )

        w.work()
示例#10
0
文件: test_worker.py 项目: askyer/rq
    def test_custom_exc_handling(self):
        """Custom exception handling."""
        def black_hole(job, *exc_info):
            # Don't fall through to default behaviour (moving to failed queue)
            return False

        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        w = Worker([q], exc_handler=black_hole)
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 0)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.is_failed, True)
示例#11
0
 def test_job_times(self):
     """job times are set correctly."""
     q = Queue('foo')
     w = Worker([q])
     before = utcnow()
     before = before.replace(microsecond=0)
     job = q.enqueue(say_hello)
     self.assertIsNotNone(job.enqueued_at)
     self.assertIsNone(job.started_at)
     self.assertIsNone(job.ended_at)
     self.assertEqual(
         w.work(burst=True), True,
         'Expected at least some work done.'
     )
     self.assertEqual(job.result, 'Hi there, Stranger!')
     after = utcnow()
     job.refresh()
     self.assertTrue(
         before <= job.enqueued_at <= after,
         'Not %s <= %s <= %s' % (before, job.enqueued_at, after)
     )
     self.assertTrue(
         before <= job.started_at <= after,
         'Not %s <= %s <= %s' % (before, job.started_at, after)
     )
     self.assertTrue(
         before <= job.ended_at <= after,
         'Not %s <= %s <= %s' % (before, job.ended_at, after)
     )
示例#12
0
文件: test_worker.py 项目: askyer/rq
    def test_timeouts(self):
        """Worker kills jobs after timeout."""
        sentinel_file = '/tmp/.rq_sentinel'

        q = Queue()
        w = Worker([q])

        # Put it on the queue with a timeout value
        res = q.enqueue(create_file_after_timeout,
                        args=(sentinel_file, 4),
                        timeout=1)

        try:
            os.unlink(sentinel_file)
        except OSError as e:
            if e.errno == 2:
                pass

        self.assertEquals(os.path.exists(sentinel_file), False)
        w.work(burst=True)
        self.assertEquals(os.path.exists(sentinel_file), False)

        # TODO: Having to do the manual refresh() here is really ugly!
        res.refresh()
        self.assertIn('JobTimeoutException', as_text(res.exc_info))
示例#13
0
文件: test_worker.py 项目: askyer/rq
    def test_cancelled_jobs_arent_executed(self):  # noqa
        """Cancelling jobs."""

        SENTINEL_FILE = '/tmp/rq-tests.txt'

        try:
            # Remove the sentinel if it is leftover from a previous test run
            os.remove(SENTINEL_FILE)
        except OSError as e:
            if e.errno != 2:
                raise

        q = Queue()
        job = q.enqueue(create_file, SENTINEL_FILE)

        # Here, we cancel the job, so the sentinel file may not be created
        assert q.count == 1
        job.cancel()
        assert q.count == 1

        w = Worker([q])
        w.work(burst=True)
        assert q.count == 0

        # Should not have created evidence of execution
        self.assertEquals(os.path.exists(SENTINEL_FILE), False)
示例#14
0
文件: test_worker.py 项目: askyer/rq
    def test_worker_sets_job_status(self):
        """Ensure that worker correctly sets job status."""
        q = Queue()
        w = Worker([q])

        job = q.enqueue(say_hello)
        self.assertEqual(job.get_status(), Status.QUEUED)
        self.assertEqual(job.is_queued, True)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, False)

        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FINISHED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, True)
        self.assertEqual(job.is_failed, False)

        # Failed jobs should set status to "failed"
        job = q.enqueue(div_by_zero, args=(1,))
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FAILED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, True)
示例#15
0
文件: test_worker.py 项目: askyer/rq
 def test_worker_ttl(self):
     """Worker ttl."""
     w = Worker([])
     w.register_birth()  # ugly: our test should only call public APIs
     [worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
     self.assertIsNotNone(self.testconn.ttl(worker_key))
     w.register_death()
示例#16
0
    def test_worker_registration(self):
        """Ensure worker.key is correctly set in Redis."""
        foo_queue = Queue(name='foo')
        bar_queue = Queue(name='bar')
        worker = Worker([foo_queue, bar_queue])

        register(worker)
        redis = worker.connection

        self.assertTrue(redis.sismember(worker.redis_workers_keys, worker.key))
        self.assertEqual(Worker.count(connection=redis), 1)
        self.assertTrue(
            redis.sismember(WORKERS_BY_QUEUE_KEY % foo_queue.name, worker.key)
        )
        self.assertEqual(Worker.count(queue=foo_queue), 1)
        self.assertTrue(
            redis.sismember(WORKERS_BY_QUEUE_KEY % bar_queue.name, worker.key)
        )
        self.assertEqual(Worker.count(queue=bar_queue), 1)

        unregister(worker)
        self.assertFalse(redis.sismember(worker.redis_workers_keys, worker.key))
        self.assertFalse(
            redis.sismember(WORKERS_BY_QUEUE_KEY % foo_queue.name, worker.key)
        )
        self.assertFalse(
            redis.sismember(WORKERS_BY_QUEUE_KEY % bar_queue.name, worker.key)
        )
示例#17
0
    def test_work_fails(self):
        """Failing jobs are put on the failed queue."""
        q = Queue()
        self.assertEqual(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEqual(q.count, 1)

        # keep for later
        enqueued_at_date = str(job.enqueued_at)

        w = Worker([q])
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEqual(q.count, 0)
        failed_job_registry = FailedJobRegistry(queue=q)
        self.assertTrue(job in failed_job_registry)
        self.assertEqual(w.get_current_job_id(), None)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEqual(job.origin, q.name)

        # Should be the original enqueued_at date, not the date of enqueueing
        # to the failed queue
        self.assertEqual(str(job.enqueued_at), enqueued_at_date)
        self.assertTrue(job.exc_info)  # should contain exc_info
示例#18
0
    def test_suspend_worker_execution(self):
        """Test Pause Worker Execution"""

        SENTINEL_FILE = '/tmp/rq-tests.txt'

        try:
            # Remove the sentinel if it is leftover from a previous test run
            os.remove(SENTINEL_FILE)
        except OSError as e:
            if e.errno != 2:
                raise

        q = Queue()
        q.enqueue(create_file, SENTINEL_FILE)

        w = Worker([q])

        suspend(self.testconn)

        w.work(burst=True)
        assert q.count == 1

        # Should not have created evidence of execution
        self.assertEqual(os.path.exists(SENTINEL_FILE), False)

        resume(self.testconn)
        w.work(burst=True)
        assert q.count == 0
        self.assertEqual(os.path.exists(SENTINEL_FILE), True)
示例#19
0
    def test_work_is_unreadable(self):
        """Unreadable jobs are put on the failed job registry."""
        q = Queue()
        self.assertEqual(q.count, 0)

        # NOTE: We have to fake this enqueueing for this test case.
        # What we're simulating here is a call to a function that is not
        # importable from the worker process.
        job = Job.create(func=div_by_zero, args=(3,), origin=q.name)
        job.save()

        job_data = job.data
        invalid_data = job_data.replace(b'div_by_zero', b'nonexisting')
        assert job_data != invalid_data
        self.testconn.hset(job.key, 'data', zlib.compress(invalid_data))

        # We use the low-level internal function to enqueue any data (bypassing
        # validity checks)
        q.push_job_id(job.id)

        self.assertEqual(q.count, 1)

        # All set, we're going to process it
        w = Worker([q])
        w.work(burst=True)   # should silently pass
        self.assertEqual(q.count, 0)

        failed_job_registry = FailedJobRegistry(queue=q)
        self.assertTrue(job in failed_job_registry)
示例#20
0
    def test_worker_logs_success(self):
        """Worker can log success response of various jobs"""
        q = Queue()
        w = Worker([q])

        # ascii response
        job = q.enqueue(say_hello)
        w.prepare_job_execution(job)
        try:
            success = w.perform_job(job, q)
        except UnicodeEncodeError:
            self.fail('perform_job raised UnicodeEncodeError unexpectedly')
        self.assertTrue(success)

        # bytestring response
        job = q.enqueue(return_bytestring)
        w.prepare_job_execution(job)
        try:
            success = w.perform_job(job, q)
        except UnicodeDecodeError:
            self.fail('perform_job raised UnicodeDecodeError unexpectedly')
        self.assertTrue(success)

        # unicode response
        job = q.enqueue(return_unicode)
        w.prepare_job_execution(job)
        try:
            success = w.perform_job(job, q)
        except UnicodeEncodeError:
            self.fail('perform_job raised UnicodeEncodeError unexpectedly')
        self.assertTrue(success)
示例#21
0
 def test_worker_ttl(self):
     """Worker ttl."""
     w = Worker([])
     w.register_birth()
     [worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
     self.assertIsNotNone(self.testconn.ttl(worker_key))
     w.register_death()
示例#22
0
文件: test_worker.py 项目: kaulie/rq
 def test_work_via_string_argument(self):
     """Worker processes work fed via string arguments."""
     q = Queue("foo")
     w = Worker([q])
     job = q.enqueue("tests.fixtures.say_hello", name="Frank")
     self.assertEquals(w.work(burst=True), True, "Expected at least some work done.")
     self.assertEquals(job.result, "Hi there, Frank!")
示例#23
0
    def __init__(self, *nargs, **kwargs):

        processes = kwargs['processes'] if 'processes' in kwargs else 5

        self.gevent_pool = gevent.pool.Pool(int(processes))

        Worker.__init__(self, *nargs, **kwargs)
示例#24
0
def main():
    args = parse_args()

    if args.path:
        sys.path = args.path.split(':') + sys.path

    settings = {}
    if args.config:
        settings = read_config_file(args.config)

    setup_default_arguments(args, settings)

    # Other default arguments
    if args.sentry_dsn is None:
        args.sentry_dsn = settings.get('SENTRY_DSN', None)

    setup_loghandlers(args)
    setup_redis(args)
    try:
        queues = map(Queue, args.queues)
        w = Worker(queues, name=args.name)

        # Should we configure Sentry?
        if args.sentry_dsn:
            from raven import Client
            from rq.contrib.sentry import register_sentry
            client = Client(args.sentry_dsn)
            register_sentry(client, w)

        w.work(burst=args.burst)
    except ConnectionError as e:
        print(e)
        sys.exit(1)
示例#25
0
 def test_worker_ttl(self):
     """Worker ttl."""
     w = Worker([], connection=self.conn)
     w.register_birth()  # ugly: our test should only call public APIs
     [worker_key] = self.testconn.smembers(WORKERS_KEY)
     self.assertIsNotNone(self.testconn.ttl(worker_key))
     w.register_death()
示例#26
0
 def test_log_job_description_false(self, mock_logger_info):
     """Check that log_job_description False causes job lifespan to not be logged."""
     q = Queue()
     w = Worker([q], log_job_description=False)
     job = q.enqueue(say_hello, args=('Frank',), result_ttl=10)
     w.dequeue_job_and_maintain_ttl(10)
     self.assertNotIn("Frank", mock_logger_info.call_args[0][2])
示例#27
0
def test_all_queues():
    """All queues"""

    q1 = Queue('first-queue')
    q2 = Queue('second-queue')
    q3 = Queue('third-queue')

    # Ensure a queue is added only once a job is enqueued
    assert not len((yield from Queue.all()))
    yield from q1.enqueue(say_hello)
    assert len((yield from Queue.all())) == 1

    # Ensure this holds true for multiple queues
    yield from q2.enqueue(say_hello)
    yield from q3.enqueue(say_hello)
    names = [q.name for q in (yield from Queue.all())]
    assert len((yield from Queue.all())) == 3

    # Verify names
    assert 'first-queue' in names
    assert 'second-queue' in names
    assert 'third-queue' in names

    # Now empty two queues
    with SynchronousConnection():
        w = SynchronousWorker([SynchronousQueue('second-queue'),
                               SynchronousQueue('third-queue')])
    w.work(burst=True)

    # Queue.all() should still report the empty queues
    assert len((yield from Queue.all())) == 3
示例#28
0
def test_failed_job_max_tries_1__move_to_dlq():
    q = Queue()
    failed_q = get_failed_queue()
    dlq = Queue('dead_letter_queue')

    # we could test with one worker here, but we don't want
    # the test to depend on when the Worker performs maint.
    # tasks (before or after processing jobs)
    w = Worker([q])
    rw = RetryWorker([q], retry_config=dict(
        max_tries=1, delays=[]))

    # run job that will fail
    job = q.enqueue(error_fun)
    w.work(burst=True)
    assert q.count == 0
    assert get_failed_queue().count == 1

    # run retry worker
    rw.work(burst=True)
    job.refresh()

    assert q.count == 0
    assert failed_q.count == 0
    assert dlq.count == 1
    assert job.meta['tries'] == 1
示例#29
0
文件: test_worker.py 项目: askyer/rq
 def test_work_via_string_argument(self):
     """Worker processes work fed via string arguments."""
     q = Queue('foo')
     w = Worker([q])
     job = q.enqueue('tests.fixtures.say_hello', name='Frank')
     self.assertEquals(w.work(burst=True), True,
                       'Expected at least some work done.')
     self.assertEquals(job.result, 'Hi there, Frank!')
示例#30
0
    def handle(self, *args, **options):
        listen = ['high', 'default', 'low']
        redis_url = os.getenv('REDIS_URL', 'redis://127.0.0.1:6379')
        conn = redis.from_url(redis_url)

        with Connection(conn):
            worker = Worker(map(Queue, listen))
            worker.work()
示例#31
0
 def work(self):
     redis_conn = self.get_redis_client(self.redis_url)
     with Connection(connection=redis_conn):
         qs = self.queue_names or ['default']
         worker = Worker(qs)
         worker.work()
示例#32
0
                     ])
    log.error('worker error in job',
              func_name=job.func_name,
              args=job.args,
              exc_info=(exc_type, exc_value, traceback))

    # reset job state and retry the job
    if exc_type != PersistentError:
        job.set_status(JobStatus.QUEUED)
        job.exc_info = None
        q.enqueue_job(job)
    else:
        statsd.increment('worker_persistent_error',
                         tags=[
                             'job:%s' % job.func_name,
                             'error_type:%s' % exc_type,
                             'error:%s' % exc_value
                         ])
        log.error('PersistentError: not retrying', e=exc_value)


if __name__ == '__main__':
    with Connection():
        queue_names = [q.name]
        worker_name = str(uuid4())
        w = Worker(queue_names,
                   name=worker_name,
                   connection=redis_conn,
                   exception_handlers=[rq_error_handler])
        w.work()
示例#33
0
from redis import Redis
from rq import Queue, Worker

redis = Redis(host='redis', port=6379)
queue = Queue('model_prediction', connection=redis)

if __name__ == '__main__':
    print('Starting Worker')
    worker = Worker([queue], connection=redis, name='model_prediction')
    worker.work()
    print('Ending Worker')
示例#34
0
def get_worker(redis_url, redis_password, queue_name="job_scheduler_queue"):
    queue = redis_queue(redis_url, redis_password, queue_name)
    return Worker(
        queues=[queue],
        connection=queue.connection
    )
示例#35
0
#! /rq/vevn/bin/ python
import os

from rq import Queue, Connection, Worker
from redis import Redis

listen = ['default']  # listen ??
redis_url = 'redis://redis:6379'  # path to Redis url
redis_con = Redis.from_url(redis_url)  # connection for Redis server

#
# create a worker process to listen for queue tasks.
#
if __name__ == "__main__":
    # Provide queue names to listen to as arguments to this script,
    with Connection(redis_con):  # create connection
        worker = Worker(list(map(Queue, listen)))  # default queue
        worker.work()  # worker ??
示例#36
0
https://devcenter.heroku.com/articles/python-rq

See "Queuing jobs" section on how to enqueue jobs.
https://devcenter.heroku.com/articles/python-rq#queuing-jobs

The worker does not start by default after deployment.  It will be necessary
to start manually such as:
heroku ps:scale worker=N
where N is the number of workers that you want.
"""

import os

import redis
from rq import Connection
from rq import Queue
from rq import Worker


LISTEN = ['high', 'default', 'low']

REDIS_URL = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')

CONN = redis.from_url(REDIS_URL)


if __name__ == '__main__':
  with Connection(CONN):
    worker = Worker(map(Queue, LISTEN))
    worker.work()
示例#37
0
def worker(queues='default'):
    with Connection(redis_connection):
        w = Worker(queues)
        w.work()
示例#38
0
 def test_custom_job_class(self):
     """Ensure Worker accepts custom job class."""
     q = Queue()
     worker = Worker([q], job_class=CustomJob)
     self.assertEqual(worker.job_class, CustomJob)
示例#39
0

# set up bot logging
bot_logger = bot_log_setup()

if __name__ == '__main__':
    # parse cli arguments into cliargs dictionary
    cliargs_bot = vars(parse_cli_args())

    # Redis queue names
    listen = ['diskover_crawl']

    print("""\033[31m
    
     ___  _ ____ _  _ ____ _  _ ____ ____     ;
     |__> | ==== |-:_ [__]  \/  |=== |--<    ["]
     ____ ____ ____ _  _ _    ___  ____ ___ /[_]\\
     |___ |--< |--| |/\| |___ |==] [__]  |   ] [ v%s
     
     Redis RQ worker bot for diskover crawler
     Crawling all your stuff.

    \033[0m""" % (diskover.version))

    with Connection(redis_conn):
        w = Worker(listen)
        if cliargs_bot['burst']:
            w.work(burst=True)
        else:
            w.work()
示例#40
0
def run_worker(burst=False):
    w = Worker(queues=[os.environ.get("LCC_QUEUE_NAME", "lcc")],
               connection=connection)
    w.work(burst=burst)
示例#41
0
import sys

from redis import Redis
from rq import Queue, Connection, Worker

from social_relay.config import REDIS_HOST, REDIS_PORT, REDIS_DB


if __name__ == '__main__':
    # Provide queue names to listen to as arguments to this script,
    # similar to rqworker
    conn = Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
    with Connection(conn):
        qs = map(Queue, sys.argv[1:]) or [Queue()]
        w = Worker(qs)
        w.work()
示例#42
0
def runworker():
    redis_url = app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    with Connection(redis_connection):
        worker = Worker(app.config['QUEUES'])
        worker.work()
示例#43
0
def run_worker():
    redis_conn = Redis()
    with redis_conn:
        w = Worker("al_test", connection=redis_conn)
        w.work()
示例#44
0
文件: mail.py 项目: qyqx/tracim
    def take_action(self, parsed_args):
        super().take_action(parsed_args)

        with Connection():
            w = Worker(['mail_sender'])
            w.work()
示例#45
0
文件: test_worker.py 项目: tds333/rq
    def test_custom_exc_handling(self):
        """Custom exception handling."""
        def first_handler(job, *exc_info):
            job.meta = {'first_handler': True}
            job.save_meta()
            return True

        def second_handler(job, *exc_info):
            job.meta.update({'second_handler': True})
            job.save_meta()

        def black_hole(job, *exc_info):
            # Don't fall through to default behaviour (moving to failed queue)
            return False

        q = Queue()
        self.assertEqual(q.count, 0)
        job = q.enqueue(div_by_zero)

        w = Worker([q], exception_handlers=first_handler)
        w.work(burst=True)

        # Check the job
        job.refresh()
        self.assertEqual(job.is_failed, True)
        self.assertTrue(job.meta['first_handler'])

        job = q.enqueue(div_by_zero)
        w = Worker([q], exception_handlers=[first_handler, second_handler])
        w.work(burst=True)

        # Both custom exception handlers are run
        job.refresh()
        self.assertEqual(job.is_failed, True)
        self.assertTrue(job.meta['first_handler'])
        self.assertTrue(job.meta['second_handler'])

        job = q.enqueue(div_by_zero)
        w = Worker(
            [q],
            exception_handlers=[first_handler, black_hole, second_handler])
        w.work(burst=True)

        # second_handler is not run since it's interrupted by black_hole
        job.refresh()
        self.assertEqual(job.is_failed, True)
        self.assertTrue(job.meta['first_handler'])
        self.assertEqual(job.meta.get('second_handler'), None)
示例#46
0
    def test_worker_sets_result_ttl(self):
        """Ensure that Worker properly sets result_ttl for individual jobs."""
        q = Queue()
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=10)
        w = Worker([q])
        w.work(burst=True)
        self.assertNotEqual(self.testconn._ttl(job.key), 0)

        # Job with -1 result_ttl don't expire
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=-1)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn._ttl(job.key), -1)

        # Job with result_ttl = 0 gets deleted immediately
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=0)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn.get(job.key), None)
示例#47
0
import os
import redis
from rq import Worker, Queue, Connection

#this is the file that spins up a worker to process redis queues.

listen = ['in_twitter_queue']

redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')  #heroku only

conn = redis.from_url(redis_url)

if __name__ == '__main__':
    with Connection(conn):  #sets up connection
        worker = Worker(map(Queue, listen))
        worker.work()  #worker is spun up to listen to queue
示例#48
0
from django.contrib.contenttypes.models import ContentType
from django.test import override_settings
from django.urls import reverse
from django.utils.timezone import make_aware
from django_rq.queues import get_connection
from rest_framework import status
from rq import Worker

from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, RackGroup, RackRole, Site
from extras.api.views import ReportViewSet, ScriptViewSet
from extras.models import ConfigContext, CustomField, ExportTemplate, ImageAttachment, Tag
from extras.reports import Report
from extras.scripts import BooleanVar, IntegerVar, Script, StringVar
from utilities.testing import APITestCase, APIViewTestCases

rq_worker_running = Worker.count(get_connection('default'))


class AppTest(APITestCase):
    def test_root(self):

        url = reverse('extras-api:api-root')
        response = self.client.get('{}?format=api'.format(url), **self.header)

        self.assertEqual(response.status_code, 200)


class CustomFieldTest(APIViewTestCases.APIViewTestCase):
    model = CustomField
    brief_fields = ['id', 'name', 'url']
    create_data = [
示例#49
0
from __future__ import with_statement

import os
import redis
from rq import Worker, Queue, Connection

from config import REDIS_HOST
from config import REDIS_PORT
from config import REDIS_DB
from config import QUEUES_LISTEN

# Get redis url
redis_url = 'redis://' + REDIS_HOST + ':' + str(REDIS_PORT)

# Get redis connection
redis_conn = redis.from_url(redis_url)

# Get redis queue object for each listing
qH = Queue('high', connection=redis_conn)
qN = Queue('normal', connection=redis_conn)
qL = Queue('low', connection=redis_conn)

# Create redis database object
rDB = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)

if __name__ == '__main__':
    # start a worker
    with Connection(redis_conn):
        worker = Worker(map(Queue, QUEUES_LISTEN))
        worker.work()
示例#50
0
def get_workers():
	with Connection(get_redis_conn()):
		workers = Worker.all()
		return workers
示例#51
0
 def test_custom_queue_class(self):
     """Ensure Worker accepts custom queue class."""
     q = CustomQueue()
     worker = Worker([q], queue_class=CustomQueue)
     self.assertEqual(worker.queue_class, CustomQueue)
示例#52
0
    def test_statistics(self):
        """Successful and failed job counts are saved properly"""
        q = Queue()
        job = q.enqueue(div_by_zero)
        w = Worker([q])
        w.register_birth()

        self.assertEqual(w.failed_job_count, 0)
        self.assertEqual(w.successful_job_count, 0)
        self.assertEqual(w.total_working_time, 0)

        registry = StartedJobRegistry(connection=w.connection)
        job.started_at = utcnow()
        job.ended_at = job.started_at + timedelta(seconds=0.75)
        w.handle_job_failure(job)
        w.handle_job_success(job, q, registry)

        w.refresh()
        self.assertEqual(w.failed_job_count, 1)
        self.assertEqual(w.successful_job_count, 1)
        self.assertEqual(w.total_working_time,
                         1500000)  # 1.5 seconds in microseconds

        w.handle_job_failure(job)
        w.handle_job_success(job, q, registry)

        w.refresh()
        self.assertEqual(w.failed_job_count, 2)
        self.assertEqual(w.successful_job_count, 2)
        self.assertEqual(w.total_working_time, 3000000)
示例#53
0
import os

import redis
from rq import Worker, Queue, Connection
from rq.handlers import move_to_failed_queue

from exception_handler import post_to_db

listen = ['thumbnails']

# Heroku provides the env variable REDISTOGO_URL for Heroku RedisToGo;
# the default redis://redis_worker:6379 points to the local docker redis
redis_url = os.getenv('REDISTOGO_URL', 'redis://redis_worker:6379')
connection = redis.from_url(redis_url)

if __name__ == '__main__':
    with Connection(connection):
        worker = Worker(map(Queue, listen),
                        exception_handlers=[post_to_db, move_to_failed_queue])
        worker.work()
示例#54
0
    def test_find_by_key(self):
        """Worker.find_by_key restores queues, state and job_id."""
        queues = [Queue('foo'), Queue('bar')]
        w = Worker(queues)
        w.register_death()
        w.register_birth()
        w.set_state(WorkerStatus.STARTED)
        worker = Worker.find_by_key(w.key)
        self.assertEqual(worker.queues, queues)
        self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
        self.assertEqual(worker._job_id, None)
        self.assertTrue(worker.key in Worker.all_keys(worker.connection))

        # If worker is gone, its keys should also be removed
        worker.connection.delete(worker.key)
        Worker.find_by_key(worker.key)
        self.assertFalse(worker.key in Worker.all_keys(worker.connection))
示例#55
0
def start_worker():
    with Connection(conn):
        worker = Worker(queues=queue)
        worker.work()
示例#56
0
    def test_worker_all(self):
        """Worker.all() works properly"""
        foo_queue = Queue('foo')
        bar_queue = Queue('bar')

        w1 = Worker([foo_queue, bar_queue], name='w1')
        w1.register_birth()
        w2 = Worker([foo_queue], name='w2')
        w2.register_birth()

        self.assertEqual(set(Worker.all(connection=foo_queue.connection)),
                         set([w1, w2]))
        self.assertEqual(set(Worker.all(queue=foo_queue)), set([w1, w2]))
        self.assertEqual(set(Worker.all(queue=bar_queue)), set([w1]))

        w1.register_death()
        w2.register_death()
示例#57
0
文件: worker.py 项目: lee-ch/pyalerts
from __future__ import print_function, absolute_import
import os
import sys
import redis

import pyalerts
import pyalerts.jobs

from pyalerts.jobs import *
from pyalerts.scheduler.queue import conn, listen

from rq import Worker, Queue, Connection, get_current_job

if __name__ == '__main__':
    with Connection(conn):
        worker = Worker(list(map(Queue, listen)))
        worker.work()
示例#58
0
def work():
    redis_conn = Redis()
    parser = Worker(['analyzing'], connection=redis_conn)

    parser.work()
示例#59
0
def rq_worker():
    worker = Worker([task_manager], connection=redis)
    worker.work()
示例#60
0
def scrape_tree_meta(paths, cliargs, reindex_dict):
    worker = get_worker_name()
    tree_dirs = []
    tree_files = []
    if cliargs['qumulo']:
        qumulo = True
        from diskover_qumulo import qumulo_get_dir_meta, qumulo_get_file_meta
    else:
        qumulo = False
    totalcrawltime = 0

    # check if other bots are idle and throw them some jobs (dir paths)
    if len(paths) >= cliargs['batchsize']:
        workers_idle = 0
        workers = Worker.all(connection=redis_conn)
        num_workers = len(workers)
        for w in workers:
            if w._state == "idle":
                workers_idle += 1
            if workers_idle > num_workers // 2:
                workers_idle = True
                break
        q_len = len(q_crawl)
        if q_len == 0 and workers_idle == True:
            # take half the paths randomly
            shuffle(paths)
            n = len(paths) // 2
            tosspaths = paths[:n]
            paths = paths[n:]
            q_crawl.enqueue(scrape_tree_meta,
                            args=(
                                tosspaths,
                                cliargs,
                                reindex_dict,
                            ))

    for path in paths:
        starttime = time.time()
        root, dirs, files = path
        totaldirsize = 0
        totaldiritems_subdirs = len(dirs)
        totaldiritems_files = 0

        # check if stats embeded in data from diskover tree walk client
        if type(root) is tuple:
            statsembeded = True
        else:
            statsembeded = False
        if qumulo:
            if root['path'] != '/':
                root_path = root['path'].rstrip(os.path.sep)
            else:
                root_path = root['path']
            dmeta = qumulo_get_dir_meta(worker, root, cliargs, reindex_dict,
                                        redis_conn)
        else:
            if statsembeded:
                root_path = root[0]
                dmeta = get_dir_meta(worker,
                                     root,
                                     cliargs,
                                     reindex_dict,
                                     statsembeded=True)
            else:
                root_path = root
                dmeta = get_dir_meta(worker,
                                     root_path,
                                     cliargs,
                                     reindex_dict,
                                     statsembeded=False)
        if dmeta == "sametimes":
            # fetch meta data for directory and all it's files (doc sources) from index2 since
            # directory times haven't changed
            dir_source, files_source = get_metadata(root_path, cliargs)
            datenow = datetime.utcnow().isoformat()
            for file_source in files_source:
                # update indexed at time
                file_source['indexing_date'] = datenow
                # update worker name
                file_source['worker_name'] = worker
                tree_files.append(('file', file_source))
            if dir_source:
                # update indexed at time
                dir_source['indexing_date'] = datenow
                # update worker name
                dir_source['worker_name'] = worker
                # update crawl time
                elapsed = time.time() - starttime
                dir_source['crawl_time'] = round(elapsed, 6)
                tree_dirs.append(dir_source)
                totalcrawltime += elapsed
        # get meta off disk since times different in Redis than on disk
        elif dmeta:
            # check if meta for files embeded
            if statsembeded:
                for file in files:
                    fmeta = get_file_meta(worker,
                                          file,
                                          cliargs,
                                          reindex_dict,
                                          statsembeded=True)
                    if fmeta:
                        tree_files.append(fmeta)
                        # add file size to totaldirsize
                        totaldirsize += fmeta['filesize']
                        totaldiritems_files += 1
            else:
                for file in files:
                    if qumulo:
                        fmeta = qumulo_get_file_meta(worker, file, cliargs,
                                                     reindex_dict)
                    else:
                        fmeta = get_file_meta(worker,
                                              os.path.join(root_path, file),
                                              cliargs,
                                              reindex_dict,
                                              statsembeded=False)
                    if fmeta:
                        tree_files.append(fmeta)
                        # add file size to totaldirsize
                        totaldirsize += fmeta['filesize']
                        totaldiritems_files += 1

            # update crawl time
            elapsed = time.time() - starttime
            dmeta['crawl_time'] = round(elapsed, 6)
            # update directory meta filesize, items
            dmeta['filesize'] = totaldirsize
            dmeta['items_files'] = totaldiritems_files
            dmeta['items_subdirs'] = totaldiritems_subdirs
            totaldiritems = totaldiritems_files + totaldiritems_subdirs
            dmeta['items'] += totaldiritems
            tree_dirs.append(dmeta)
            totalcrawltime += elapsed

        # check if doc count is more than es chunksize and bulk add to es
        if len(tree_dirs) + len(tree_files) >= config['es_chunksize']:
            td = tree_dirs[:]
            tf = tree_files[:]
            es_bulk_add(worker, td, tf, cliargs, totalcrawltime)
            del tree_dirs[:]
            del tree_files[:]
            totalcrawltime = 0

    # bulk add to es
    if len(tree_dirs) > 0 or len(tree_files) > 0:
        es_bulk_add(worker, tree_dirs, tree_files, cliargs, totalcrawltime)