def setUp(self):
self.q = Queue()
# pre-call this so we don't need to use multi-process
# otherwise this is called within the classmethod
PQ.listen('default', ['default'])
# Fire off a notification of a fake job enqueued
self.q.notify(1)
def test_dequeue_any_multiple(self):
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEqual(queue, self.fooq)
self.assertEqual(job.func, say_hello)
self.assertEqual(job.origin, self.fooq.name)
self.assertEqual(
job.args[0], 'for Foo', 'Foo should be dequeued first.')
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEqual(queue, self.barq)
self.assertEqual(job.func, say_hello)
self.assertEqual(job.origin, self.barq.name)
self.assertEqual(
job.args[0], 'for Bar', 'Bar should be dequeued second.')
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
func = args[0]
args = args[1:]
async = not options.get('sync')
timeout = options.get('timeout')
queue = options.get('queue')
conn = options.get('conn')
if options['serial']:
queue = queue or 'serial'
q = SerialQueue.create(queue, connection=conn)
else:
queue = queue or 'default'
q = Queue.create(queue, connection=conn)
if timeout:
job = q.enqueue_call(func, args=args, timeout=timeout, async=async)
else:
job = q.enqueue_call(func, args=args, async=async)
if verbosity and job.id:
print('Job %i created' % job.id)
elif verbosity:
print('Job complete')
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
func = args[0]
args = args[1:]
async = not options.get('sync')
timeout = options.get('timeout')
queue = options.get('queue')
conn = options.get('conn')
if options['serial']:
queue = queue or 'serial'
q = SerialQueue.create(queue, connection=conn)
else:
queue = queue or 'default'
q = Queue.create(queue, connection=conn)
if timeout:
job = q.enqueue_call(func, args=args, timeout=timeout, async=async)
else:
job = q.enqueue_call(func, args=args, async=async)
if verbosity and job.id:
print('Job %i created' % job.id)
elif verbosity:
print('Job complete')
def test_quarantine_job(self):
"""Requeueing existing jobs."""
get_failed_queue().quarantine(
self.job, Exception('Some fake error')) # noqa
self.assertEqual(sorted(PQ.all()), sorted([get_failed_queue()])) # noqa
self.assertEqual(get_failed_queue().count, 1)
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from django.conf import settings
from pq.queue import Queue, SerialQueue
from pq.worker import Worker
sentry_dsn = options.get('sentry_dsn')
if not sentry_dsn:
sentry_dsn = settings.SENTRY_DSN if hasattr(settings, 'SENTRY_DSN') else None
verbosity = int(options.get('verbosity'))
queues = []
for queue in args:
q = Queue.objects.get(name=queue)
if q.serial:
queues.append(SerialQueue.create(name=queue))
else:
queues.append(Queue.create(name=queue))
w = Worker.create(queues, name=options.get('name'), connection=options['connection'])
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from pq.contrib.sentry import register_sentry
client = Client(sentry_dsn)
register_sentry(client, w)
w.work(burst=options['burst'])
def test_queue_creation_conflict_issue2(self):
"""Ordinary queue shouldn't ever become a serial queue"""
q = Queue.create()
self.assertFalse(q.serial)
q.enqueue(do_nothing)
self.q.enqueue(do_nothing)
w = Worker([q, self.q])
w.work(burst=True)
def test_queue_creation_conflict_issue2(self):
"""Ordinary queue shouldn't ever become a serial queue"""
q = Queue.create()
self.assertFalse(q.serial)
q.enqueue(do_nothing)
self.q.enqueue(do_nothing)
w = Worker([q, self.q])
w.work(burst=True)
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
func = args[1]
if args[0].lower() == 'now':
at = now()
else:
at = parser.parse(args[0])
if not at.tzinfo:
at = at.replace(tzinfo=get_default_timezone())
args = args[2:]
if options.get('mtwtf'):
weekdays = (0, 1, 2, 3, 4)
else:
weekdays = (
options.get('mo'),
options.get('tu'),
options.get('we'),
options.get('th'),
options.get('fr'),
options.get('sa'),
options.get('su'),
)
weekdays = [w for w in weekdays if isinstance(w, integer_types)]
timeout = options.get('timeout')
queue = options.get('queue')
conn = options.get('conn')
if options['serial']:
queue = queue or 'serial'
q = SerialQueue.create(queue, connection=conn, scheduled=True)
else:
queue = queue or 'default'
q = Queue.create(queue, connection=conn, scheduled=True)
job = q.schedule_call(
at, func,
args=args,
timeout=timeout,
repeat=options['repeat'],
interval=options['interval'],
between=options['between'],
weekdays=weekdays
)
if verbosity:
print('Job %i created' % job.id)
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
func = args[1]
if args[0].lower() == 'now':
at = now()
else:
at = parser.parse(args[0])
if not at.tzinfo:
at = at.replace(tzinfo=get_default_timezone())
args = args[2:]
if options.get('mtwtf'):
weekdays = (0,1,2,3,4)
else:
weekdays = (
options.get('mo'),
options.get('tu'),
options.get('we'),
options.get('th'),
options.get('fr'),
options.get('sa'),
options.get('su'),
)
weekdays = [w for w in weekdays if isinstance(w, integer_types)]
timeout = options.get('timeout')
queue = options.get('queue')
conn = options.get('conn')
if options['serial']:
queue = queue or 'serial'
q = SerialQueue.create(queue, connection=conn, scheduled=True)
else:
queue = queue or 'default'
q = Queue.create(queue, connection=conn, scheduled=True)
job = q.schedule_call(at, func, args=args,
timeout=timeout,
repeat=options['repeat'],
interval=options['interval'],
between=options['between'],
weekdays=weekdays
)
if verbosity:
print('Job %i created' % job.id)
def pq(function, *args, **kwargs):
'''
Adds ``function`` to the `django-pq <https://github.com/bretth/django-pq>`_
default queue, with any passed in
``args`` and ``kwargs`` It will create this queue if it doesn't exist.
'''
from pq.queue import Queue
queue = Queue.create()
# Try to create the queue table, but if the database isn't set up yet
# then don't create it
try:
queue.save()
except DatabaseError:
pass
queue.enqueue(function, *args, **kwargs)
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
timeout = options.get('timeout')
for queue in args:
if options['serial']:
q = SerialQueue.create(queue)
else:
q = Queue.create(queue)
q.connection = options.get('conn')
q.scheduled = options.get('scheduled')
if timeout:
q.default_timeout = timeout
q.save()
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
verbosity = int(options.get('verbosity', 1))
timeout = options.get('timeout')
for queue in args:
if options['serial']:
q = SerialQueue.create(queue)
else:
q = Queue.create(queue)
q.connection = options.get('conn')
q.scheduled = options.get('scheduled')
if timeout:
q.default_timeout = timeout
q.save()
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
func = args[1]
if args[0].lower() == 'now':
at = now()
else:
at = parser.parse(args[0])
if not at.tzinfo:
at = at.replace(tzinfo=get_default_timezone())
args = args[2:]
if options.get('mtwtf'):
weekdays = (0,1,2,3,4)
else:
weekdays = (
options.get('mo'),
options.get('tu'),
options.get('we'),
options.get('th'),
options.get('fr'),
options.get('sa'),
options.get('su'),
)
weekdays = [w for w in weekdays if w]
timeout = options.get('timeout')
queue = options.get('queue')
if options['serial']:
queue = queue or 'serial'
q = SerialQueue.create(queue)
else:
queue = queue or 'default'
q = Queue.create(queue)
q.schedule_call(at, func, args=args,
timeout=timeout,
repeat=options['repeat'],
interval=options['interval'],
between=options['between'],
weekdays=weekdays
)
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement
this method.
"""
from pq.queue import Queue, SerialQueue
func = args[0]
args = args[1:]
timeout = options.get("timeout")
queue = options.get("queue")
if options["serial"]:
queue = queue or "serial"
q = SerialQueue.create(queue)
else:
queue = queue or "default"
q = Queue.create(queue)
if timeout:
q.enqueue(func, *args, timeout=timeout)
else:
q.enqueue(func, *args)
def test_listen_for_jobs_select(self):
"""Test the 2nd part of the _listen_for_jobs method which
blocks and waits for postgresql to notify it"""
queue_name = PQ._listen_for_jobs(['default', 'farq'], 'default', 5)
self.assertEqual('farq', queue_name)
def test_validated_name(self, name):
with self.assertRaises(InvalidQueueName):
PQ.validated_name(name)
def test_listen_for_jobs(self):
"""Test the first part of the _listen_for_jobs method which polls
for notifications"""
queue_name = PQ._listen_for_jobs(['default'], 'default', 1)
self.assertEqual('default', queue_name)
def test_dequeue_any_serial_lock(self):
"""Test that it raises a DequeueTimeout timeout"""
with self.assertRaises(DequeueTimeout):
Queue.dequeue_any([self.sq], timeout=1)
def test_listen(self):
"""Postgresql LISTEN on channel with default connection"""
conn = PQ.listen('default', ['default'])
self.assertIsNotNone(conn)
def test_dequeue_timeout(self):
with self.assertRaises(DequeueTimeout):
PQ.dequeue_any([self.q], timeout=1)
def test_dequeue_any_single(self):
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEquals(job.func, say_hello)
self.assertEquals(queue, self.barq)
def test_dequeue_any_serial_lock(self):
"""Test that it raises a DequeueTimeout timeout"""
with self.assertRaises(DequeueTimeout):
Queue.dequeue_any([self.sq], timeout=1)
def test_dequeue_any_serial(self):
job, queue = Queue.dequeue_any([self.sq], timeout=10)
self.assertEquals(job.func, do_nothing)
def test_default_queue_create_multiple(self):
queue = Queue.create()
self.assertEqual(queue.name, 'default')
queue = SerialQueue.create()
self.assertEqual(queue.name, 'default (serial)')
def test_default_queue_create_multiple(self):
queue = Queue.create()
self.assertEqual(queue.name, 'default')
queue = SerialQueue.create()
self.assertEqual(queue.name, 'default (serial)')
def test_dequeue_any_serial(self):
job, queue = Queue.dequeue_any([self.sq], timeout=10)
self.assertEquals(job.func, do_nothing)
def test_dequeue_any_serial_lock_expired(self):
"""Test that it raises a DequeueTimeout timeout"""
time.sleep(1)
job, queue = Queue.dequeue_any([self.sq], timeout=1)
self.assertEquals(self.job.id, job.id)
def test_dequeue_any_serial_lock_expired(self):
"""Test that it raises a DequeueTimeout timeout"""
time.sleep(1)
job, queue = Queue.dequeue_any([self.sq], timeout=1)
self.assertEquals(self.job.id, job.id)
def test_dequeue_any_empty(self):
"""Fetching work from any given queue."""
self.assertEquals(PQ.dequeue_any([self.fooq, self.barq], None), None)