def test_add_job(self):
sid = uuid4().hex
queue = Queue(sid)
queue.add_job(Job(test_job, 'there'))
queue.add_job(Job(test_job, arg1='test', arg2='test'))
sleep(10)
self.assertEqual(queue.num_jobs(), 2)
queue.delete()
def test_remove_job(self):
sid = uuid4().hex
queue = Queue(sid)
queue.add_job(Job(test_job, 'there'))
queue.add_job(Job(test_job, arg1='test', arg2='test'))
for job in queue.jobs:
queue.remove_job(job)
sleep(10)
self.assertEqual(queue.num_jobs(), 0)
queue.delete()
def jobs(self):
"""
Iterate through all existing jobs in the cheapest and quickest possible
way.
By default we will:
- Use the maximum batch size to reduce calls to SQS. This will
reduce the cost of running the service, as less requests equals
less dollars.
- Wait for as long as possible (*20 seconds*) for a message to be
sent to us (*if none are in the queue already*). This way, we'll
reduce our total request count, and spend less dollars.
.. note::
This method is a generator which will continue to return results
until this SQS queue is emptied.
"""
total_jobs = self.num_jobs()
while total_jobs:
for message in self.queue.get_messages(
num_messages = self.BATCH_SIZE,
wait_time_seconds = self.WAIT_SECONDS,
):
yield Job.from_message(message)
total_jobs = self.num_jobs()
def jobs(self):
"""
Iterate through all existing jobs in the cheapest and quickest possible
way.
By default we will:
- Use the maximum batch size to reduce calls to SQS. This will
reduce the cost of running the service, as less requests equals
less dollars.
- Wait for as long as possible (*20 seconds*) for a message to be
sent to us (*if none are in the queue already*). This way, we'll
reduce our total request count, and spend less dollars.
.. note::
This method is a generator which will continue to return results
until this SQS queue is emptied.
"""
total_jobs = self.num_jobs()
while total_jobs:
for message in self.queue.get_messages(
num_messages=self.BATCH_SIZE,
wait_time_seconds=self.WAIT_SECONDS,
):
yield Job.from_message(message)
total_jobs = self.num_jobs()
def test_create_job_from_message(self):
message = Message(body=dumps({
'callable': random_job,
'args': (),
'kwargs': {},
}))
job = Job.from_message(message)
self.assertEqual(message.get_body(), job.message.get_body())
def test_create_job_from_message(self):
message = Message(body=dumps({
'callable': random_job,
'args': (),
'kwargs': {},
}))
job = Job.from_message(message)
self.assertEqual(message.get_body(), job.message.get_body())
def test_run(self):
job = Job(random_job, 'hi', arg2='there')
job.run()
self.assertTrue(job.run_time >= 1)
self.assertEqual(job.result, 'yo!')
job = Job(bad_job)
job.run()
self.assertIsInstance(job.exception, ZeroDivisionError)
def test_work(self):
sid = uuid4().hex
queue = Queue(sid)
queue.add_job(Job(test_job, 'hi', 'there'))
sleep(10)
self.assertEqual(queue.num_jobs(), 1)
worker = Worker([queue])
worker.work(True)
sleep(10)
self.assertEqual(queue.num_jobs(), 0)
queue.delete()
def test_run(self):
job = Job(random_job, 'hi', arg2='there')
job.run()
self.assertTrue(job.run_time >= 1)
self.assertEqual(job.result, 'yo!')
job = Job(bad_job)
job.run()
self.assertIsInstance(job.exception, ZeroDivisionError)
def run():
try:
job_id = str(uuid.uuid4())
uid = str(uuid.uuid4())
print("Creating queue")
q = Queue("simpleq_test")
job = Job(job_id, None, log_stuff, uid)
print(f"job_id: {job_id}")
print(f"uid: {uid}")
for n in range(10):
q.add_job(job, delay=5)
sleep(10)
print(f"q.num_jobs(): {q.num_jobs()}")
finally:
q.delete()
def jobs(self):
"""
Iterate through all existing jobs in the cheapest and quickest possible
way.
By default we will:
- Use the maximum batch size to reduce calls to SQS. This will
reduce the cost of running the service, as less requests equals
less dollars.
- Wait for as long as possible (*20 seconds*) for a message to be
sent to us (*if none are in the queue already*). This way, we'll
reduce our total request count, and spend less dollars.
.. note::
This method is a generator which will continue to return results
until this SQS queue is emptied.
"""
messages = self.queue.receive_messages(
AttributeNames=["All"],
MaxNumberOfMessages=self.BATCH_SIZE,
WaitTimeSeconds=self.WAIT_SECONDS,
MessageAttributeNames=["id"],
)
duplicate_job_groups = dict()
for message in messages:
job = Job.from_message(message)
if job.id is not None and job.id in duplicate_job_groups:
duplicate_job_groups[job.id].append(job)
continue
else:
duplicate_job_groups[job.id] = []
yield job
self._cleanup_duplicate_jobs(duplicate_job_groups)
def test_create_job(self):
job = Job(random_job, 'hi', arg2='there')
self.assertIsInstance(job, Job)
