def test_add_job(self): sid = uuid4().hex queue = Queue(sid) queue.add_job(Job(test_job, 'there')) queue.add_job(Job(test_job, arg1='test', arg2='test')) sleep(10) self.assertEqual(queue.num_jobs(), 2) queue.delete()
def test_remove_job(self): sid = uuid4().hex queue = Queue(sid) queue.add_job(Job(test_job, 'there')) queue.add_job(Job(test_job, arg1='test', arg2='test')) for job in queue.jobs: queue.remove_job(job) sleep(10) self.assertEqual(queue.num_jobs(), 0) queue.delete()
def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ total_jobs = self.num_jobs() while total_jobs: for message in self.queue.get_messages( num_messages = self.BATCH_SIZE, wait_time_seconds = self.WAIT_SECONDS, ): yield Job.from_message(message) total_jobs = self.num_jobs()
def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ total_jobs = self.num_jobs() while total_jobs: for message in self.queue.get_messages( num_messages=self.BATCH_SIZE, wait_time_seconds=self.WAIT_SECONDS, ): yield Job.from_message(message) total_jobs = self.num_jobs()
def test_create_job_from_message(self): message = Message(body=dumps({ 'callable': random_job, 'args': (), 'kwargs': {}, })) job = Job.from_message(message) self.assertEqual(message.get_body(), job.message.get_body())
def test_run(self): job = Job(random_job, 'hi', arg2='there') job.run() self.assertTrue(job.run_time >= 1) self.assertEqual(job.result, 'yo!') job = Job(bad_job) job.run() self.assertIsInstance(job.exception, ZeroDivisionError)
def test_work(self): sid = uuid4().hex queue = Queue(sid) queue.add_job(Job(test_job, 'hi', 'there')) sleep(10) self.assertEqual(queue.num_jobs(), 1) worker = Worker([queue]) worker.work(True) sleep(10) self.assertEqual(queue.num_jobs(), 0) queue.delete()
def run(): try: job_id = str(uuid.uuid4()) uid = str(uuid.uuid4()) print("Creating queue") q = Queue("simpleq_test") job = Job(job_id, None, log_stuff, uid) print(f"job_id: {job_id}") print(f"uid: {uid}") for n in range(10): q.add_job(job, delay=5) sleep(10) print(f"q.num_jobs(): {q.num_jobs()}") finally: q.delete()
def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ messages = self.queue.receive_messages( AttributeNames=["All"], MaxNumberOfMessages=self.BATCH_SIZE, WaitTimeSeconds=self.WAIT_SECONDS, MessageAttributeNames=["id"], ) duplicate_job_groups = dict() for message in messages: job = Job.from_message(message) if job.id is not None and job.id in duplicate_job_groups: duplicate_job_groups[job.id].append(job) continue else: duplicate_job_groups[job.id] = [] yield job self._cleanup_duplicate_jobs(duplicate_job_groups)
def test_create_job(self): job = Job(random_job, 'hi', arg2='there') self.assertIsInstance(job, Job)