def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ total_jobs = self.num_jobs() while total_jobs: for message in self.queue.get_messages( num_messages = self.BATCH_SIZE, wait_time_seconds = self.WAIT_SECONDS, ): yield Job.from_message(message) total_jobs = self.num_jobs()
def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ total_jobs = self.num_jobs() while total_jobs: for message in self.queue.get_messages( num_messages=self.BATCH_SIZE, wait_time_seconds=self.WAIT_SECONDS, ): yield Job.from_message(message) total_jobs = self.num_jobs()
def test_create_job_from_message(self): message = Message(body=dumps({ 'callable': random_job, 'args': (), 'kwargs': {}, })) job = Job.from_message(message) self.assertEqual(message.get_body(), job.message.get_body())
def jobs(self): """ Iterate through all existing jobs in the cheapest and quickest possible way. By default we will: - Use the maximum batch size to reduce calls to SQS. This will reduce the cost of running the service, as less requests equals less dollars. - Wait for as long as possible (*20 seconds*) for a message to be sent to us (*if none are in the queue already*). This way, we'll reduce our total request count, and spend less dollars. .. note:: This method is a generator which will continue to return results until this SQS queue is emptied. """ messages = self.queue.receive_messages( AttributeNames=["All"], MaxNumberOfMessages=self.BATCH_SIZE, WaitTimeSeconds=self.WAIT_SECONDS, MessageAttributeNames=["id"], ) duplicate_job_groups = dict() for message in messages: job = Job.from_message(message) if job.id is not None and job.id in duplicate_job_groups: duplicate_job_groups[job.id].append(job) continue else: duplicate_job_groups[job.id] = [] yield job self._cleanup_duplicate_jobs(duplicate_job_groups)