def queue(self): if not self._queue: self._queue = self.conn.sqs.create_queue(self.queue_name) self._queue.set_message_class(RawMessage) topic = SNSTopic(self.region, self.topic_name) topic.subscribe_sqs_queue(self.queue) return self._queue
def _setup_topic(self): if self._topic: return logger.debug("setting up topic %s", self._topic_name) self._topic = SNSTopic(self._conn, self._topic_name) logger.debug("subscribing queue %s to topic %s", self._queue_name, self._topic_name) self._topic.subscribe_sqs_queue(self._queue)
class AWSReactor(Reactor): def __init__(self, conn_mgr, topic_name, state_domain_name, queue_name, state_backend=SDBStateBackend): self._conn = conn_mgr self._topic_name = topic_name self._queue_name = queue_name self._topic = None self._queue = None self._state_backend = state_backend(conn_mgr.sdb, state_domain_name) super(AWSReactor, self).__init__() def _setup_queue(self): if self._queue: return logger.debug("setting up queue %s", self._queue_name) self._queue = self._conn.sqs.create_queue(self._queue_name) self._queue.set_message_class(RawMessage) def _setup_topic(self): if self._topic: return logger.debug("setting up topic %s", self._topic_name) self._topic = SNSTopic(self._conn, self._topic_name) logger.debug("subscribing queue %s to topic %s", self._queue_name, self._topic_name) self._topic.subscribe_sqs_queue(self._queue) def get_result(self, **kwargs): wait_time = kwargs.get('wait_time', 0) visibility_timeout = kwargs.get('visibility_timeout', None) self._setup_queue() self._setup_topic() logger.debug("Getting result from queue %s.", self._queue_name) result = self._queue.read(visibility_timeout=visibility_timeout, wait_time_seconds=wait_time) result_obj = None if result: result_message = json.loads(result.get_body())['Message'] result_dict = json.loads(result_message) result_obj = results.Result.deserialize(result_dict, origin=result) result_obj.validate() return result_obj
class SQSProbe(Probe): def __init__(self, conn_mgr, task_queue, results_topic, state_domain, state_backend=SDBStateBackend): self._conn = conn_mgr self._queue_name = task_queue self._topic_name = results_topic self._topic = None self._queue = None self._state_backend = state_backend(conn_mgr.sdb, state_domain) super(SQSProbe, self).__init__() def _setup_queue(self, **kwargs): if self._queue: return logger.debug("setting up queue %s", self._queue_name) self._queue = self._conn.sqs.create_queue(self._queue_name) def _setup_topic(self, **kwargs): if self._topic: return logger.debug("setting up topic %s", self._topic_name) self._topic = SNSTopic(self._conn, self._topic_name) def get_task(self, **kwargs): self._setup_queue(**kwargs) wait_time = kwargs.get('queue_wait_time') timeout = kwargs.get('monitor_timeout') + 3 logger.debug("Getting task from queue %s.", self._queue_name) task_item = self._queue.read(visibility_timeout=timeout, wait_time_seconds=wait_time) task = None if task_item: task = Task.deserialize(json.loads(task_item.get_body()), origin=task_item) return task def resubmit_task(self, task, delay, **kwargs): self._setup_queue(**kwargs) task.increment_attempt() logger.debug("Resubmitting task %s with %d second delay.", task.id, delay) m = Message() m.set_body(json.dumps(task.serialize())) return self._queue.write(m, delay_seconds=delay) def submit_result(self, result, **kwargs): self._setup_topic() logger.debug("%s - submitting '%s/%s' result", result.id, result.state_name, result.state_type_name) return self._topic.publish(json.dumps(result.serialize()))
def topic(self): if not self._topic: self._topic = SNSTopic(self.region, self.topic_name) return self._topic
def _setup_topic(self, **kwargs): if self._topic: return logger.debug("setting up topic %s", self._topic_name) self._topic = SNSTopic(self._conn, self._topic_name)