def _mock_exchange_create_and_connect(self, verifier): self.mox.StubOutWithMock(verifier, 'exchange') self.verifier_with_notifications.exchange().AndReturn('exchange') self.mox.StubOutWithMock(message_service, 'create_exchange') exchange = self.mox.CreateMockAnything() message_service.create_exchange('exchange', 'topic', durable=True) \ .AndReturn(exchange) self.mox.StubOutWithMock(message_service, 'create_connection') conn = self.mox.CreateMockAnything() conn.__enter__().AndReturn(conn) conn.__exit__(None, None, None) message_service.create_connection(HOST, PORT, USERID, PASSWORD, "librabbitmq", VIRTUAL_HOST).AndReturn(conn)
def run(self): logger = _get_child_logger() self.run_startup() if self.enable_notifications: exchange_name = self.exchange() exchange = message_service.create_exchange( exchange_name, 'topic', durable=self.config.durable_queue()) routing_keys = self.config.topics()[exchange_name] with message_service.create_connection( self.config.host(), self.config.port(), self.config.userid(), self.config.password(), "librabbitmq", self.config.virtual_host()) as conn: def callback(result): attempt = 0 retry_limit = self.config.get_exponential_limit() while attempt < retry_limit: self.stats['timestamp'] = self._utcnow() try: (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) break except exceptions.ObjectDoesNotExist: if attempt < retry_limit-1: logger.warn("ObjectDoesNotExist in callback, " "attempting to reconnect and try " "again.") close_connection() reset_queries() else: logger.error("ObjectDoesNotExist in callback " "again, giving up.") # Avoiding unnecessary sleep() break except librabbitmq.ConnectionError as e: logger.error("ConnectionEror found while trying to connect to RabbitMQ. \ Attempting the {}th time.".format(attempt)) except Exception, e: msg = "ERROR in Callback %s: %s" % (exchange_name, e) logger.exception(msg) break attempt += 1 # Exponentially timed backoff time.sleep((2 ** attempt) / 1000.0 + (random.randint(0, 1000) / 1000.0)) self.stats['timestamp'] = self._utcnow() total = self.stats.get('total_processed', 0) + 1 self.stats['total_processed'] = total try: self._run(callback=callback) except Exception, e: msg = "ERROR during Verification %s: %s" % (exchange_name, e) logger.exception(msg) return True
def run(self): logger = _get_child_logger() self.run_startup() if self.enable_notifications: exchange_name = self.exchange() exchange = message_service.create_exchange( exchange_name, 'topic', durable=self.config.durable_queue()) routing_keys = self.config.topics()[exchange_name] with message_service.create_connection( self.config.host(), self.config.port(), self.config.userid(), self.config.password(), "librabbitmq", self.config.virtual_host()) as conn: def callback(result): attempt = 0 while attempt < 2: self.stats['timestamp'] = self._utcnow() try: (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) break except exceptions.ObjectDoesNotExist: if attempt < 1: logger.warn("ObjectDoesNotExist in callback, " "attempting to reconnect and try " "again.") close_connection() reset_queries() else: logger.error("ObjectDoesNotExist in callback " "again, giving up.") except Exception, e: msg = "ERROR in Callback %s: %s" % (exchange_name, e) logger.exception(msg) break attempt += 1 self.stats['timestamp'] = self._utcnow() total = self.stats.get('total_processed', 0) + 1 self.stats['total_processed'] = total try: self._run(callback=callback) except Exception, e: msg = "ERROR during Verification %s: %s" % (exchange_name, e) logger.exception(msg) return True
def scrub_with_notifications(args): print "!!!!!! WARNING: SENDING TO RABBIT !!!!!!" print "!!!!!! Sleeping for 30 seconds !!!!!!" print "!!!!!! before proceeding !!!!!!" time.sleep(30) with open(args.rabbit_config) as fp: rabbit_config = json.load(fp) exchange = msg.create_exchange(rabbit_config['exchange'], 'topic', durable=rabbit_config['durable_queue']) conn_conf = (rabbit_config['host'], rabbit_config['port'], rabbit_config['userid'], rabbit_config['password'], 'librabbitmq', rabbit_config['virtual_host']) with msg.create_connection(*conn_conf) as conn: def send_notif(notif): msg.send_notification(notif, rabbit_config['routing_key'], conn, exchange) count = scrub(args, send_notif=send_notif) return count
def run(self): if self.enable_notifications: exchange_name = self.exchange() exchange = message_service.create_exchange( exchange_name, 'topic', durable=self.config.durable_queue()) routing_keys = self.config.topics()[exchange_name] with message_service.create_connection( self.config.host(), self.config.port(), self.config.userid(), self.config.password(), "librabbitmq", self.config.virtual_host()) as conn: def callback(result): (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) try: self._run(callback=callback) except Exception, e: print e raise e
def run(self): logger = _get_child_logger() self.run_startup() if self.enable_notifications: exchange_name = self.exchange() exchange = message_service.create_exchange( exchange_name, 'topic', durable=self.config.durable_queue()) routing_keys = self.config.topics()[exchange_name] with message_service.create_connection( self.config.host(), self.config.port(), self.config.userid(), self.config.password(), "librabbitmq", self.config.virtual_host()) as conn: def callback(result): attempt = 0 retry_limit = self.config.get_exponential_limit() while attempt < retry_limit: self.stats['timestamp'] = self._utcnow() try: (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) break except exceptions.ObjectDoesNotExist: if attempt < retry_limit - 1: logger.warn("ObjectDoesNotExist in callback, " "attempting to reconnect and try " "again.") close_connection() reset_queries() else: logger.error("ObjectDoesNotExist in callback " "again, giving up.") # Avoiding unnecessary sleep() break except librabbitmq.ConnectionError as e: logger.error( "ConnectionEror found while trying to connect to RabbitMQ. \ Attempting the {}th time.".format( attempt)) except Exception, e: msg = "ERROR in Callback %s: %s" % (exchange_name, e) logger.exception(msg) break attempt += 1 # Exponentially timed backoff time.sleep((2**attempt) / 1000.0 + (random.randint(0, 1000) / 1000.0)) self.stats['timestamp'] = self._utcnow() total = self.stats.get('total_processed', 0) + 1 self.stats['total_processed'] = total try: self._run(callback=callback) except Exception, e: msg = "ERROR during Verification %s: %s" % (exchange_name, e) logger.exception(msg) return True