def _mock_exchange_create_and_connect(self, verifier):
self.mox.StubOutWithMock(verifier, 'exchange')
self.verifier_with_notifications.exchange().AndReturn('exchange')
self.mox.StubOutWithMock(message_service, 'create_exchange')
exchange = self.mox.CreateMockAnything()
message_service.create_exchange('exchange', 'topic', durable=True) \
.AndReturn(exchange)
self.mox.StubOutWithMock(message_service, 'create_connection')
conn = self.mox.CreateMockAnything()
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None)
message_service.create_connection(HOST, PORT, USERID,
PASSWORD, "librabbitmq",
VIRTUAL_HOST).AndReturn(conn)
def _mock_exchange_create_and_connect(self, verifier):
self.mox.StubOutWithMock(verifier, 'exchange')
self.verifier_with_notifications.exchange().AndReturn('exchange')
self.mox.StubOutWithMock(message_service, 'create_exchange')
exchange = self.mox.CreateMockAnything()
message_service.create_exchange('exchange', 'topic', durable=True) \
.AndReturn(exchange)
self.mox.StubOutWithMock(message_service, 'create_connection')
conn = self.mox.CreateMockAnything()
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None)
message_service.create_connection(HOST, PORT, USERID, PASSWORD,
"librabbitmq",
VIRTUAL_HOST).AndReturn(conn)
def run(self):
logger = _get_child_logger()
self.run_startup()
if self.enable_notifications:
exchange_name = self.exchange()
exchange = message_service.create_exchange(
exchange_name, 'topic',
durable=self.config.durable_queue())
routing_keys = self.config.topics()[exchange_name]
with message_service.create_connection(
self.config.host(), self.config.port(),
self.config.userid(), self.config.password(),
"librabbitmq", self.config.virtual_host()) as conn:
def callback(result):
attempt = 0
retry_limit = self.config.get_exponential_limit()
while attempt < retry_limit:
self.stats['timestamp'] = self._utcnow()
try:
(verified, exist) = result
if verified:
self.send_verified_notification(
exist, conn, exchange,
routing_keys=routing_keys)
break
except exceptions.ObjectDoesNotExist:
if attempt < retry_limit-1:
logger.warn("ObjectDoesNotExist in callback, "
"attempting to reconnect and try "
"again.")
close_connection()
reset_queries()
else:
logger.error("ObjectDoesNotExist in callback "
"again, giving up.")
# Avoiding unnecessary sleep()
break
except librabbitmq.ConnectionError as e:
logger.error("ConnectionEror found while trying to connect to RabbitMQ. \
Attempting the {}th time.".format(attempt))
except Exception, e:
msg = "ERROR in Callback %s: %s" % (exchange_name,
e)
logger.exception(msg)
break
attempt += 1
# Exponentially timed backoff
time.sleep((2 ** attempt) / 1000.0 + (random.randint(0, 1000) / 1000.0))
self.stats['timestamp'] = self._utcnow()
total = self.stats.get('total_processed', 0) + 1
self.stats['total_processed'] = total
try:
self._run(callback=callback)
except Exception, e:
msg = "ERROR during Verification %s: %s" % (exchange_name,
e)
logger.exception(msg)
return True
def run(self):
logger = _get_child_logger()
self.run_startup()
if self.enable_notifications:
exchange_name = self.exchange()
exchange = message_service.create_exchange(
exchange_name, 'topic', durable=self.config.durable_queue())
routing_keys = self.config.topics()[exchange_name]
with message_service.create_connection(
self.config.host(),
self.config.port(), self.config.userid(),
self.config.password(), "librabbitmq",
self.config.virtual_host()) as conn:
def callback(result):
attempt = 0
while attempt < 2:
self.stats['timestamp'] = self._utcnow()
try:
(verified, exist) = result
if verified:
self.send_verified_notification(
exist,
conn,
exchange,
routing_keys=routing_keys)
break
except exceptions.ObjectDoesNotExist:
if attempt < 1:
logger.warn("ObjectDoesNotExist in callback, "
"attempting to reconnect and try "
"again.")
close_connection()
reset_queries()
else:
logger.error("ObjectDoesNotExist in callback "
"again, giving up.")
except Exception, e:
msg = "ERROR in Callback %s: %s" % (exchange_name,
e)
logger.exception(msg)
break
attempt += 1
self.stats['timestamp'] = self._utcnow()
total = self.stats.get('total_processed', 0) + 1
self.stats['total_processed'] = total
try:
self._run(callback=callback)
except Exception, e:
msg = "ERROR during Verification %s: %s" % (exchange_name,
e)
logger.exception(msg)
return True
def run(self):
logger = _get_child_logger()
self.run_startup()
if self.enable_notifications:
exchange_name = self.exchange()
exchange = message_service.create_exchange(
exchange_name, 'topic',
durable=self.config.durable_queue())
routing_keys = self.config.topics()[exchange_name]
with message_service.create_connection(
self.config.host(), self.config.port(),
self.config.userid(), self.config.password(),
"librabbitmq", self.config.virtual_host()) as conn:
def callback(result):
attempt = 0
while attempt < 2:
self.stats['timestamp'] = self._utcnow()
try:
(verified, exist) = result
if verified:
self.send_verified_notification(
exist, conn, exchange,
routing_keys=routing_keys)
break
except exceptions.ObjectDoesNotExist:
if attempt < 1:
logger.warn("ObjectDoesNotExist in callback, "
"attempting to reconnect and try "
"again.")
close_connection()
reset_queries()
else:
logger.error("ObjectDoesNotExist in callback "
"again, giving up.")
except Exception, e:
msg = "ERROR in Callback %s: %s" % (exchange_name,
e)
logger.exception(msg)
break
attempt += 1
self.stats['timestamp'] = self._utcnow()
total = self.stats.get('total_processed', 0) + 1
self.stats['total_processed'] = total
try:
self._run(callback=callback)
except Exception, e:
msg = "ERROR during Verification %s: %s" % (exchange_name,
e)
logger.exception(msg)
return True
def scrub_with_notifications(args):
print "!!!!!! WARNING: SENDING TO RABBIT !!!!!!"
print "!!!!!! Sleeping for 30 seconds !!!!!!"
print "!!!!!! before proceeding !!!!!!"
time.sleep(30)
with open(args.rabbit_config) as fp:
rabbit_config = json.load(fp)
exchange = msg.create_exchange(rabbit_config['exchange'],
'topic',
durable=rabbit_config['durable_queue'])
conn_conf = (rabbit_config['host'], rabbit_config['port'],
rabbit_config['userid'], rabbit_config['password'],
'librabbitmq', rabbit_config['virtual_host'])
with msg.create_connection(*conn_conf) as conn:
def send_notif(notif):
msg.send_notification(notif, rabbit_config['routing_key'],
conn, exchange)
count = scrub(args, send_notif=send_notif)
return count
def scrub_with_notifications(args):
print "!!!!!! WARNING: SENDING TO RABBIT !!!!!!"
print "!!!!!! Sleeping for 30 seconds !!!!!!"
print "!!!!!! before proceeding !!!!!!"
time.sleep(30)
with open(args.rabbit_config) as fp:
rabbit_config = json.load(fp)
exchange = msg.create_exchange(rabbit_config['exchange'],
'topic',
durable=rabbit_config['durable_queue'])
conn_conf = (rabbit_config['host'], rabbit_config['port'],
rabbit_config['userid'], rabbit_config['password'],
'librabbitmq', rabbit_config['virtual_host'])
with msg.create_connection(*conn_conf) as conn:
def send_notif(notif):
msg.send_notification(notif, rabbit_config['routing_key'],
conn, exchange)
count = scrub(args, send_notif=send_notif)
return count
def run(self):
if self.enable_notifications:
exchange_name = self.exchange()
exchange = message_service.create_exchange(
exchange_name, 'topic',
durable=self.config.durable_queue())
routing_keys = self.config.topics()[exchange_name]
with message_service.create_connection(
self.config.host(), self.config.port(),
self.config.userid(), self.config.password(),
"librabbitmq", self.config.virtual_host()) as conn:
def callback(result):
(verified, exist) = result
if verified:
self.send_verified_notification(
exist, conn, exchange, routing_keys=routing_keys)
try:
self._run(callback=callback)
except Exception, e:
print e
raise e
def run(self):
logger = _get_child_logger()
self.run_startup()
if self.enable_notifications:
exchange_name = self.exchange()
exchange = message_service.create_exchange(
exchange_name, 'topic', durable=self.config.durable_queue())
routing_keys = self.config.topics()[exchange_name]
with message_service.create_connection(
self.config.host(),
self.config.port(), self.config.userid(),
self.config.password(), "librabbitmq",
self.config.virtual_host()) as conn:
def callback(result):
attempt = 0
retry_limit = self.config.get_exponential_limit()
while attempt < retry_limit:
self.stats['timestamp'] = self._utcnow()
try:
(verified, exist) = result
if verified:
self.send_verified_notification(
exist,
conn,
exchange,
routing_keys=routing_keys)
break
except exceptions.ObjectDoesNotExist:
if attempt < retry_limit - 1:
logger.warn("ObjectDoesNotExist in callback, "
"attempting to reconnect and try "
"again.")
close_connection()
reset_queries()
else:
logger.error("ObjectDoesNotExist in callback "
"again, giving up.")
# Avoiding unnecessary sleep()
break
except librabbitmq.ConnectionError as e:
logger.error(
"ConnectionEror found while trying to connect to RabbitMQ. \
Attempting the {}th time.".format(
attempt))
except Exception, e:
msg = "ERROR in Callback %s: %s" % (exchange_name,
e)
logger.exception(msg)
break
attempt += 1
# Exponentially timed backoff
time.sleep((2**attempt) / 1000.0 +
(random.randint(0, 1000) / 1000.0))
self.stats['timestamp'] = self._utcnow()
total = self.stats.get('total_processed', 0) + 1
self.stats['total_processed'] = total
try:
self._run(callback=callback)
except Exception, e:
msg = "ERROR during Verification %s: %s" % (exchange_name,
e)
logger.exception(msg)
return True
