def postprocess_job(klass, job, state):
if job.uuid in klass.current_footprints:
del klass.current_footprints[job.uuid]
if state == 'failed':
transaction_id, platefile_id = klass.get_plate_info(job.output)
if transaction_id and platefile_id:
idx_transaction_failed = {
'platefile_id': platefile_id,
'transaction_id': transaction_id
}
request = {
'sequence_number':
0,
'requestor':
'',
'method':
'IndexTransactionFailed',
'payload':
protocols.pack(protobuf.IndexTransactionFailed,
idx_transaction_failed),
}
msg = Message(
protocols.pack(protobuf.BroxtonRequestWrapper, request))
klass.messagebus.basic_publish(msg,
exchange='ngt.platefile.index',
routing_key='index')
logger.debug("Message published: " + str(msg))
return job
def test_with(self):
with Connection(**settings.connect_args) as conn:
self.assertEqual(conn.transport is None, False)
with conn.channel(1) as ch:
self.assertEqual(1 in conn.channels, True)
#
# Do something with the channel
#
ch.access_request('/data', active=True, write=True)
ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True)
msg = Message('unittest message',
content_type='text/plain',
application_headers={'foo': 7, 'bar': 'baz'})
ch.basic_publish(msg, 'unittest.fanout')
#
# check that the channel was closed
#
self.assertEqual(1 in conn.channels, False)
self.assertEqual(ch.is_open, False)
#
# Check that the connection was closed
#
self.assertEqual(conn.transport, None)
def save_friends(self, twitter_id, msg, hdrs, next, correlation_id):
tw_profiles = TwitterProfile.objects.active().filter(
appuser_id=unicode(twitter_id))
if not tw_profiles:
return None
tw_profile = tw_profiles[0]
t2 = TwitterFriend._meta.db_table
twitter_friends = TwitterProfile.objects.active().extra(
tables=[t2],
where=[
'appuser_id=%s.friend_id' % t2,
'%s.twitter_profile_id=%%s' % t2
],
params=[tw_profile.pk]).distinct()
_log.debug("Twitter friends: %s", twitter_friends[:15])
for p in tw_profiles:
self._save_site_friends(p.user_profile, twitter_friends)
final_reply_to = hdrs.get('final_reply_to', next)
if final_reply_to:
hdrs['next'] = final_reply_to
hdrs['user_profile_id'] = tw_profile.user_profile.pk
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply # this will be sent to the exchange/queue to which next points
return None
def postprocess(self):
if self.job.uuid in self.current_footprints:
del self.current_footprints[self.job.uuid]
if self.job.status == 'failed':
transaction_id, platefile_id = self._get_plate_info(
self.job.output)
if transaction_id and platefile_id:
idx_transaction_failed = {
'platefile_id': platefile_id,
'transaction_id': transaction_id
}
request = {
'sequence_number':
0,
'requestor':
'',
'method':
'TransactionFailed',
'payload':
protocols.pack(protobuf.IndexTransactionFailed,
idx_transaction_failed),
}
msg = Message(
protocols.pack(protobuf.BroxtonRequestWrapper, request))
self.messagebus.basic_publish(msg,
exchange='ngt.platefile.index_0',
routing_key='index')
return self.job
def search(self, twitter_id, msg, hdrs, next, correlation_id):
cache_key = hdrs.get('cache_key', None)
if cache_key:
cache.cache.delete(cache_key)
ts = msg.properties.get('timestamp', None)
if ts and ts < (datetime.now() - timedelta(seconds=3600 * 5)):
_log.debug("Skipping stale search request for id %s", twitter_id)
return None # Don't process stale requests
params = msg.body
tweets = utils.search(params)
_log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id,
msg.properties)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
data = {'timestamp': time.time()}
data['results'] = tweets
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
_log.debug("Sending search results for %s to %s", twitter_id, next)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply
return None
def get_followees(self, twitter_id, msg, hdrs, next, correlation_id):
access_token = hdrs.get('access_token', '')
followees = utils.get_followees(twitter_id, access_token=access_token)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
_log.debug(
"Followee data lost. Skipping friendship computation for twitter_id %s",
twitter_id)
return None
data['followees'] = followees
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=
'twitter/twitter.compute.save_friends', # the function will be asked to send its reply here
)
return reply # this will be sent to the exchange/queue to which next points
return None
def test_defaults(self):
"""
Test how a queue defaults to being bound to an AMQP default
exchange, and how publishing defaults to the default exchange, and
basic_get defaults to getting from the most recently declared queue,
and queue_delete defaults to deleting the most recently declared
queue.
"""
self.ch.access_request('/data', active=True, write=True, read=True)
msg = Message('unittest message',
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
qname, _, _ = self.ch.queue_declare()
self.ch.basic_publish(msg, routing_key=qname)
msg2 = self.ch.basic_get(no_ack=True)
self.assertEqual(msg, msg2)
n = self.ch.queue_purge()
self.assertEqual(n, 0)
n = self.ch.queue_delete()
self.assertEqual(n, 0)
def test_publish(self):
tkt = self.ch.access_request('/data', active=True, write=True)
self.assertEqual(tkt, self.ch.default_ticket)
self.ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True)
msg = Message('unittest message',
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
self.ch.basic_publish(msg, 'unittest.fanout')
def test_invalid_header(self):
"""
Test sending a message with an unserializable object in the header
http://code.google.com/p/py-amqplib/issues/detail?id=17
"""
self.ch.access_request('/data', active=True, write=True, read=True)
qname, _, _ = self.ch.queue_declare()
msg = Message(application_headers={'test': None})
self.assertRaises(ValueError,
self.ch.basic_publish,
msg,
routing_key=qname)
def test_queue(self):
self.ch.access_request('/data', active=True, write=True, read=True)
my_routing_key = 'unittest.test_queue'
msg = Message('unittest message',
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
qname, _, _ = self.ch.queue_declare()
self.ch.queue_bind(qname, 'amq.direct', routing_key=my_routing_key)
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg, msg2)
def test_large(self):
"""
Test sending some extra large messages.
"""
self.ch.access_request('/data', active=True, write=True, read=True)
qname, _, _ = self.ch.queue_declare()
for multiplier in [100, 1000, 10000]:
msg = Message('unittest message' * multiplier,
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
self.ch.basic_publish(msg, routing_key=qname)
msg2 = self.ch.basic_get(no_ack=True)
self.assertEqual(msg, msg2)
def get_friends(self, twitter_id, msg, hdrs, next, correlation_id):
ds = DataStorage()
data = ds[correlation_id]
if data is None:
_log.debug(
"Follower/Followee data lost. Skipping friendship computation for twitter_id %s",
twitter_id)
return None
if 'followers' not in data or 'followees' not in data:
_log.debug(
"Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s",
twitter_id)
del ds[correlation_id]
ds.close()
return None
followers = data['followers']
followees = data['followees']
del ds[correlation_id]
ds.close()
try:
tw_profiles = TwitterProfile.objects.active().filter(
appuser_id=unicode(twitter_id)).order_by('-pk')
tw_profile = tw_profiles[:1].get()
except TwitterProfile.DoesNotExist:
return None
friends = utils.get_friends(twitter_id, followees, followers)
_log.debug("Twitter friends %s...", list(friends)[:5])
self._save_twitter_friends(tw_profiles, friends)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply # this will be sent to the exchange/queue to which next points
return None
def _publish(n, Connection, Message, name):
props = {"delivery_mode": DM}
kwargs = {"properties": props}
if "amqplib" in repr(Message):
kwargs = props
_cleanup(Connection, name)
conn = Connection()
chan = conn.channel()
chan = _declare(chan, name)
if name == 'librabbitmq':
_Message = lambda: (serialize(MESSAGE), props)
else:
_Message = lambda: Message(serialize(MESSAGE), **kwargs)
start = time()
for i in xrange(n + 2000):
if not i % 10000:
print(i)
message = _Message()
chan.basic_publish(message, exchange=name, routing_key=name)
print(time() - start)
def command_handler(msg):
""" Unpack a message and process commands
Speaks the RpcRequest protocol.
"""
global command_request_count
command_request_count += 1
mb.basic_ack(msg.delivery_tag)
#cmd = protocols.unpack(protocols.Command, msg.body)
request = protocols.unpack(protobuf.RpcRequestWrapper, msg.body)
logger.debug("command_handler got a command: %s" % str(request.method))
response = dotdict()
response.sequence_number = request.sequence_number
if request.method in command_map:
t0 = datetime.now()
try:
response.payload = globals()[command_map[request.method]](request.payload)
response.error = False
except Exception as e:
logger.error("Error in command '%s': %s %s" % (request.method, type(e), str(e.args)))
# TODO: send a stack trace.
traceback.print_tb(sys.exc_info()[2]) # traceback
response.payload = ''
response.error = True
response.error_string = str(e) or ''
t1 = datetime.now()
logger.info("COMMAND %s finished in %s." % (request.method, str(t1-t0)))
else:
logger.error("Invalid Command: %s" % request.method)
response.payload = ''
response.error = True
response.error_string = "Invalid Command: %s" % request.method
#mb.basic_ack(msg.delivery_tag)
response_bytes = protocols.pack(protobuf.RpcResponseWrapper, response)
mb.basic_publish(Message(response_bytes), routing_key=request.requestor)
def test_basic_return(self):
self.ch.access_request('/data', active=True, write=True)
self.ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True)
msg = Message('unittest message',
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
self.ch.basic_publish(msg, 'unittest.fanout')
self.ch.basic_publish(msg, 'unittest.fanout', immediate=True)
self.ch.basic_publish(msg, 'unittest.fanout', mandatory=True)
self.ch.basic_publish(msg,
'unittest.fanout',
immediate=True,
mandatory=True)
self.ch.close()
#
# 3 of the 4 messages we sent should have been returned
#
self.assertEqual(self.ch.returned_messages.qsize(), 3)
def test_encoding(self):
self.ch.access_request('/data', active=True, write=True, read=True)
my_routing_key = 'unittest.test_queue'
qname, _, _ = self.ch.queue_declare()
self.ch.queue_bind(qname, 'amq.direct', routing_key=my_routing_key)
#
# No encoding, body passed through unchanged
#
msg = Message('hello world')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertFalse(hasattr(msg2, 'content_encoding'))
self.assertTrue(isinstance(msg2.body, str))
self.assertEqual(msg2.body, 'hello world')
#
# Default UTF-8 encoding of unicode body, returned as unicode
#
msg = Message(u'hello world')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'UTF-8')
self.assertTrue(isinstance(msg2.body, unicode))
self.assertEqual(msg2.body, u'hello world')
#
# Explicit latin1 encoding, still comes back as unicode
#
msg = Message(u'hello world', content_encoding='latin1')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'latin1')
self.assertTrue(isinstance(msg2.body, unicode))
self.assertEqual(msg2.body, u'hello world')
#
# Plain string with specified encoding comes back as unicode
#
msg = Message('hello w\xf6rld', content_encoding='latin1')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'latin1')
self.assertTrue(isinstance(msg2.body, unicode))
self.assertEqual(msg2.body, u'hello w\u00f6rld')
#
# Plain string with bogus encoding
#
msg = Message('hello w\xf6rld', content_encoding='I made this up')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'I made this up')
self.assertTrue(isinstance(msg2.body, str))
self.assertEqual(msg2.body, 'hello w\xf6rld')
#
# Turn off auto_decode for remaining tests
#
self.ch.auto_decode = False
#
# Unicode body comes back as utf-8 encoded str
#
msg = Message(u'hello w\u00f6rld')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'UTF-8')
self.assertTrue(isinstance(msg2.body, str))
self.assertEqual(msg2.body, 'hello w\xc3\xb6rld')
#
# Plain string with specified encoding stays plain string
#
msg = Message('hello w\xf6rld', content_encoding='latin1')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'latin1')
self.assertTrue(isinstance(msg2.body, str))
self.assertEqual(msg2.body, 'hello w\xf6rld')
#
# Explicit latin1 encoding, comes back as str
#
msg = Message(u'hello w\u00f6rld', content_encoding='latin1')
self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key)
msg2 = self.ch.basic_get(qname, no_ack=True)
self.assertEqual(msg2.content_encoding, 'latin1')
self.assertTrue(isinstance(msg2.body, str))
self.assertEqual(msg2.body, 'hello w\xf6rld')
