def postprocess_job(klass, job, state): if job.uuid in klass.current_footprints: del klass.current_footprints[job.uuid] if state == 'failed': transaction_id, platefile_id = klass.get_plate_info(job.output) if transaction_id and platefile_id: idx_transaction_failed = { 'platefile_id': platefile_id, 'transaction_id': transaction_id } request = { 'sequence_number': 0, 'requestor': '', 'method': 'IndexTransactionFailed', 'payload': protocols.pack(protobuf.IndexTransactionFailed, idx_transaction_failed), } msg = Message( protocols.pack(protobuf.BroxtonRequestWrapper, request)) klass.messagebus.basic_publish(msg, exchange='ngt.platefile.index', routing_key='index') logger.debug("Message published: " + str(msg)) return job
def test_with(self): with Connection(**settings.connect_args) as conn: self.assertEqual(conn.transport is None, False) with conn.channel(1) as ch: self.assertEqual(1 in conn.channels, True) # # Do something with the channel # ch.access_request('/data', active=True, write=True) ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True) msg = Message('unittest message', content_type='text/plain', application_headers={'foo': 7, 'bar': 'baz'}) ch.basic_publish(msg, 'unittest.fanout') # # check that the channel was closed # self.assertEqual(1 in conn.channels, False) self.assertEqual(ch.is_open, False) # # Check that the connection was closed # self.assertEqual(conn.transport, None)
def save_friends(self, twitter_id, msg, hdrs, next, correlation_id): tw_profiles = TwitterProfile.objects.active().filter( appuser_id=unicode(twitter_id)) if not tw_profiles: return None tw_profile = tw_profiles[0] t2 = TwitterFriend._meta.db_table twitter_friends = TwitterProfile.objects.active().extra( tables=[t2], where=[ 'appuser_id=%s.friend_id' % t2, '%s.twitter_profile_id=%%s' % t2 ], params=[tw_profile.pk]).distinct() _log.debug("Twitter friends: %s", twitter_friends[:15]) for p in tw_profiles: self._save_site_friends(p.user_profile, twitter_friends) final_reply_to = hdrs.get('final_reply_to', next) if final_reply_to: hdrs['next'] = final_reply_to hdrs['user_profile_id'] = tw_profile.user_profile.pk reply = Message( '', delivery_mode=self.delivery_mode, content_type='text/plain', correlation_id=correlation_id, application_headers=hdrs, reply_to=None, ) return reply # this will be sent to the exchange/queue to which next points return None
def postprocess(self): if self.job.uuid in self.current_footprints: del self.current_footprints[self.job.uuid] if self.job.status == 'failed': transaction_id, platefile_id = self._get_plate_info( self.job.output) if transaction_id and platefile_id: idx_transaction_failed = { 'platefile_id': platefile_id, 'transaction_id': transaction_id } request = { 'sequence_number': 0, 'requestor': '', 'method': 'TransactionFailed', 'payload': protocols.pack(protobuf.IndexTransactionFailed, idx_transaction_failed), } msg = Message( protocols.pack(protobuf.BroxtonRequestWrapper, request)) self.messagebus.basic_publish(msg, exchange='ngt.platefile.index_0', routing_key='index') return self.job
def search(self, twitter_id, msg, hdrs, next, correlation_id): cache_key = hdrs.get('cache_key', None) if cache_key: cache.cache.delete(cache_key) ts = msg.properties.get('timestamp', None) if ts and ts < (datetime.now() - timedelta(seconds=3600 * 5)): _log.debug("Skipping stale search request for id %s", twitter_id) return None # Don't process stale requests params = msg.body tweets = utils.search(params) _log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id, msg.properties) ds = DataStorage() data = ds[correlation_id] if data is None: data = {'timestamp': time.time()} data['results'] = tweets data['timestamp'] = time.time() ds[correlation_id] = data ds.close() _log.debug("Sending search results for %s to %s", twitter_id, next) if next: hdrs['next'] = next reply = Message( '', delivery_mode=self.delivery_mode, content_type='text/plain', correlation_id=correlation_id, application_headers=hdrs, reply_to=None, ) return reply return None
def get_followees(self, twitter_id, msg, hdrs, next, correlation_id): access_token = hdrs.get('access_token', '') followees = utils.get_followees(twitter_id, access_token=access_token) ds = DataStorage() data = ds[correlation_id] if data is None: _log.debug( "Followee data lost. Skipping friendship computation for twitter_id %s", twitter_id) return None data['followees'] = followees data['timestamp'] = time.time() ds[correlation_id] = data ds.close() if next: hdrs['next'] = next reply = Message( '', delivery_mode=self.delivery_mode, content_type='text/plain', correlation_id=correlation_id, application_headers=hdrs, reply_to= 'twitter/twitter.compute.save_friends', # the function will be asked to send its reply here ) return reply # this will be sent to the exchange/queue to which next points return None
def test_defaults(self): """ Test how a queue defaults to being bound to an AMQP default exchange, and how publishing defaults to the default exchange, and basic_get defaults to getting from the most recently declared queue, and queue_delete defaults to deleting the most recently declared queue. """ self.ch.access_request('/data', active=True, write=True, read=True) msg = Message('unittest message', content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) qname, _, _ = self.ch.queue_declare() self.ch.basic_publish(msg, routing_key=qname) msg2 = self.ch.basic_get(no_ack=True) self.assertEqual(msg, msg2) n = self.ch.queue_purge() self.assertEqual(n, 0) n = self.ch.queue_delete() self.assertEqual(n, 0)
def test_publish(self): tkt = self.ch.access_request('/data', active=True, write=True) self.assertEqual(tkt, self.ch.default_ticket) self.ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True) msg = Message('unittest message', content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) self.ch.basic_publish(msg, 'unittest.fanout')
def test_invalid_header(self): """ Test sending a message with an unserializable object in the header http://code.google.com/p/py-amqplib/issues/detail?id=17 """ self.ch.access_request('/data', active=True, write=True, read=True) qname, _, _ = self.ch.queue_declare() msg = Message(application_headers={'test': None}) self.assertRaises(ValueError, self.ch.basic_publish, msg, routing_key=qname)
def test_queue(self): self.ch.access_request('/data', active=True, write=True, read=True) my_routing_key = 'unittest.test_queue' msg = Message('unittest message', content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) qname, _, _ = self.ch.queue_declare() self.ch.queue_bind(qname, 'amq.direct', routing_key=my_routing_key) self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg, msg2)
def test_large(self): """ Test sending some extra large messages. """ self.ch.access_request('/data', active=True, write=True, read=True) qname, _, _ = self.ch.queue_declare() for multiplier in [100, 1000, 10000]: msg = Message('unittest message' * multiplier, content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) self.ch.basic_publish(msg, routing_key=qname) msg2 = self.ch.basic_get(no_ack=True) self.assertEqual(msg, msg2)
def get_friends(self, twitter_id, msg, hdrs, next, correlation_id): ds = DataStorage() data = ds[correlation_id] if data is None: _log.debug( "Follower/Followee data lost. Skipping friendship computation for twitter_id %s", twitter_id) return None if 'followers' not in data or 'followees' not in data: _log.debug( "Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s", twitter_id) del ds[correlation_id] ds.close() return None followers = data['followers'] followees = data['followees'] del ds[correlation_id] ds.close() try: tw_profiles = TwitterProfile.objects.active().filter( appuser_id=unicode(twitter_id)).order_by('-pk') tw_profile = tw_profiles[:1].get() except TwitterProfile.DoesNotExist: return None friends = utils.get_friends(twitter_id, followees, followers) _log.debug("Twitter friends %s...", list(friends)[:5]) self._save_twitter_friends(tw_profiles, friends) if next: hdrs['next'] = next reply = Message( '', delivery_mode=self.delivery_mode, content_type='text/plain', correlation_id=correlation_id, application_headers=hdrs, reply_to=None, ) return reply # this will be sent to the exchange/queue to which next points return None
def _publish(n, Connection, Message, name): props = {"delivery_mode": DM} kwargs = {"properties": props} if "amqplib" in repr(Message): kwargs = props _cleanup(Connection, name) conn = Connection() chan = conn.channel() chan = _declare(chan, name) if name == 'librabbitmq': _Message = lambda: (serialize(MESSAGE), props) else: _Message = lambda: Message(serialize(MESSAGE), **kwargs) start = time() for i in xrange(n + 2000): if not i % 10000: print(i) message = _Message() chan.basic_publish(message, exchange=name, routing_key=name) print(time() - start)
def command_handler(msg): """ Unpack a message and process commands Speaks the RpcRequest protocol. """ global command_request_count command_request_count += 1 mb.basic_ack(msg.delivery_tag) #cmd = protocols.unpack(protocols.Command, msg.body) request = protocols.unpack(protobuf.RpcRequestWrapper, msg.body) logger.debug("command_handler got a command: %s" % str(request.method)) response = dotdict() response.sequence_number = request.sequence_number if request.method in command_map: t0 = datetime.now() try: response.payload = globals()[command_map[request.method]](request.payload) response.error = False except Exception as e: logger.error("Error in command '%s': %s %s" % (request.method, type(e), str(e.args))) # TODO: send a stack trace. traceback.print_tb(sys.exc_info()[2]) # traceback response.payload = '' response.error = True response.error_string = str(e) or '' t1 = datetime.now() logger.info("COMMAND %s finished in %s." % (request.method, str(t1-t0))) else: logger.error("Invalid Command: %s" % request.method) response.payload = '' response.error = True response.error_string = "Invalid Command: %s" % request.method #mb.basic_ack(msg.delivery_tag) response_bytes = protocols.pack(protobuf.RpcResponseWrapper, response) mb.basic_publish(Message(response_bytes), routing_key=request.requestor)
def test_basic_return(self): self.ch.access_request('/data', active=True, write=True) self.ch.exchange_declare('unittest.fanout', 'fanout', auto_delete=True) msg = Message('unittest message', content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) self.ch.basic_publish(msg, 'unittest.fanout') self.ch.basic_publish(msg, 'unittest.fanout', immediate=True) self.ch.basic_publish(msg, 'unittest.fanout', mandatory=True) self.ch.basic_publish(msg, 'unittest.fanout', immediate=True, mandatory=True) self.ch.close() # # 3 of the 4 messages we sent should have been returned # self.assertEqual(self.ch.returned_messages.qsize(), 3)
def test_encoding(self): self.ch.access_request('/data', active=True, write=True, read=True) my_routing_key = 'unittest.test_queue' qname, _, _ = self.ch.queue_declare() self.ch.queue_bind(qname, 'amq.direct', routing_key=my_routing_key) # # No encoding, body passed through unchanged # msg = Message('hello world') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertFalse(hasattr(msg2, 'content_encoding')) self.assertTrue(isinstance(msg2.body, str)) self.assertEqual(msg2.body, 'hello world') # # Default UTF-8 encoding of unicode body, returned as unicode # msg = Message(u'hello world') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'UTF-8') self.assertTrue(isinstance(msg2.body, unicode)) self.assertEqual(msg2.body, u'hello world') # # Explicit latin1 encoding, still comes back as unicode # msg = Message(u'hello world', content_encoding='latin1') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'latin1') self.assertTrue(isinstance(msg2.body, unicode)) self.assertEqual(msg2.body, u'hello world') # # Plain string with specified encoding comes back as unicode # msg = Message('hello w\xf6rld', content_encoding='latin1') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'latin1') self.assertTrue(isinstance(msg2.body, unicode)) self.assertEqual(msg2.body, u'hello w\u00f6rld') # # Plain string with bogus encoding # msg = Message('hello w\xf6rld', content_encoding='I made this up') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'I made this up') self.assertTrue(isinstance(msg2.body, str)) self.assertEqual(msg2.body, 'hello w\xf6rld') # # Turn off auto_decode for remaining tests # self.ch.auto_decode = False # # Unicode body comes back as utf-8 encoded str # msg = Message(u'hello w\u00f6rld') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'UTF-8') self.assertTrue(isinstance(msg2.body, str)) self.assertEqual(msg2.body, 'hello w\xc3\xb6rld') # # Plain string with specified encoding stays plain string # msg = Message('hello w\xf6rld', content_encoding='latin1') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'latin1') self.assertTrue(isinstance(msg2.body, str)) self.assertEqual(msg2.body, 'hello w\xf6rld') # # Explicit latin1 encoding, comes back as str # msg = Message(u'hello w\u00f6rld', content_encoding='latin1') self.ch.basic_publish(msg, 'amq.direct', routing_key=my_routing_key) msg2 = self.ch.basic_get(qname, no_ack=True) self.assertEqual(msg2.content_encoding, 'latin1') self.assertTrue(isinstance(msg2.body, str)) self.assertEqual(msg2.body, 'hello w\xf6rld')