def main(): logging.basicConfig() logging.getLogger().setLevel(logging.WARN) client = Stomp(stomp_config) client.connect() client.send(stomp_queue, body=stomp_body) client.subscribe(stomp_queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT, 'activemq.prefetchSize': 1}) if client.canRead(timeout=5): frame = client.receiveFrame() print 'Got %s' % frame.info() client.ack(frame) frame_body = str(frame.body) if frame_body == stomp_body: print "OK: Message received" status = 'ok' else: print "WARNING: Incorrect message body; is %s, should be %s" % (frame_body, stomp_body) status = 'warning' else: print "CRITICAL: Timed out while trying to collect the message" status = 'critical' client.disconnect() client.close(flush=True) return exit_codes[status]
def test_4_integration_stomp_1_1(self): if StompSpec.VERSION_1_1 not in commands.versions(VERSION): print 'This broker does not support STOMP protocol version 1.1' return client = Stomp(self.getConfig(StompSpec.VERSION_1_1)) client.connect(host=VIRTUALHOST) client.send(self.DESTINATION, 'test message 1') client.send(self.DESTINATION, 'test message 2') self.assertFalse(client.canRead(self.TIMEOUT)) token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'}) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertFalse(client.canRead(self.TIMEOUT)) client.unsubscribe(token) client.send(self.DESTINATION, 'test message 3', receipt='4711') self.assertTrue(client.canRead(self.TIMEOUT)) self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'})) self.assertFalse(client.canRead(self.TIMEOUT)) client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'}) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertFalse(client.canRead(self.TIMEOUT)) client.disconnect(receipt='4712') self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4712'})) self.assertRaises(StompConnectionError, client.receiveFrame) client.connect(host=VIRTUALHOST) client.disconnect(receipt='4711') self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'})) client.close() self.assertRaises(StompConnectionError, client.canRead, 0)
def writeJson(value,*argv): '''Writes the specified value to an output file Takes: value-> List or Dict *argv: Available options: 'stomp' 'post' 'outfile' Returns: none ''' outJson=json.dumps(value) print outJson if 'stomp' in argv: CONFIG = StompConfig(stomp_config.server, stomp_config.login,stomp_config.passcode) QUEUE = stomp_config.queue client = Stomp(CONFIG) client.connect() client.send(QUEUE, outJson) client.disconnect() elif 'outfile' in argv: with open(outfile,'w') as jsonFile: jsonFile.write(outJson) elif 'post' in argv: #TODO Post to php server pass
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self): if BROKER == 'rabbitmq': print 'Broker does not support unicode characters. Skipping this test case.' return version = StompSpec.VERSION_1_1 client = Stomp(self.getConfig(version)) try: client.connect(host=VIRTUALHOST, versions=[version]) except StompProtocolError as e: print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version) return specialCharactersHeader = u'fen\xeatre:\r\n' headers = {specialCharactersHeader: u'\xbfqu\xe9 tal?, s\xfc\xdf'} client.send(self.DESTINATION, body='test message 1', headers=headers) self.assertFalse(client.canRead(self.TIMEOUT)) token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertTrue(client.canRead(self.TIMEOUT)) frame = client.receiveFrame() client.ack(frame) self.assertEquals(frame.version, version) self.assertEquals(frame.headers[specialCharactersHeader], headers[specialCharactersHeader]) self.assertFalse(client.canRead(self.TIMEOUT)) client.unsubscribe(token) client.disconnect(receipt='4712')
def _test_4_integration_stomp(self, version): client = Stomp(self.getConfig(version)) try: client.connect(host=VIRTUALHOST, versions=[version]) except StompProtocolError as e: print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)) return client.send(self.DESTINATION, b'test message 1') client.send(self.DESTINATION, b'test message 2') self.assertFalse(client.canRead(self.TIMEOUT)) token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertFalse(client.canRead(self.TIMEOUT)) client.unsubscribe(token) client.send(self.DESTINATION, b'test message 3', receipt='4711') self.assertTrue(client.canRead(self.TIMEOUT)) self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'})) self.assertFalse(client.canRead(self.TIMEOUT)) client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertFalse(client.canRead(self.TIMEOUT)) client.disconnect(receipt='4712') self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712'})) self.assertRaises(StompConnectionError, client.receiveFrame) client.connect(host=VIRTUALHOST) client.disconnect(receipt='4711') self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'})) client.close() self.assertRaises(StompConnectionError, client.canRead, 0)
def call_route(request_queue, response_queue, request): """ """ config = { "stomp": { "server": '192.168.0.3', "port": 61613, "timeout": 15, } } stomp_config = StompConfig("tcp://%s:%d" % (config['stomp']['server'], config['stomp']['port']), version=StompSpec.VERSION_1_0) stomp = Stomp(stomp_config) stomp.connect() jms_id = str(uuid4()) token = stomp.subscribe(response_queue, {'JMSCorrelationID': jms_id}) stomp.send(request_queue, json.dumps(request), {'JMSCorrelationID': jms_id}) response = None if stomp.canRead(config['stomp']['timeout']): response = stomp.receiveFrame() stomp.unsubscribe(token) return response
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self): if BROKER == 'rabbitmq': print('Broker does not support unicode characters. Skipping this test case.') return version = StompSpec.VERSION_1_1 client = Stomp(self.getConfig(version)) try: client.connect(host=VIRTUALHOST, versions=[version]) except StompProtocolError as e: print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)) return key = b'fen\xc3\xaatre'.decode('utf-8') value = b'\xc2\xbfqu\xc3\xa9 tal?'.decode('utf-8') headers = {key: value} client.send(self.DESTINATION, body=b'test message 1', headers=headers) self.assertFalse(client.canRead(self.TIMEOUT)) token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertTrue(client.canRead(self.TIMEOUT)) frame = client.receiveFrame() client.ack(frame) self.assertEqual(frame.version, version) self.assertEqual(frame.headers[key], headers[key]) self.assertFalse(client.canRead(self.TIMEOUT)) client.unsubscribe(token) client.disconnect(receipt='4712')
def test_1_integration(self): config = self.getConfig(StompSpec.VERSION_1_0) client = Stomp(config) client.connect(host=VIRTUALHOST) client.send(self.DESTINATION, b'test message 1') client.send(self.DESTINATION, b'test message 2') self.assertFalse(client.canRead(self.TIMEOUT)) client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertTrue(client.canRead(self.TIMEOUT)) client.ack(client.receiveFrame()) self.assertFalse(client.canRead(self.TIMEOUT))
def run(self): files = os.listdir(self.path) for queue_file in files: archive_file = '{0}/archive/{1}'.format(self.path, queue_file) if queue_file.startswith('archive'): pass else: with open("{0}/{1}".format(self.path, queue_file), 'r') as qf: get_lines = list(qf) for line in get_lines: dts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') client = Stomp(config=self.config) client.connect() print "Sending message {0} to queue {1}".format(line, queue_file) with open(archive_file, 'a') as af: af.write("{0} Sent message: {1} to queue {2}\n".format(dts, line, queue_file)) client.send(queue_file, json.dumps(line)) client.disconnect()
def test_3_socket_failure_and_replay(self): client = Stomp(self.getConfig(StompSpec.VERSION_1_0)) client.connect(host=VIRTUALHOST) headers = {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL} token = client.subscribe(self.DESTINATION, headers) client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect self.assertRaises(StompConnectionError, client.receiveFrame) client.connect(host=VIRTUALHOST) client.send(self.DESTINATION, b'test message 1') client.ack(client.receiveFrame()) client.unsubscribe(token) headers = {StompSpec.ID_HEADER: 'bla', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL} client.subscribe(self.DESTINATION, headers) headers[StompSpec.DESTINATION_HEADER] = self.DESTINATION client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect self.assertRaises(StompConnectionError, client.receiveFrame) client.connect(host=VIRTUALHOST) client.send(self.DESTINATION, b'test message 2') client.ack(client.receiveFrame()) client.unsubscribe((StompSpec.ID_HEADER, 'bla')) client.disconnect()
def send_message(messageBody, destination=None, queueName=None): client = None if destination != None: client = Stomp(StompConfig(destination)) else: client = Stomp(StompConfig("tcp://localhost:61613")) QUEUE = None if queueName != None: QUEUE = queueName else: QUEUE = "pods2jbpm" #client = Stomp(CONFIG) client.connect() body = messageBody client.send(QUEUE, body) client.disconnect()
def send(self): """ Create a new stomp configuration client, connect and then serializes message by message posting them to your consumers in TOPIC standard and disconnect. """ try: configuration = StompConfig(uri=self._broker_uri) client = Stomp(configuration) client.connect(connectTimeout=self._broker_timeout) for message in self._queue: serialized_message = json.dumps(message, ensure_ascii=False) client.send(self._queue_destination, serialized_message) client.disconnect() except Exception, e: self.log.error(u"QueueManagerError - Error on sending objects from queue.") self.log.debug(e)
def test_2_transaction(self): config = self.getConfig(StompSpec.VERSION_1_0) client = Stomp(config) client.connect(host=VIRTUALHOST) client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) self.assertFalse(client.canRead(self.TIMEOUT)) with client.transaction(4711) as transaction: self.assertEqual(transaction, '4711') client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction}) self.assertFalse(client.canRead(0)) self.assertTrue(client.canRead(self.TIMEOUT)) frame = client.receiveFrame() self.assertEqual(frame.body, b'test message') client.ack(frame) with client.transaction(4713, receipt='4712') as transaction: self.assertEqual(transaction, '4713') self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-begin'})) client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction}) client.send(self.DESTINATION, b'test message without transaction') self.assertTrue(client.canRead(self.TIMEOUT)) frame = client.receiveFrame() self.assertEqual(frame.body, b'test message without transaction') client.ack(frame) self.assertFalse(client.canRead(0)) frames = [client.receiveFrame() for _ in range(2)] frames = list(sorted(frames, key=lambda f: f.command)) frame = frames[0] client.ack(frame) self.assertEqual(frame.body, b'test message') frame = frames[1] self.assertEqual(frame, StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-commit'})) try: with client.transaction(4714) as transaction: self.assertEqual(transaction, '4714') client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction}) raise RuntimeError('poof') except RuntimeError as e: self.assertEqual(str(e), 'poof') else: raise self.assertFalse(client.canRead(self.TIMEOUT)) client.disconnect()
class StompClient(BaseComponent): """ Send and Receive messages from a STOMP queue """ channel = "stomp" def init(self, host, port, username=None, password=None, connect_timeout=3, connected_timeout=3, version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"], heartbeats=(0, 0), ssl_context=None, use_ssl=True, key_file=None, cert_file=None, ca_certs=None, ssl_version=ssl.PROTOCOL_SSLv23, key_file_password=None, proxy_host=None, proxy_port=None, proxy_user=None, proxy_password=None, channel=channel): """ Initialize StompClient. Called after __init__ """ self.channel = channel if proxy_host: LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port) else: LOG.info("Connect to %s:%s", host, port) if use_ssl and not ssl_context: ssl_params = dict(key_file=key_file, cert_file=cert_file, ca_certs=ca_certs, ssl_version=ssl_version, password=key_file_password) LOG.info("Request to use old-style socket wrapper: %s", ssl_params) ssl_context = ssl_params if use_ssl: uri = "ssl://%s:%s" % (host, port) else: uri = "tcp://%s:%s" % (host, port) # Configure failover options so it only tries to connect once self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context, version=version, login=username, passcode=password) self._heartbeats = heartbeats self._accept_versions = accept_versions self._connect_timeout = connect_timeout self._connected_timeout = connected_timeout Stomp._transportFactory = EnhancedStompFrameTransport Stomp._transportFactory.proxy_host = proxy_host Stomp._transportFactory.proxy_port = proxy_port Stomp._transportFactory.proxy_user = proxy_user Stomp._transportFactory.proxy_password = proxy_password self._client = Stomp(self._stomp_config) self._subscribed = {} self.server_heartbeat = None self.client_heartbeat = None self.ALLOWANCE = 2 # multiplier for heartbeat timeouts @property def connected(self): if self._client.session: return self._client.session.state == StompSession.CONNECTED else: return False @property def subscribed(self): return self._subscribed.keys() @property def stomp_logger(self): return LOG_CATEGORY @handler("disconnect") def _disconnect(self, receipt=None): if self.connected: self._client.disconnect(receipt=receipt) self._client.close(flush=True) self.fire(disconnected(reconnect=False)) self._subscribed = {} return "disconnected" def start_heartbeats(self): LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat, self._client.serverHeartBeat) if self._client.clientHeartBeat: if self.client_heartbeat: # Timer already exists, just reset it self.client_heartbeat.reset() else: LOG.info("Client will send heartbeats to server") # Send heartbeats at 80% of agreed rate self.client_heartbeat = Timer((self._client.clientHeartBeat / 1000.0) * 0.8, client_heartbeat(), persist=True) self.client_heartbeat.register(self) else: LOG.info("No Client heartbeats will be sent") if self._client.serverHeartBeat: if self.server_heartbeat: # Timer already exists, just reset it self.server_heartbeat.reset() else: LOG.info("Requested heartbeats from server.") # Allow a grace period on server heartbeats self.server_heartbeat = Timer((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE, server_heartbeat(), persist=True) self.server_heartbeat.register(self) else: LOG.info("Expecting no heartbeats from Server") @handler("connect") def connect(self, event, host=None, *args, **kwargs): """ connect to Stomp server """ LOG.info("Connect to Stomp...") try: self._client.connect(heartBeats=self._heartbeats, host=host, versions=self._accept_versions, connectTimeout=self._connect_timeout, connectedTimeout=self._connected_timeout) LOG.info("State after Connection Attempt: %s", self._client.session.state) if self.connected: LOG.info("Connected to %s", self._stomp_server) self.fire(connected()) self.start_heartbeats() return "success" except StompConnectionError: LOG.debug(traceback.format_exc()) self.fire(connection_failed(self._stomp_server)) event.success = False return "fail" @handler("server_heartbeat") def check_server_heartbeat(self, event): """ Confirm that heartbeat from server hasn't timed out """ now = time.time() last = self._client.lastReceived or 0 if last: elapsed = now - last else: elapsed = -1 LOG.debug("Last received data %d seconds ago", elapsed) if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE + last) < now: LOG.error("Server heartbeat timeout. %d seconds since last heartbeat. Disconnecting.", elapsed) event.success = False self.fire(heartbeat_timeout()) if self.connected: self._client.disconnect() # TODO: Try to auto-reconnect? @handler("client_heartbeat") def send_heartbeat(self, event): if self.connected: LOG.debug("Sending heartbeat") try: self._client.beat() except StompConnectionError: event.success = False self.fire(disconnected()) @handler("generate_events") def generate_events(self, event): if not self.connected: return try: if self._client.canRead(1): frame = self._client.receiveFrame() LOG.debug("Recieved frame %s", frame) self.fire(message(frame)) except StompConnectionError: self.fire(disconnected()) @handler("send") def send(self, event, destination, body, headers=None, receipt=None): LOG.debug("send()") if not self.connected: LOG.error("Can't send when Stomp is disconnected") self.fire(on_stomp_error(None, Exception("Message send attempted with stomp disconnected"))) event.success = False return try: self._client.send(destination, body=body.encode('utf-8'), headers=headers, receipt=receipt) LOG.debug("Message sent") except StompConnectionError as err: event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(None, err)) @handler("subscribe") def _subscribe(self, event, destination, ack=ACK_CLIENT_INDIVIDUAL): if ack not in ACK_MODES: raise ValueError("Invalid client ack mode specified") LOG.info("Subscribe to message destination %s", destination) try: # Set ID to match destination name for easy reference later frame, token = self._client.subscribe(destination, headers={StompSpec.ACK_HEADER: ack, 'id': destination}) self._subscribed[destination] = token except StompConnectionError as err: event.success = False self.fire(disconnected()) except StompError as err: event.success = False LOG.debug(traceback.format_exc()) self.fire(on_stomp_error(None, err)) @handler("unsubscribe") def _unsubscribe(self, event, destination): if destination not in self._subscribed: LOG.error("Unsubscribe Request Ignored. Not subscribed to %s", destination) return try: token = self._subscribed.pop(destination) frame = self._client.unsubscribe(token) LOG.debug("Unsubscribed: %s", frame) except StompConnectionError as err: event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(frame, err)) @handler("message") def on_message(self, event, headers, message): LOG.info("Stomp message received") @handler("ack") def ack_frame(self, event, frame): LOG.debug("ack_frame()") try: self._client.ack(frame) LOG.debug("Ack Sent") except StompConnectionError as err: LOG.error("Error sending ack") event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(frame, err)) def get_subscription(self, frame): """ Get subscription from frame """ LOG.info(self._subscribed) _, token = self._client.message(frame) return self._subscribed[token]
class Client(object): def __init__(self): self.stompest = None self.greenlet = None self.subscriptions = {} self._last_id = 0 def _next_id(self): self._last_id += 1 return self._last_id def connect(self): if not self.stompest: CONFIG, EXTRA = _get_config() self._hostname = EXTRA.get('hostname', None) self.stompest = Stomp(CONFIG) if self.stompest.session.state != StompSession.DISCONNECTED: return while True: try: self.stompest.connect(host=self._hostname) logger.info('Connected') break except StompConnectTimeout: continue if not self.greenlet: self.greenlet = gevent.spawn(self._run) def _run(self): while True: try: frame = self.stompest.receiveFrame() self.stompest.ack(frame) if frame.command == 'ERROR': logger.error(frame.info()) elif frame.command == 'MESSAGE': token = self.stompest.message(frame) if self.subscriptions.get(token): subscription = self.subscriptions[token] subscription.call(frame) else: logger.error("Received a message for %s (%s) but there was no matching subscription." % (frame.headers.get(StompSpec.DESTINATION_HEADER, '???'), token)) else: logger.warning("Unknown frame: %s" % frame.info()) # @todo Handle receipts except (gevent.GreenletExit, KeyboardInterrupt): # @todo Include a receipt in the disconnect. And instead of breaking right away wait for the # receipt frame before disconnecting and consider waiting on any greenlets we started. self.stompest.disconnect() break except StompConnectionError: # We've been disconnected from the server. Try reconnecting to it. self.connect() def on(self, destination, callback): self.connect() token = self.stompest.subscribe(destination, { StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL, StompSpec.ID_HEADER: self._next_id(), }) subscription = Subscription( conn=self, destination=destination, token=token, callback=callback ) self.subscriptions[subscription.token] = subscription; # @todo consider adding optional support for additional custom headers def send(self, cmd, destination): self.connect() body = json.dumps(cmd) headers = {} headers[StompSpec.CONTENT_TYPE_HEADER] = 'application/json;charset=UTF-8' self.stompest.send(destination, body, headers) def join(self): try: self.connect() except (gevent.GreenletExit, KeyboardInterrupt): return try: gevent.joinall([self.greenlet]) except KeyboardInterrupt: self.greenlet.kill(block=True)
def calculate(): pathwayConfig = app.config[Pathway.CONFIG] try: ts = str(time.time()) parameters = dict(request.form) for field in parameters: parameters[field] = parameters[field][0] parameters['idstr'] = ts filelist = request.files studyList = [] num_studies = int(parameters['num_studies']) for i in xrange(1,num_studies+1): studyKey = "study_" + str(i) studyObj = {} studyObj['lambda'] = parameters['lambda_' + str(i)] del parameters['lambda_'+str(i)] studyObj['sample_sizes'] = [] for resourceInd in range(1,int(parameters['num_resource_' + str(i)])+1): studyObj['sample_sizes'].append(parameters['sample_size_' + str(i) + '_' + str(resourceInd)]) del parameters['sample_size_' + str(i) + '_' + str(resourceInd)] del parameters['num_resource_' + str(i)] studyFile = filelist[studyKey] if studyFile.filename: filename = os.path.join(os.getcwd(),app.config['UPLOAD_FOLDER'],ts + '-' + str(i) + '.study') studyObj['filename'] = filename studyFile.save(filename) else: return Pathway.buildFailure("The file seems to be missing from Study #" + i + ".") studyList.append(studyObj) del parameters['num_studies'] parameters['studies'] = studyList if parameters['pathway_type'] == 'file_pathway': pathFile = filelist['file_pathway'] if pathFile.filename: filename = os.path.join(app.config['UPLOAD_FOLDER'],ts + '.pathway') parameters['pathway'] = filename pathFile.save(filename) else: return Pathway.buildFailure("The pathway file seems to be missing.") elif parameters['pathway_type'] == 'database_pathway': parameters['pathway'] = os.path.join(app.config['PATHWAY_FOLDER'],parameters['database_pathway']) else: return Pathway.buildFailure("The pathway file seems to be missing.") if "database_pathway" in parameters: del parameters['database_pathway'] if "selectAll" in parameters: del parameters['selectAll'] if "selectItem" in parameters: del parameters['selectItem'] superpop = {} subpop = {} for population in parameters['populations'].split(","): population = population.split('|') if os.path.isfile(os.path.join(app.config['POPULATION_FOLDER'],population[0],population[1]+'.txt')): superpop[population[0]] = 1 subpop_file = os.path.join(app.config['POPULATION_FOLDER'],population[0],population[1]+'.txt') with open(subpop_file, 'r') as subpop_file: for line in subpop_file: subpop[line.strip()] = 1 else: return Pathway.buildFailure("An invalid population was submitted.") if (len(superpop) > 1): return Pathway.buildFailure("An invalid population was submitted.") del parameters['populations'] for population in superpop: parameters['plink'] = app.config['PLINK_PATTERN'].replace("$pop",population) parameters['population'] = [] for population in subpop: parameters['population'].append(population) parameters['outdir'] = app.config['OUT_FOLDER'] parameters['refinep'] = parameters.get('refinep',"").lower() in ['true','t','1'] parameters['gene_subset'] = parameters.get('gene_subset',"").lower() in ['true','t','1'] jsonout = {"submittedTime": parameters['idstr'], "payload": parameters} with open(os.path.join(app.config['OUT_FOLDER'],str(parameters['idstr'])+'.json'),'w') as outfile: json.dump(jsonout,outfile) client = Stomp(pathwayConfig[Pathway.QUEUE_CONFIG]) client.connect() client.send(pathwayConfig.getAsString(Pathway.QUEUE_NAME), json.dumps(parameters)) client.disconnect() return Pathway.buildSuccess("The request has been received. An email will be sent when the calculation has completed.") except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print("EXCEPTION------------------------------", exc_type, fname, exc_tb.tb_lineno) return Pathway.buildFailure(str(e))
clientConsumer = Stomp(config) clientConsumer.connect(host=host) # Subscribe to the topic where the answers are posted to clientConsumer.subscribe(destination='/topic/docstore-reply', headers={'id': 'required-for-STOMP-1.1'}) # Send a query queries = range(1,15) headers = { 'transformation': 'TEXT', 'event': 'QueryDocelem', 'reply-to': '/topic/docstore-reply' } client.send(destination=destination, body="scai.fhg.de/voc/ccg", headers=headers) for i in queries: client.send(destination=destination, body="scai.fhg.de/concept/" + str(i), headers=headers) print u"# ErasmusMC ontology file".encode('utf-8') print u"VR 0.0".encode('utf-8') print u"ON CCG_Ontology".encode('utf-8') print u"--".encode('utf-8') @timeout(1) def receiveFrames(): frame = clientConsumer.receiveFrame() xml = ElementTree.fromstring(frame.body) modelText = xml.find('docelem').find('model').text
class StompClient(object): def __init__(self, config): """Init the Stompest wrapper client. :type config: dict :param config: The configuration for the STOM client. I.e. {'host': 'tcp://127.0.0.1', 'queue': '/queue/test', 'transaction': True, 'username': '******', 'password': '******'} The transaction attribute defines if messages should be published in transactions. """ self.host = config['host'] self.queue = config['queue'] self.transactions_enabled = config['transaction'] self.transaction = None auth_header = {} if 'username' in config and 'password' in config: auth_header.update( {StompSpec.LOGIN_HEADER: config['username'], StompSpec.PASSCODE_HEADER: config['password']}) self.client = Stomp(StompConfig(self.host)) try: self.client.connect(headers=auth_header) except (error.StompConnectTimeout, error.StompProtocolError) as e: raise ClientErrors.ConnectionError( "Could not connect to broker: %s" % e) def close(self): """Close the connection.""" try: self.client.disconnect() except error.StompConnectionError as e: raise ClientErrors.ConnectionError( "Could not close connection: %s" % e) def publish(self, messages, omit_transaction=False, durable=True): """Publish a message to the queue. :type message: string or list of strings. :param message: The message(s) to publish. :type omit_transaction: bool :param omit_transaction: Set to True if you would like to publish a message outside a transaction. :type durable: bool :param durable: If this message should be durable. """ if omit_transaction: header = None elif self.transactions_enabled: if not self.transaction: self.transaction = str(uuid.uuid4()) try: self.client.begin(transaction=self.transaction) except error.StompProtocolError as e: raise ClientErrors.ProtocolError( "Could not start transaction: %s" % e) header = {StompSpec.TRANSACTION_HEADER: self.transaction, 'durable': 'true' if durable else 'false'} else: header = None if isinstance(messages, (basestring, dict, scim.Event)): messages = [messages] for msg in messages: try: if isinstance(msg, dict): del msg['routing-key'] msg = json.dumps(msg) elif isinstance(msg, scim.Event): msg = json.dumps(msg.get_payload()) self.client.send(self.queue, msg, header) except error.StompConnectionError as e: raise ClientErrors.ConnectionError( "Could not publish '%s' to broker: %s" % (msg, e)) def commit(self): """Commit the current transaction.""" if self.transaction: self.client.commit(transaction=self.transaction) self.transaction = None def rollback(self): """Roll back (ABORT) the current transaction.""" if self.transaction: self.client.abort(transaction=self.transaction) self.transaction = None
class Client(object): def __init__(self): self.stompest = None self.greenlet = None self.subscriptions = {} self._last_id = 0 def _next_id(self): self._last_id += 1 return self._last_id def connect(self): if not self.stompest: CONFIG, EXTRA = _get_config() self._hostname = EXTRA.get('hostname', None) self.stompest = Stomp(CONFIG) if self.stompest.session.state != StompSession.DISCONNECTED: return while True: try: self.stompest.connect(host=self._hostname) logger.info('Connected') break except StompConnectTimeout: continue if not self.greenlet: self.greenlet = gevent.spawn(self._run) def _run(self): while True: try: frame = self.stompest.receiveFrame() self.stompest.ack(frame) if frame.command == 'ERROR': logger.error(frame.info()) elif frame.command == 'MESSAGE': token = self.stompest.message(frame) if self.subscriptions.get(token): subscription = self.subscriptions[token] subscription.call(frame) else: logger.error( "Received a message for %s (%s) but there was no matching subscription." % (frame.headers.get(StompSpec.DESTINATION_HEADER, '???'), token)) else: logger.warning("Unknown frame: %s" % frame.info()) # @todo Handle receipts except (gevent.GreenletExit, KeyboardInterrupt): # @todo Include a receipt in the disconnect. And instead of breaking right away wait for the # receipt frame before disconnecting and consider waiting on any greenlets we started. self.stompest.disconnect() break except StompConnectionError: # We've been disconnected from the server. Try reconnecting to it. self.connect() def on(self, destination, callback): self.connect() token = self.stompest.subscribe( destination, { StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL, StompSpec.ID_HEADER: self._next_id(), }) subscription = Subscription(conn=self, destination=destination, token=token, callback=callback) self.subscriptions[subscription.token] = subscription # @todo consider adding optional support for additional custom headers def send(self, cmd, destination): self.connect() body = json.dumps(cmd) headers = {} headers[ StompSpec.CONTENT_TYPE_HEADER] = 'application/json;charset=UTF-8' self.stompest.send(destination, body, headers) def join(self): try: self.connect() except (gevent.GreenletExit, KeyboardInterrupt): return try: gevent.joinall([self.greenlet]) except KeyboardInterrupt: self.greenlet.kill(block=True)
#!/usr/bin/python from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://localhost:61613') QUEUE = '/queue/test' if __name__ == '__main__': client = Stomp(CONFIG) client.connect() client.send(QUEUE, 'Hello'.encode()) client.send(QUEUE, 'YAYYYYYY'.encode()) client.disconnect()
class StompClient(BaseComponent): """ Send and Receive messages from a STOMP queue """ channel = "stomp" def init(self, host, port, username=None, password=None, connect_timeout=3, connected_timeout=3, version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"], heartbeats=(0, 0), ssl_context=None, use_ssl=True, key_file=None, cert_file=None, ca_certs=None, ssl_version=ssl.PROTOCOL_SSLv23, key_file_password=None, proxy_host=None, proxy_port=None, proxy_user=None, proxy_password=None, channel=channel): """ Initialize StompClient. Called after __init__ """ self.channel = channel if proxy_host: LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port) else: LOG.info("Connect to %s:%s", host, port) if use_ssl and not ssl_context: ssl_params = dict(key_file=key_file, cert_file=cert_file, ca_certs=ca_certs, ssl_version=ssl_version, password=key_file_password) LOG.info("Request to use old-style socket wrapper: %s", ssl_params) ssl_context = ssl_params if use_ssl: uri = "ssl://%s:%s" % (host, port) else: uri = "tcp://%s:%s" % (host, port) # Configure failover options so it only tries to connect once self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context, version=version, login=username, passcode=password) self._heartbeats = heartbeats self._accept_versions = accept_versions self._connect_timeout = connect_timeout self._connected_timeout = connected_timeout Stomp._transportFactory = EnhancedStompFrameTransport Stomp._transportFactory.proxy_host = proxy_host Stomp._transportFactory.proxy_port = proxy_port Stomp._transportFactory.proxy_user = proxy_user Stomp._transportFactory.proxy_password = proxy_password self._client = Stomp(self._stomp_config) self._subscribed = {} self.server_heartbeat = None self.client_heartbeat = None self.ALLOWANCE = 2 # multiplier for heartbeat timeouts @property def connected(self): if self._client.session: return self._client.session.state == StompSession.CONNECTED else: return False @property def subscribed(self): return self._subscribed.keys() @property def stomp_logger(self): return LOG_CATEGORY @handler("disconnect") def _disconnect(self, receipt=None): if self.connected: self._client.disconnect(receipt=receipt) self._client.close(flush=True) self.fire(disconnected(reconnect=False)) self._subscribed = {} return "disconnected" def start_heartbeats(self): LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat, self._client.serverHeartBeat) if self._client.clientHeartBeat: if self.client_heartbeat: # Timer already exists, just reset it self.client_heartbeat.reset() else: LOG.info("Client will send heartbeats to server") # Send heartbeats at 80% of agreed rate self.client_heartbeat = Timer((self._client.clientHeartBeat / 1000.0) * 0.8, client_heartbeat(), persist=True) self.client_heartbeat.register(self) else: LOG.info("No Client heartbeats will be sent") if self._client.serverHeartBeat: if self.server_heartbeat: # Timer already exists, just reset it self.server_heartbeat.reset() else: LOG.info("Requested heartbeats from server.") # Allow a grace period on server heartbeats self.server_heartbeat = Timer((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE, server_heartbeat(), persist=True) self.server_heartbeat.register(self) else: LOG.info("Expecting no heartbeats from Server") @handler("connect") def connect(self, event, host=None, *args, **kwargs): """ connect to Stomp server """ LOG.info("Connect to Stomp...") try: self._client.connect(heartBeats=self._heartbeats, host=host, versions=self._accept_versions, connectTimeout=self._connect_timeout, connectedTimeout=self._connected_timeout) LOG.info("State after Connection Attempt: %s", self._client.session.state) if self.connected: LOG.info("Connected to %s", self._stomp_server) self.fire(connected()) self.start_heartbeats() return "success" except StompConnectionError: LOG.debug(traceback.format_exc()) self.fire(connection_failed(self._stomp_server)) event.success = False return "fail" @handler("server_heartbeat") def check_server_heartbeat(self, event): """ Confirm that heartbeat from server hasn't timed out """ now = time.time() last = self._client.lastReceived or 0 if last: elapsed = now - last else: elapsed = -1 LOG.debug("Last received data %d seconds ago", elapsed) if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE + last) < now: LOG.error("Server heartbeat timeout. %d seconds since last heartbeat. Disconnecting.", elapsed) event.success = False self.fire(heartbeat_timeout()) if self.connected: self._client.disconnect() # TODO: Try to auto-reconnect? @handler("client_heartbeat") def send_heartbeat(self, event): if self.connected: LOG.debug("Sending heartbeat") try: self._client.beat() except StompConnectionError: event.success = False self.fire(disconnected()) @handler("generate_events") def generate_events(self, event): if not self.connected: return try: if self._client.canRead(1): frame = self._client.receiveFrame() LOG.debug("Recieved frame %s", frame) self.fire(message(frame)) except StompConnectionError: self.fire(disconnected()) @handler("send") def send(self, event, destination, body, headers=None, receipt=None): LOG.debug("send()") if not self.connected: LOG.error("Can't send when Stomp is disconnected") self.fire(on_stomp_error(None, Exception("Message send attempted with stomp disconnected"))) event.success = False return try: self._client.send(destination, body=body.encode('utf-8'), headers=headers, receipt=receipt) LOG.debug("Message sent") except StompConnectionError: event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(None, err)) @handler("subscribe") def _subscribe(self, event, destination, ack=ACK_CLIENT_INDIVIDUAL): if ack not in ACK_MODES: raise ValueError("Invalid client ack mode specified") LOG.info("Subscribe to message destination %s", destination) try: # Set ID to match destination name for easy reference later frame, token = self._client.subscribe(destination, headers={StompSpec.ACK_HEADER: ack, 'id': destination}) self._subscribed[destination] = token except StompConnectionError: event.success = False self.fire(disconnected()) except StompError as err: event.success = False LOG.debug(traceback.format_exc()) self.fire(on_stomp_error(None, err)) @handler("unsubscribe") def _unsubscribe(self, event, destination): if destination not in self._subscribed: LOG.error("Unsubscribe Request Ignored. Not subscribed to %s", destination) return try: token = self._subscribed.pop(destination) frame = self._client.unsubscribe(token) LOG.debug("Unsubscribed: %s", frame) except StompConnectionError: event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(frame, err)) @handler("message") def on_message(self, event, headers, message): LOG.info("Stomp message received") @handler("ack") def ack_frame(self, event, frame): LOG.debug("ack_frame()") try: self._client.ack(frame) LOG.debug("Ack Sent") except StompConnectionError: LOG.error("Error sending ack") event.success = False self.fire(disconnected()) except StompError as err: LOG.error("Error sending ack") event.success = False self.fire(on_stomp_error(frame, err)) def get_subscription(self, frame): """ Get subscription from frame """ LOG.info(self._subscribed) _, token = self._client.message(frame) return self._subscribed[token]
def send_stomp(message): config = StompConfig(stomp_uri) client = Stomp(config) client.connect() client.send(stomp_destination, message) client.disconnect()
import json from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://*****:*****@mail.nih.gov','timeStamp':'2015-06-25','outputDir':'/home/user/python/testOutDir'})) client.disconnect()
logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://localhost:7777', version='1.2') if __name__ == '__main__': client = Stomp(CONFIG) client.connect(connectedTimeout=4) ID = 'bingo' client.send( '/my/test/destination', # destination 'THIS-IS-A-SEND-BODY', # body headers={'receipt': ID}) # headers answer = client.receiveFrame() receiptID = answer.headers["receipt-id"] returnValue = 0 if receiptID != ID: print "Receipt header wrong:" + receiptID returnValue = 1 else: print "Correct receipt id received: " + receiptID client.disconnect() sys.exit(returnValue)
class JMSClient(object): """Class JMSClient """ _mh = None _client = None _host = None _port = None _user = None _passw = None _verbose = None _is_connected = None def __init__(self, verbose=False): """Class constructor Called when the object is initialized Args: verbose (bool): verbose mode """ try: self._mh = MasterHead.get_head() self._verbose = verbose if (self._verbose): basicConfig() getLogger().setLevel(DEBUG) except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) @property def client(self): """ STOMP client property getter """ return self._client @property def host(self): """ server host property getter """ return self._host @property def port(self): """ server port property getter """ return self._port @property def user(self): """ username property getter """ return self._user @property def passw(self): """ user password property getter """ return self._passw @property def verbose(self): """ verbose mode property getter """ return self._verbose @property def is_connected(self): """ is_connected property getter """ return self._is_connected def connect(self, host, port=61613, user=None, passw=None, timeout=10): """Method connects to server Args: host (str): hostname port (str): port user (str): username passw (str): password timeout (int): timeout Returns: bool: result Raises: event: jms_before_connect event: jms_after_connected """ try: msg = 'host:{0}, port:{1}, user:{2}, passw:{3}, timeout:{4}'.format( host, port, user, passw, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_connecting', msg), self._mh.fromhere()) ev = event.Event( 'jms_before_connect', host, port, user, passw, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) timeout = ev.argv(4) self._host = host self._port = port self._user = user self._passw = passw if (ev.will_run_default()): self._client = Stomp(StompConfig('tcp://{0}:{1}'.format(self._host, self._port), login=self._user, passcode=self._passw)) self._client.connect( connectTimeout=timeout, connectedTimeout=timeout) self._is_connected = True self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_connected'), self._mh.fromhere()) ev = event.Event('jms_after_connect') self._mh.fire_event(ev) return True except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def disconnect(self): """Method disconnects from server Args: none Returns: bool: result """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_disconnecting'), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return False else: self._client.disconnect() self._client.close() self._is_connected = False self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_disconnected'), self._mh.fromhere()) return True except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def send(self, destination_name, message, destination_type='queue', headers={}): """Method sends message JMS headers - JMSCorrelationID, JMSExpiration, JMSDeliveryMode, JMSPriority, JMSReplyTo, JMSType Args: destination_name (str): queue|topic name message (str): message destination_type (str): queue|topic headers (dict): JMS headers, key - title, value - string Returns: bool: result Raises: event: jms_before_send event: jms_after_send """ try: msg = 'destination_name:{0}, message:{1}, destination_type:{2}, headers:{3}'.format( destination_name, message, destination_type, headers) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_sending_msg', msg), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return False ev = event.Event( 'jms_before_send', destination_name, message, destination_type, headers) if (self._mh.fire_event(ev) > 0): destination_name = ev.argv(0) message = ev.argv(1) destination_type = ev.argv(2) headers = ev.argv(3) if (ev.will_run_default()): headers_new = {} for key, value in headers.items(): if (key in mapping): headers_new[mapping[key]] = value self._client.send('/{0}/{1}'.format(destination_type, destination_name), message if ( version_info[0] == 2) else message.encode('utf-8'), headers_new) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_msg_sent'), self._mh.fromhere()) ev = event.Event('jms_after_send') self._mh.fire_event(ev) return True except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def receive(self, destination_name, cnt=1): """Method receives messages Args: destination_name (str): queue name cnt (int): count of messages Returns: list: messages as dictionary {'message', JMS headers} Raises: event: jms_before_receive event: jms_after_receive """ try: msg = 'destination_name:{0}, count:{1}'.format( destination_name, cnt) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_receiving_msg', msg), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return None ev = event.Event('jms_before_receive', destination_name, cnt) if (self._mh.fire_event(ev) > 0): destination_name = ev.argv(0) cnt = ev.argv(1) if (ev.will_run_default()): token = self._client.subscribe('/queue/{0}'.format(destination_name), {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) msgs = [] i = 0 while (i < cnt and self._client.canRead(1)): frame = self._client.receiveFrame() if (frame.command != 'MESSAGE'): break self._client.ack(frame) msgs.append(frame) i = i + 1 self._client.unsubscribe(token) messages = [] for msg in msgs: message = {} message['message'] = msg.body.decode() for header in msg.rawHeaders: if (header[0] in mapping.values()): message[ list(mapping.keys())[list(mapping.values()).index(header[0])]] = header[1] messages.append(message) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_msg_received', len(messages)), self._mh.fromhere()) ev = event.Event('jms_after_receive') self._mh.fire_event(ev) return messages except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return None def browse(self, destination_name, cnt=100, jms_correlation_id=None, jms_type=None): """Method browses queue Args: destination_name (str): queue name cnt (int): count of messages jms_correlation_id (str): requested JMSCorrelationID jms_type (str): requested JMSType Returns: list: messages as dictionary {'message', JMS headers} Raises: event: jms_before_browse event: jms_after_browse """ try: msg = 'destination_name:{0}, count:{1}, jms_correlation_id:{2}, jms_type:{3}'.format( destination_name, cnt, jms_correlation_id, jms_type) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_browsing', msg), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return None ev = event.Event( 'jms_before_browse', destination_name, cnt, jms_correlation_id, jms_type) if (self._mh.fire_event(ev) > 0): destination_name = ev.argv(0) cnt = ev.argv(1) jms_correlation_id = ev.argv(2) jms_type = ev.argv(3) if (ev.will_run_default()): token = self._client.subscribe('/queue/{0}'.format(destination_name), {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) msgs = [] i = 0 while (i < cnt and self._client.canRead(1)): frame = self._client.receiveFrame() correlation_id = None type = None for header in frame.rawHeaders: if (header[0] == 'correlation-id'): correlation_id = header[1] elif (header[0] == 'type'): type = header[1] if ((jms_correlation_id == None or jms_correlation_id == correlation_id) and (jms_type == None or jms_type == type)): msgs.append(frame) i = i + 1 self._client.unsubscribe(token) messages = [] for msg in msgs: message = {} message['message'] = msg.body.decode() for header in msg.rawHeaders: if (header[0] in mapping.values()): message[ list(mapping.keys())[list(mapping.values()).index(header[0])]] = header[1] messages.append(message) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_msg_received', len(messages)), self._mh.fromhere()) ev = event.Event('jms_after_browse') self._mh.fire_event(ev) return messages except StompError as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return None
class ByteportStompClient(AbstractByteportClient): DEFAULT_BROKER_HOST = 'stomp.byteport.se' STORE_QUEUE_NAME = '/queue/simple_string_dev_message' SUPPORTED_CHANNEL_TYPES = ['topic', 'queue'] client = None def __init__(self, namespace, login, passcode, broker_host=DEFAULT_BROKER_HOST, device_uid=None, channel_type='topic'): ''' Create a ByteportStompClient. This is a thin wrapper to the underlying STOMP-client that connets to the Byteport Broker If a device_uid is given, a subscription will be made for Messages sent through Byteport. The channel_type must be either 'topic' or 'queue'. Set top topic if unsure on what to use (use queue if you need to use multiple consumers for a single device, this is not how most applications are set up). :param namespace: :param login: Broker username (Byteport web users are _not_ valid broker users). Ask [email protected] for access. :param passcode: Broker passcode :param broker_hosts: [optional] A list of brokers to connect to :param device_uid: [optional] The device UID to subscribe for messages on :param channel_type: [optional] Defaults to queue. :param channel_key: [optional] Must match the configured key in the Byteport Device Manager ''' self.namespace = str(namespace) self.device_uid = device_uid if channel_type not in self.SUPPORTED_CHANNEL_TYPES: raise Exception("Unsupported channel type: %s" % channel_type) broker_url = 'tcp://%s:61613' % broker_host self.CONFIG = StompConfig(broker_url, version=StompSpec.VERSION_1_2) self.client = Stomp(self.CONFIG) try: self.client.connect(headers={'login': login, 'passcode': passcode}, host='/') logging.info("Connected to Stomp broker at %s using protocol version %s" % (broker_host, self.client.session.version)) # Set up a subscription on the correct queue if a Specific device UID was given if self.device_uid: subscribe_headers = dict() subscribe_headers[StompSpec.ACK_HEADER] = StompSpec.ACK_CLIENT_INDIVIDUAL subscribe_headers[StompSpec.ID_HEADER] = '0' device_message_queue_name = '/%s/device_messages_%s.%s' % (channel_type, namespace, device_uid) self.subscription_token = self.client.subscribe(device_message_queue_name, subscribe_headers) logging.info("Subscribing to channel %s" % device_message_queue_name) except StompProtocolError as e: logging.error("Client socket connected, but probably failed to login. (ProtocolError)") raise except StompConnectionError: logging.error("Failed to connect to Stomp Broker at %s" % broker_host) raise def disconnect(self): if self.subscription_token: try: self.client.unsubscribe(self.subscription_token) except Exception as e: logging.error(u'Unsubscribe failed, reason %s' % e) self.client.disconnect() def __send_json_message(self, json): self.client.send(self.STORE_QUEUE_NAME, json) def __send_message(self, uid, data_string, timestamp=None): if timestamp: timestamp = self.auto_timestamp(timestamp) else: timestamp = int(time.time()) if not uid: uid = self.device_uid if not uid: raise Exception("Can not send data without valid Device UID!") message = dict() message['uid'] = str(uid) message['namespace'] = self.namespace message['data'] = str(data_string) message['timestamp'] = str(timestamp) self.__send_json_message(json.dumps([message])) def store(self, data=None, device_uid=None, timestamp=None): if type(data) != dict: raise ByteportClientException("Data must be of type dict") for key in data.keys(): self.verify_field_name(key) delimited_data = ';'.join("%s=%s" % (key, self.utf8_encode_value(val)) for (key, val) in data.iteritems()) self.__send_message(device_uid, delimited_data, timestamp)
class client_activemq(): def __init__(self, worker_threads=2): self.log = logging.getLogger('client_activemq') self.log.debug('Initializing client_activemq') self.host = '127.0.0.1' #manually set here until there is a better place self.port = 61613 #manually set here until there is a better place self.worker_threads = worker_threads self.client = None self.connection_issue = True self.continue_running = True self.subscriptions = {} self.outbound_msg_queue = [] self.outbound_msg_queue_lazy = [] self.threads = [] self.work_queue = Queue.Queue() connection_handler = threading.Thread(target=self.connection_handler) connection_handler.daemon = True connection_handler.start() send_event_hopeful_thread = threading.Thread( target=self.send_event_hopeful_thread) send_event_hopeful_thread.daemon = True send_event_hopeful_thread.start() send_event_lazy_thread = threading.Thread( target=self.send_event_lazy_thread) send_event_lazy_thread.daemon = True send_event_lazy_thread.start() publish_loop = threading.Thread(target=self.publish_loop) publish_loop.daemon = True publish_loop.start() def init_connection(self): if (self.client != None and self.client.session.state == 'connected'): self.log.error('Re-init connection') try: self.client.disconnect() except: pass config = StompConfig('tcp://%s:%s' % (self.host, self.port), version=StompSpec.VERSION_1_1, check=True) self.client = Stomp(config) self.client.connect(heartBeats=(30000, 0), connectTimeout=1, connectedTimeout=1) def build_worker_pool(self, capacity): for thread in self.threads: if thread.is_alive() == False: self.threads.remove(thread) while len(self.threads) < capacity: t = threading.Thread(target=self.worker, args=(self.work_queue, )) t.daemon = True t.start() self.threads.append(t) def connection_handler(self): #This func will just try to reconnect repeatedly in a thread if we're flagged as having an issue. while (True): if (self.connection_issue == True): try: self.init_connection() self.connection_issue = False time.sleep(0.1) for subscription in self.subscriptions: self.log.info('Re-subscribing upon reconnection: %s' % subscription) self.subscribe( subscription, self.subscriptions[subscription]['callback_class'], self.subscriptions[subscription]['callback'], True, self.subscriptions[subscription]['selector']) except Exception as e: self.log.critical('Except: %s' % e) else: try: self.client.beat() except: self.log.warning('Failed to heartbeat?') self.build_worker_pool(self.worker_threads) time.sleep(10) def subscribe(self, queue, callback_class, callback, resub=False, selector=None): #This needs to exist so we can keep track of what subs we have and re-sub on reconnect if queue in self.subscriptions and not resub: self.log.info('Ignoring existing subscription %s' % queue) return True #we're already subscribed this_uuid = '%s' % uuid.uuid4() if (self.connection_issue == True): self.log.info( 'Cannot process subscription request as were not properly connected' ) self.subscriptions[queue] = { 'uuid': this_uuid, 'callback': callback, 'callback_class': callback_class, 'selector': selector } return None self.log.info('subscribe(%s %s %s %s)' % (queue, callback_class, callback, resub)) try: self.subscriptions[queue] = { 'uuid': this_uuid, 'callback': callback, 'callback_class': callback_class, 'selector': selector } if selector != None: self.client.subscribe( queue, { StompSpec.ID_HEADER: this_uuid, StompSpec.ACK_HEADER: StompSpec.ACK_AUTO, 'selector': selector }) else: self.client.subscribe( queue, { StompSpec.ID_HEADER: this_uuid, StompSpec.ACK_HEADER: StompSpec.ACK_AUTO, }) except Exception as e: self.log.fatal('%s' % e) self.connection_issue = True def unsubscribe(self, queue): if queue not in self.subscriptions: return False #cant unsubscribe, we're not subscribed if (self.connection_issue == True): del self.subscriptions[queue] return None try: self.client.unsubscribe(self.subscriptions[queue]['uuid']) del self.subscriptions[queue] except Exception as e: self.log.error('%s' % e) self.log.error('%s' % traceback.format_exc()) sys.exc_clear() self.connection_issue = True def send_event_lazy(self, destination, body, headers={}, persistent=False): headers['time_queued'] = time.time() self.outbound_msg_queue_lazy.append({ 'destination': destination, 'body': body, 'headers': headers, 'persistent': persistent }) def send_event_lazy_thread(self): while self.continue_running: time.sleep(0.001) #If it gets there, then great, if not, well we tried! if (self.connection_issue == True): continue while len(self.outbound_msg_queue_lazy ) > 0 and self.connection_issue == False: try: item = self.outbound_msg_queue_lazy.pop(0) if item['persistent'] == True: item['headers']['persistent'] = 'true' else: item['headers']['persistent'] = 'false' item['headers']['time_sent'] = time.time() self.client.send(item['destination'], json.dumps(item['body']), item['headers']) except Exception as e: self.log.critical('Except: %s' % e) self.outbound_msg_queue_lazy.insert(0, item) self.connection_issue = True def send_event_hopeful(self, destination, body, persist): self.outbound_msg_queue.append({ 'destination': destination, 'body': body }) def send_event_hopeful_thread(self): while self.continue_running: time.sleep(0.001) if (self.connection_issue == True): continue while len(self.outbound_msg_queue ) > 0 and self.connection_issue == False: try: item = self.outbound_msg_queue.pop(0) self.client.send(item['destination'], json.dumps(item['body']), {'persistent': 'false'}) except Exception as e: self.log.critical('Except: %s' % e) self.outbound_msg_queue.insert(0, item) self.connection_issue = True def worker(self, queue): while self.continue_running: try: item = queue.get() for x in 1, 2, 3: try: item['callback'](item['callback_class'], item['data'], item['headers']) break except Exception as e: self.log.error('Exception in worker thread: %s' % e) traceback.print_exc() sys.exc_clear() time.sleep(0.01) queue.task_done() except: pass def publish_loop(self): self.log.debug('publish_loop() init') time.sleep(0.2) while self.continue_running: if self.connection_issue == False: try: if not self.client.canRead(0.01): continue try: frame = self.client.receiveFrame() data = json.loads(frame.body) queue = frame.headers['destination'] try: time_sent = float(frame.headers['time_sent']) time_queued = float(frame.headers['time_queued']) send_latency = (time.time() - time_sent) queue_latency = (time_sent - time_queued) if queue_latency > 0.1: print 'Queue Latency: %s' % (queue_latency) if send_latency > 0.1: print 'Send Latency: %s' % (send_latency) except: pass self.work_queue.put({ 'callback': self.subscriptions[queue]['callback'], 'callback_class': self.subscriptions[queue]['callback_class'], 'data': data, 'headers': frame.headers, }) #self.client.ack(frame) except Exception as e: raise self.log.critical('Except: %s' % e) #self.client.nack(frame) except Exception as e: self.log.fatal('except: %s' % e) self.connection_issue = True
#read the csv file values on column 5 #the csv file is created by running create_csv.py fields = ['id', 'num5'] df = pd.read_csv('tests.csv', skipinitialspace=True, usecols=fields) x = df.name.values q = str(x) #check to see if values C or G exist on column 5 #if yes create test db and store row values if 'C' or 'G' in q: conn = sqlite3.connect("test.sqlite") conn.execute("CREATE TABLE if not exists Data (id TEXT, col5 TEXT)") df = pd.read_csv("tests.csv") df.to_sql("Data", conn, if_exists='append', index=False) #check to see if only values C and G exist on column 5 #if yes create connect to apacheMq and store values elif 'C' and 'G'in q: CONFIG = StompConfig('tcp://localhost:61613') QUEUE = '/queue/test' if __name__ == '__main__': client = Stomp(CONFIG) client.connect() #client.send(QUEUE, json.dumps{(q)}.encode()) client.send(QUEUE, q.encode()) #client.send(QUEUE, 'test 2'.encode()) client.disconnect() else: print ('No matches found')
from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://localhost:61613') QUEUE = '/queue/test' if __name__ == '__main__': client = Stomp(CONFIG) client.connect() client.send(QUEUE, 'test message 1'.encode()) client.send(QUEUE, 'test message 2'.encode()) client.disconnect()
class StompClient(BaseComponent): channel = "stomp" def init( self, host, port, username=None, password=None, connect_timeout=3, connected_timeout=3, version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"], heartbeats=(0, 0), ssl_context=None, use_ssl=True, key_file=None, cert_file=None, ca_certs=None, ssl_version=ssl.PROTOCOL_SSLv23, # means 'latest available' key_file_password=None, proxy_host=None, proxy_port=None, proxy_user=None, proxy_password=None, channel=channel, stomp_params=None): """ Initialize StompClient. Called after __init__ """ self.channel = channel if proxy_host: LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port) else: LOG.info("Connect to %s:%s", host, port) if use_ssl and not ssl_context: ssl_params = dict(key_file=key_file, cert_file=cert_file, ca_certs=ca_certs, ssl_version=ssl_version, password=key_file_password) LOG.info("Request to use old-style socket wrapper: %s", ssl_params) ssl_context = ssl_params if use_ssl: uri = "ssl://%s:%s" % (host, port) else: uri = "tcp://%s:%s" % (host, port) # Configure failover options so it only tries based on settings # build any parameters passed # every connection has at least these two: maxReconnectAttempts, startupMaxReconnectAttempts items = [item.split("=", 2) for item in stomp_params.split(",")] if stomp_params else None connection_params = { item[0].strip(): item[1].strip() for item in items } if items else {} if "maxReconnectAttempts" not in connection_params: connection_params[ 'maxReconnectAttempts'] = DEFAULT_MAX_RECONNECT_ATTEMPTS if "startupMaxReconnectAttempts" not in connection_params: connection_params[ 'startupMaxReconnectAttempts'] = DEFAULT_STARTUP_MAX_RECONNECT_ATTEMPTS self._stomp_server = "failover:({0})?{1}".format( uri, ",".join( ["{}={}".format(k, v) for k, v in connection_params.items()])) LOG.debug("Stomp uri: {}".format(self._stomp_server)) self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context, version=version, login=username, passcode=password) self._heartbeats = heartbeats self._accept_versions = accept_versions self._connect_timeout = connect_timeout self._connected_timeout = connected_timeout Stomp._transportFactory = EnhancedStompFrameTransport Stomp._transportFactory.proxy_host = proxy_host Stomp._transportFactory.proxy_port = proxy_port Stomp._transportFactory.proxy_user = proxy_user Stomp._transportFactory.proxy_password = proxy_password self._client = Stomp(self._stomp_config) self._subscribed = {} self.server_heartbeat = None self.client_heartbeat = None self.last_heartbeat = 0 self.ALLOWANCE = 2 # multiplier for heartbeat timeouts @property def connected(self): if self._client.session: return self._client.session.state == StompSession.CONNECTED else: return False @property def socket_connected(self): try: if self._client._transport: return True except: pass return False @property def subscribed(self): return self._subscribed.keys() @property def stomp_logger(self): return LOG_CATEGORY @handler("Disconnect") def _disconnect(self, receipt=None, flush=True, reconnect=False): try: if flush: self._subscribed = {} if self.connected: self._client.disconnect(receipt=receipt) except Exception as e: LOG.error("Failed to disconnect client") try: self.fire(Disconnected(reconnect=reconnect)) self._client.close(flush=flush) except Exception as e: LOG.error("Failed to close client connection") return "disconnected" def start_heartbeats(self): LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat, self._client.serverHeartBeat) if self._client.clientHeartBeat: if self.client_heartbeat: # Timer already exists, just reset it self.client_heartbeat.reset() else: LOG.info("Client will send heartbeats to server") # Send heartbeats at 80% of agreed rate self.client_heartbeat = Timer( (self._client.clientHeartBeat / 1000.0) * 0.8, ClientHeartbeat(), persist=True) self.client_heartbeat.register(self) else: LOG.info("No Client heartbeats will be sent") if self._client.serverHeartBeat: if self.server_heartbeat: # Timer already exists, just reset it self.server_heartbeat.reset() else: LOG.info("Requested heartbeats from server.") # Allow a grace period on server heartbeats self.server_heartbeat = Timer( (self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE, ServerHeartbeat(), persist=True) self.server_heartbeat.register(self) else: LOG.info("Expecting no heartbeats from Server") @handler("Connect") def connect(self, event, host=None, *args, **kwargs): """ connect to Stomp server """ LOG.info("Connect to Stomp...") try: self._client.connect(heartBeats=self._heartbeats, host=host, versions=self._accept_versions, connectTimeout=self._connect_timeout, connectedTimeout=self._connected_timeout) LOG.debug("State after Connection Attempt: %s", self._client.session.state) if self.connected: LOG.info("Connected to %s", self._stomp_server) self.fire(Connected()) self.start_heartbeats() return "success" except StompConnectionError as err: LOG.debug(traceback.format_exc()) self.fire(ConnectionFailed(self._stomp_server)) event.success = False # This logic is added to trap the situation where resilient-circuits does not reconnect from a loss of connection # with the resilient server. In these cases, this error is not survivable and it's best to kill resilient-circuits. # If resilient-circuits is running as a service, it will restart and state would clear for a new stomp connection. except StompProtocolError as err: LOG.error(traceback.format_exc()) LOG.error("Exiting due to unrecoverable error") sys.exit(1) # this will exit resilient-circuits return "fail" @handler("ServerHeartbeat") def check_server_heartbeat(self, event): """ Confirm that heartbeat from server hasn't timed out """ now = time.time() self.last_heartbeat = self._client.lastReceived or self.last_heartbeat if self.last_heartbeat: elapsed = now - self.last_heartbeat else: elapsed = -1 if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE + self.last_heartbeat) < now: LOG.error( "Server heartbeat timeout. %d seconds since last heartbeat.", elapsed) event.success = False self.fire(HeartbeatTimeout()) @handler("ClientHeartbeat") def send_heartbeat(self, event): if self.connected: LOG.debug("Sending heartbeat") try: self._client.beat() except (StompConnectionError, StompError) as err: event.success = False self.fire(OnStompError(None, err)) @handler("generate_events") def generate_events(self, event): event.reduce_time_left(0.1) if not self.connected: return try: if self._client.canRead(0): frame = self._client.receiveFrame() LOG.debug("Recieved frame %s", frame) if frame.command == StompSpec.ERROR: self.fire(OnStompError(frame, None)) else: self.fire(Message(frame)) except (StompConnectionError, StompError) as err: LOG.error("Failed attempt to generate events.") self.fire(OnStompError(None, err)) @handler("Send") def send(self, event, destination, body, headers=None, receipt=None): LOG.debug("send()") try: self._client.send(destination, body=body.encode('utf-8'), headers=headers, receipt=receipt) LOG.debug("Message sent") except (StompConnectionError, StompError) as err: LOG.error("Error sending frame") event.success = False self.fire(OnStompError(None, err)) raise # To fire Send_failure event @handler("Subscribe") def _subscribe(self, event, destination, additional_headers=None, ack=ACK_CLIENT_INDIVIDUAL): if ack not in ACK_MODES: raise ValueError("Invalid client ack mode specified") if destination in self._client.session._subscriptions: LOG.debug("Ignoring subscribe request to %s. Already subscribed.", destination) LOG.info("Subscribe to message destination %s", destination) try: headers = {StompSpec.ACK_HEADER: ack, 'id': destination} if additional_headers: headers.update(additional_headers) # Set ID to match destination name for easy reference later frame, token = self._client.subscribe(destination, headers) self._subscribed[destination] = token except (StompConnectionError, StompError) as err: LOG.error("Failed to subscribe to queue.") event.success = False LOG.debug(traceback.format_exc()) self.fire(OnStompError(None, err)) # This logic is added to trap the situation where resilient-circuits does not reconnect from a loss of connection # with the resilient server. In these cases, this error is not survivable and it's best to kill resilient-circuits. # If resilient-circuits is running as a service, it will restart and state would clear for a new stomp connection. except StompProtocolError as err: LOG.error(traceback.format_exc()) LOG.error("Exiting due to unrecoverable error") sys.exit(1) # this will exit resilient-circuits @handler("Unsubscribe") def _unsubscribe(self, event, destination): if destination not in self._subscribed: LOG.error("Unsubscribe Request Ignored. Not subscribed to %s", destination) return try: token = self._subscribed.pop(destination) frame = self._client.unsubscribe(token) LOG.debug("Unsubscribed: %s", frame) except (StompConnectionError, StompError) as err: event.success = False LOG.error("Unsubscribe Failed.") self.fire(OnStompError(frame, err)) @handler("Message") def on_message(self, event, headers, message, queue): LOG.debug("Stomp message received") @handler("Ack") def ack_frame(self, event, frame): LOG.debug("ack_frame()") try: self._client.ack(frame) LOG.debug("Ack Sent") except (StompConnectionError, StompError) as err: LOG.error("Error sending ack") event.success = False self.fire(OnStompError(frame, err)) raise # To fire Ack_failure event def get_subscription(self, frame): """ Get subscription from frame """ _, token = self._client.message(frame) return self._subscribed[token]
''' @author: Saravana Perumal Shanmugam ''' from stompest.config import StompConfig from stompest.sync import Stomp from datetime import date import time import logging CONFIG = StompConfig('tcp://localhost:61613') QUEUE = 'jms.queue.prodcons' if __name__ == '__main__': logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) client = Stomp(CONFIG) client.connect() for index in range(10): client.send(QUEUE, 'A Message ' + str(index) + ' At ' + str(date.today())) time.sleep(1) client.disconnect()
logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://localhost:7777', version='1.2') if __name__ == '__main__': client = Stomp(CONFIG) client.connect(connectedTimeout=4) ID = 'bingo' client.send('/my/test/destination', # destination 'THIS-IS-A-SEND-BODY', # body headers={'receipt': ID}) # headers answer = client.receiveFrame() receiptID = answer.headers["receipt-id"] returnValue = 0 if receiptID != ID: print "Receipt header wrong:" + receiptID returnValue = 1 else: print "Correct receipt id received: " + receiptID client.disconnect() sys.exit(returnValue)
port = int(os.getenv('ACTIVEMQ_PORT') or 61613) destination = sys.argv[1:2] or ['/topic/event'] destination = destination[0] messages = 10000 data = 'Hello World from Python' config = StompConfig('tcp://%s:%d' % (host, port), login=user, passcode=password, version='1.1') client = Stomp(config) client.connect(host='mybroker') count = 0 start = time.time() for _ in xrange(messages): send_data = "".join((data, " ", str(count))) client.send(destination=destination, body=send_data, headers={'persistent': 'false'}) # print("Send msg [ %s ] to destination [ %s ]" % (send_data, destination)) count += 1 diff = time.time() - start print 'Sent %s frames in %f seconds' % (count, diff) client.disconnect(receipt='bye') client.receiveFrame() client.close()
frame = client.receiveFrame() try: payload = json.loads(frame.body) except json.JSONDecoder: print("Failed to decode message body: frame.body was {}".format( frame.body)) with open('/opt/apache-activemq/sendfail/malformed', 'a+') as mf: mf.write('{0}\n{1}'.format(frame.headers, frame.body)) mf.close() continue headers = frame.headers print('Sending the following message to {} : '.format(to_queue)) print('body: {}'.format(payload)) print('headers: {}'.format(frame.headers)) try: client.ack(frame) except stompest.error.StompConnectionError: print('Unable to send ACK to {}'.format(broker)) try: headers['persistent'] = 'true' client.send(to_queue, body=json.dumps(payload), headers=headers) except stompest.error.StompError: print( 'Sending message to {} failed. Writing message locally at /opt/apache-activemq/sendfail' .format(broker)) with open('/opt/apache-activemq/sendfail/{}\n'.format(from_queue), 'a+') as sf: sf.write(json.dumps(payload)) client.disconnect()
import json from stompest.config import StompConfig from stompest.sync import Stomp CONFIG = StompConfig('tcp://*****:*****@mail.nih.gov', 'timeStamp': '2015-06-25', 'outputDir': '/home/user/python/testOutDir' })) client.disconnect()
# -*- coding:utf-8 -*- import logging from stompest.config import StompConfig from stompest.sync import Stomp logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) uri = 'failover:(tcp://x:61613,tcp://y:61613,tcp://z:61613)?randomize=false,startupMaxReconnectAttempts=3,initialReconnectDelay=7,maxReconnectDelay=8,maxReconnectAttempts=0' CONFIG = StompConfig(uri) QUEUE = '/queue/liuyang-test' if __name__ == '__main__': client = Stomp(CONFIG) client.connect() client.send(QUEUE, 'test message 1') client.send(QUEUE, 'test message 2') client.disconnect()
client = Stomp(config) client.connect(host=host) body = """ <corpus> <docelems> <docelem> <uiid>scai.fhg.de/voc/ccg</uiid> <model>NS Voc\nID CCG\nNA Cologne Center for Genomics</model> </docelem> <docelem> <uiid>scai.fhg.de/semtype/1000</uiid> <model>NS SemType\nID 1000\nNA chemical compound\nDF A chemical compound (or just compound if used in the context of chemistry) is an entity consisting of two or more different atoms which associate via chemical bonds.</model> </docelem> <docelem> <uiid>scai.fhg.de/concept/1</uiid> <model>ID 1\nNA Ceritinib\nTM Ceritinib\nTM LDK378\nTM ZYKADIA\t@match=ci\nTM 1032900-25-6\nVO CCG</model> </docelem> </docelems> </corpus> """ client.send(destination=destination, body=body, headers={'transformation': 'TEXT', 'event': 'FoundCorpus'}) print "Updating Ontology" client.disconnect(receipt='bye') client.receiveFrame() client.close() sys.exit(0)
def enqueue(queue_url, queue_name, data): """ Sends a message to a queue """ client = Stomp(StompConfig(queue_url)) client.connect() client.send(queue_name, data) client.disconnect()
from stompest.config import StompConfig from stompest.sync import Stomp user = os.getenv('APOLLO_USER') or 'admin' password = os.getenv('APOLLO_PASSWORD') or 'password' host = os.getenv('APOLLO_HOST') or 'localhost' port = int(os.getenv('APOLLO_PORT') or 61613) destination = sys.argv[1:2] or ['/topic/event'] destination = destination[0] messages = 10000 data = 'Hello World from Python' config = StompConfig('tcp://%s:%d' % (host, port), login=user, passcode=password, version='1.1') client = Stomp(config) client.connect(host='mybroker') count = 0 start = time.time() for _ in xrange(messages): client.send(destination=destination, body=data, headers={'persistent': 'false'}) count += 1 diff = time.time() - start print 'Sent %s frames in %f seconds' % (count, diff) client.disconnect(receipt='bye') client.receiveFrame() client.close()
class PostProcessAdmin: def __init__(self, data, conf): logging.debug("json data: %s [%s]" % (str(data), type(data))) if not type(data) == dict: raise ValueError, "PostProcessAdmin expects a data dictionary" data["information"] = socket.gethostname() self.data = data self.conf = conf # List of error messages to be handled as information self.exceptions = self.conf.exceptions stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user, self.conf.amq_pwd) self.client = Stomp(stompConfig) self.data_file = None self.facility = None self.instrument = None self.proposal = None self.run_number = None def _process_data(self, data): """ Retrieve run information from the data dictionary provided with an incoming message. @param data: data dictionary """ if data.has_key('data_file'): self.data_file = str(data['data_file']) if os.access(self.data_file, os.R_OK) == False: raise ValueError("Data file does not exist or is not readable: %s" % self.data_file) else: raise ValueError("data_file is missing: %s" % self.data_file) if data.has_key('facility'): self.facility = str(data['facility']).upper() else: raise ValueError("Facility is missing") if data.has_key('instrument'): self.instrument = str(data['instrument']).upper() else: raise ValueError("Instrument is missing") if data.has_key('ipts'): self.proposal = str(data['ipts']).upper() else: raise ValueError("IPTS is missing") if data.has_key('run_number'): self.run_number = str(data['run_number']) else: raise ValueError("Run number is missing") def reduce(self, remote=False): """ Reduction process using job submission. @param remote: If True, the job will be submitted to a compute node """ self._process_data(self.data) try: self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data)) instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce') proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce') log_dir = os.path.join(proposal_shared_dir, "reduction_log") if not os.path.exists(log_dir): os.makedirs(log_dir) # Allow for an alternate output directory, if defined if len(self.conf.dev_output_dir.strip()) > 0: proposal_shared_dir = self.conf.dev_output_dir logging.info("Using output directory: %s" % proposal_shared_dir) # Look for run summary script summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument) if os.path.exists(summary_script) == True: summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal)) cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output logging.debug("Run summary subprocess started: " + cmd) subprocess.call(cmd, shell=True) logging.debug("Run summary subprocess completed, see " + summary_output) # Look for auto-reduction script reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument) if os.path.exists(reduce_script_path) == False: self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data)) return # Run the reduction out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log") out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err") if remote: job_handling.remote_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) else: job_handling.local_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) # Determine error condition success, status_data = job_handling.determine_success_local(self.conf, out_err) self.data.update(status_data) if success: if os.path.isfile(out_err): os.remove(out_err) self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data)) else: self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data)) except: logging.error("reduce: %s" % sys.exc_value) self.data["error"] = "Reduction: %s " % sys.exc_value self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data)) def catalog_raw(self): """ Catalog a nexus file containing raw data """ self._process_data(self.data) try: from ingest_nexus import IngestNexus self.send('/queue/' + self.conf.catalog_started, json.dumps(self.data)) if self.conf.comm_only is False: ingestNexus = IngestNexus(self.data_file) ingestNexus.execute() ingestNexus.logout() self.send('/queue/' + self.conf.catalog_complete, json.dumps(self.data)) except: logging.error("catalog_raw: %s" % sys.exc_value) self.data["error"] = "Catalog: %s" % sys.exc_value self.send('/queue/' + self.conf.catalog_error, json.dumps(self.data)) def catalog_reduced(self): """ Catalog reduced data files for a given run """ self._process_data(self.data) try: from ingest_reduced import IngestReduced self.send('/queue/' + self.conf.reduction_catalog_started, json.dumps(self.data)) if self.conf.comm_only is False: # Send image to the web monitor if len(self.conf.web_monitor_url.strip()) > 0: monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd} proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce') url_template = string.Template(self.conf.web_monitor_url) url = url_template.substitute(instrument=self.instrument, run_number=self.run_number) pattern = self.instrument + "_" + self.run_number + "*" for dirpath, dirnames, filenames in os.walk(proposal_shared_dir): listing = glob.glob(os.path.join(dirpath, pattern)) for filepath in listing: f, e = os.path.splitext(filepath) if e.startswith(os.extsep): e = e[len(os.extsep):] if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat") or filepath.endswith("plot_data.json"): files = {'file': open(filepath, 'rb')} # Post the file if it's small enough if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size: request = requests.post(url, data=monitor_user, files=files, verify=False) logging.info("Submitted %s [status: %s]" % (filepath, request.status_code)) ingestReduced = IngestReduced(self.facility, self.instrument, self.proposal, self.run_number) ingestReduced.execute() ingestReduced.logout() self.send('/queue/' + self.conf.reduction_catalog_complete , json.dumps(self.data)) except: logging.error("catalog_reduced: %s" % sys.exc_value) self.data["error"] = "Reduction catalog: %s" % sys.exc_value self.send('/queue/' + self.conf.reduction_catalog_error , json.dumps(self.data)) def create_reduction_script(self): """ Create a new reduction script from a template """ try: import reduction_script_writer writer = reduction_script_writer.ScriptWriter(self.data["instrument"]) writer.process_request(self.data, configuration=self.conf, send_function=self.send) except: logging.error("create_reduction_script: %s" % sys.exc_value) def send(self, destination, data): """ Send an AMQ message @param destination: AMQ queue to send to @param data: payload of the message """ logging.info("%s: %s" % (destination, data)) self.client.connect() self.client.send(destination, data) self.client.disconnect()
client.subscribe(from_queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) while client.canRead(timeout=tmout): frame = client.receiveFrame() try: payload = json.loads(frame.body) except json.JSONDecoder: print("Failed to decode message body: frame.body was {}".format(frame.body)) with open('/opt/apache-activemq/sendfail/malformed', 'a+') as mf: mf.write('{0}\n{1}'.format(frame.headers, frame.body)) mf.close() continue headers = frame.headers print('Sending the following message to {} : '.format(to_queue)) print('body: {}'.format(payload)) print('headers: {}'.format(frame.headers)) try: client.ack(frame) except stompest.error.StompConnectionError: print('Unable to send ACK to {}'.format(broker)) try: headers['persistent'] = 'true' client.send(to_queue, body=json.dumps(payload), headers=headers) except stompest.error.StompError: print('Sending message to {} failed. Writing message locally at /opt/apache-activemq/sendfail'.format(broker)) with open('/opt/apache-activemq/sendfail/{}\n'.format(from_queue), 'a+') as sf: sf.write(json.dumps(payload)) client.disconnect()
from stompest.config import StompConfig from stompest.sync import Stomp client = Stomp(StompConfig('tcp://activemq:61613')) client.connect() client.send('/queue/test', 'request sent') client.disconnect()
pp.create_reduction_script() # Check for registered processors if type(configuration.processors) == list: for p in configuration.processors: toks = p.split('.') if len(toks) == 2: processor_module = __import__("postprocessing.processors.%s" % toks[0], globals(), locals(), [toks[1], ], -1) try: processor_class = eval("processor_module.%s" % toks[1]) if namespace.queue == processor_class.get_input_queue_name(): # Instantiate and call the processor proc = processor_class(data, configuration, send_function=pp.send) proc() except: logging.error("PostProcessAdmin: Processor error: %s" % sys.exc_value) else: logging.error("PostProcessAdmin: Processors can only be specified in the format module.Processor_class") except: # If we have a proper data dictionary, send it back with an error message if type(data) == dict: data["error"] = str(sys.exc_value) stomp = Stomp(StompConfig(configuration.failover_uri, configuration.amq_user, configuration.amq_pwd)) stomp.connect() stomp.send(configuration.postprocess_error, json.dumps(data)) stomp.disconnect() raise except: logging.error("PostProcessAdmin: %s" % sys.exc_value)
# 'transformation': 'TEXT', # 'event': 'QueryDocelem', # 'reply-to': '/topic/docstore-reply' # } body = """<query> <annot>scai.fraunhofer.de/prominer-entry/hs00001</annot> <annot>scai.fraunhofer.de/prominer-entry/hs00002</annot> </query>""" headers = { 'transformation': 'TEXT', 'event': 'QueryAnnotationIndex', 'reply-to': '/topic/docstore-reply' } print "Sending to " + destination client.send(destination=destination, body=body, headers=headers) clientConsumer.subscribe(destination='/topic/docstore-reply', headers={'id': 'required-for-STOMP-1.1'}) while True: startReceive = time.time() frame = clientConsumer.receiveFrame() diffReceive = time.time() - startReceive print frame print diffReceive time.sleep(1) print "Sending to " + destination client.send(destination=destination, body=body, headers=headers)
from stompest.config import StompConfig from stompest.protocol import StompSpec from stompest.sync import Stomp CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1) QUEUE = '/queue/test' if __name__ == '__main__': client = Stomp(CONFIG) client.connect(heartBeats=(0, 10000)) client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) client.send(QUEUE, 'test message 1') client.send(QUEUE, 'test message 2') while True: frame = client.receiveFrame() print 'Got %s' % frame.info() client.ack(frame) client.disconnect()
import ssl from stompest.config import StompConfig from stompest.protocol import StompSpec from stompest.sync import Stomp context = ssl.create_default_context() # Disable cert validation for demo only context.check_hostname = False context.verify_mode = ssl.CERT_NONE CONFIG = StompConfig(uri='ssl://x.x.x.x:61614', login='******', passcode='password', check=False, sslContext=context) QUEUE = '/queue/my-queue' RQUEUE = '/queue/my-reply' if __name__ == '__main__': client = Stomp(CONFIG) client.connect() client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) frame = client.receiveFrame() print('Got %s' % frame.info()) client.ack(frame) client.send(RQUEUE, b'hello', headers={'correlation-id': 0}) client.disconnect()