def main():
logging.basicConfig()
logging.getLogger().setLevel(logging.WARN)
client = Stomp(stomp_config)
client.connect()
client.send(stomp_queue, body=stomp_body)
client.subscribe(stomp_queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT, 'activemq.prefetchSize': 1})
if client.canRead(timeout=5):
frame = client.receiveFrame()
print 'Got %s' % frame.info()
client.ack(frame)
frame_body = str(frame.body)
if frame_body == stomp_body:
print "OK: Message received"
status = 'ok'
else:
print "WARNING: Incorrect message body; is %s, should be %s" % (frame_body, stomp_body)
status = 'warning'
else:
print "CRITICAL: Timed out while trying to collect the message"
status = 'critical'
client.disconnect()
client.close(flush=True)
return exit_codes[status]
def test_4_integration_stomp_1_1(self):
if StompSpec.VERSION_1_1 not in commands.versions(VERSION):
print 'This broker does not support STOMP protocol version 1.1'
return
client = Stomp(self.getConfig(StompSpec.VERSION_1_1))
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, 'test message 1')
client.send(self.DESTINATION, 'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.send(self.DESTINATION, 'test message 3', receipt='4711')
self.assertTrue(client.canRead(self.TIMEOUT))
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'}))
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect(receipt='4712')
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4712'}))
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.disconnect(receipt='4711')
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'}))
client.close()
self.assertRaises(StompConnectionError, client.canRead, 0)
def writeJson(value,*argv):
'''Writes the specified value to an output file
Takes:
value-> List or Dict
*argv: Available options:
'stomp'
'post'
'outfile'
Returns:
none
'''
outJson=json.dumps(value)
print outJson
if 'stomp' in argv:
CONFIG = StompConfig(stomp_config.server, stomp_config.login,stomp_config.passcode)
QUEUE = stomp_config.queue
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, outJson)
client.disconnect()
elif 'outfile' in argv:
with open(outfile,'w') as jsonFile:
jsonFile.write(outJson)
elif 'post' in argv:
#TODO Post to php server
pass
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self):
if BROKER == 'rabbitmq':
print 'Broker does not support unicode characters. Skipping this test case.'
return
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)
return
specialCharactersHeader = u'fen\xeatre:\r\n'
headers = {specialCharactersHeader: u'\xbfqu\xe9 tal?, s\xfc\xdf'}
client.send(self.DESTINATION, body='test message 1', headers=headers)
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
client.ack(frame)
self.assertEquals(frame.version, version)
self.assertEquals(frame.headers[specialCharactersHeader], headers[specialCharactersHeader])
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.disconnect(receipt='4712')
def _test_4_integration_stomp(self, version):
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version))
return
client.send(self.DESTINATION, b'test message 1')
client.send(self.DESTINATION, b'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.send(self.DESTINATION, b'test message 3', receipt='4711')
self.assertTrue(client.canRead(self.TIMEOUT))
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'}))
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect(receipt='4712')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712'}))
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.disconnect(receipt='4711')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'}))
client.close()
self.assertRaises(StompConnectionError, client.canRead, 0)
def call_route(request_queue, response_queue, request):
"""
"""
config = {
"stomp": {
"server": '192.168.0.3',
"port": 61613,
"timeout": 15,
}
}
stomp_config = StompConfig("tcp://%s:%d" % (config['stomp']['server'], config['stomp']['port']), version=StompSpec.VERSION_1_0)
stomp = Stomp(stomp_config)
stomp.connect()
jms_id = str(uuid4())
token = stomp.subscribe(response_queue, {'JMSCorrelationID': jms_id})
stomp.send(request_queue, json.dumps(request), {'JMSCorrelationID': jms_id})
response = None
if stomp.canRead(config['stomp']['timeout']):
response = stomp.receiveFrame()
stomp.unsubscribe(token)
return response
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self):
if BROKER == 'rabbitmq':
print('Broker does not support unicode characters. Skipping this test case.')
return
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version))
return
key = b'fen\xc3\xaatre'.decode('utf-8')
value = b'\xc2\xbfqu\xc3\xa9 tal?'.decode('utf-8')
headers = {key: value}
client.send(self.DESTINATION, body=b'test message 1', headers=headers)
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
client.ack(frame)
self.assertEqual(frame.version, version)
self.assertEqual(frame.headers[key], headers[key])
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.disconnect(receipt='4712')
def test_1_integration(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 1')
client.send(self.DESTINATION, b'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
def run(self):
files = os.listdir(self.path)
for queue_file in files:
archive_file = '{0}/archive/{1}'.format(self.path, queue_file)
if queue_file.startswith('archive'):
pass
else:
with open("{0}/{1}".format(self.path, queue_file), 'r') as qf:
get_lines = list(qf)
for line in get_lines:
dts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
client = Stomp(config=self.config)
client.connect()
print "Sending message {0} to queue {1}".format(line, queue_file)
with open(archive_file, 'a') as af:
af.write("{0} Sent message: {1} to queue {2}\n".format(dts, line, queue_file))
client.send(queue_file, json.dumps(line))
client.disconnect()
def test_3_socket_failure_and_replay(self):
client = Stomp(self.getConfig(StompSpec.VERSION_1_0))
client.connect(host=VIRTUALHOST)
headers = {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}
token = client.subscribe(self.DESTINATION, headers)
client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 1')
client.ack(client.receiveFrame())
client.unsubscribe(token)
headers = {StompSpec.ID_HEADER: 'bla', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}
client.subscribe(self.DESTINATION, headers)
headers[StompSpec.DESTINATION_HEADER] = self.DESTINATION
client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 2')
client.ack(client.receiveFrame())
client.unsubscribe((StompSpec.ID_HEADER, 'bla'))
client.disconnect()
def send_message(messageBody, destination=None, queueName=None):
client = None
if destination != None:
client = Stomp(StompConfig(destination))
else:
client = Stomp(StompConfig("tcp://localhost:61613"))
QUEUE = None
if queueName != None:
QUEUE = queueName
else:
QUEUE = "pods2jbpm"
#client = Stomp(CONFIG)
client.connect()
body = messageBody
client.send(QUEUE, body)
client.disconnect()
def send(self):
"""
Create a new stomp configuration client, connect and
then serializes message by message posting
them to your consumers in TOPIC standard
and disconnect.
"""
try:
configuration = StompConfig(uri=self._broker_uri)
client = Stomp(configuration)
client.connect(connectTimeout=self._broker_timeout)
for message in self._queue:
serialized_message = json.dumps(message, ensure_ascii=False)
client.send(self._queue_destination, serialized_message)
client.disconnect()
except Exception, e:
self.log.error(u"QueueManagerError - Error on sending objects from queue.")
self.log.debug(e)
def test_2_transaction(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertFalse(client.canRead(self.TIMEOUT))
with client.transaction(4711) as transaction:
self.assertEqual(transaction, '4711')
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
self.assertFalse(client.canRead(0))
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEqual(frame.body, b'test message')
client.ack(frame)
with client.transaction(4713, receipt='4712') as transaction:
self.assertEqual(transaction, '4713')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-begin'}))
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
client.send(self.DESTINATION, b'test message without transaction')
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEqual(frame.body, b'test message without transaction')
client.ack(frame)
self.assertFalse(client.canRead(0))
frames = [client.receiveFrame() for _ in range(2)]
frames = list(sorted(frames, key=lambda f: f.command))
frame = frames[0]
client.ack(frame)
self.assertEqual(frame.body, b'test message')
frame = frames[1]
self.assertEqual(frame, StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-commit'}))
try:
with client.transaction(4714) as transaction:
self.assertEqual(transaction, '4714')
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
raise RuntimeError('poof')
except RuntimeError as e:
self.assertEqual(str(e), 'poof')
else:
raise
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect()
class StompClient(BaseComponent):
""" Send and Receive messages from a STOMP queue """
channel = "stomp"
def init(self, host, port, username=None, password=None,
connect_timeout=3, connected_timeout=3,
version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"],
heartbeats=(0, 0), ssl_context=None,
use_ssl=True,
key_file=None,
cert_file=None,
ca_certs=None,
ssl_version=ssl.PROTOCOL_SSLv23,
key_file_password=None,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
channel=channel):
""" Initialize StompClient. Called after __init__ """
self.channel = channel
if proxy_host:
LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port)
else:
LOG.info("Connect to %s:%s", host, port)
if use_ssl and not ssl_context:
ssl_params = dict(key_file=key_file,
cert_file=cert_file,
ca_certs=ca_certs,
ssl_version=ssl_version,
password=key_file_password)
LOG.info("Request to use old-style socket wrapper: %s", ssl_params)
ssl_context = ssl_params
if use_ssl:
uri = "ssl://%s:%s" % (host, port)
else:
uri = "tcp://%s:%s" % (host, port)
# Configure failover options so it only tries to connect once
self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri
self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context,
version=version,
login=username,
passcode=password)
self._heartbeats = heartbeats
self._accept_versions = accept_versions
self._connect_timeout = connect_timeout
self._connected_timeout = connected_timeout
Stomp._transportFactory = EnhancedStompFrameTransport
Stomp._transportFactory.proxy_host = proxy_host
Stomp._transportFactory.proxy_port = proxy_port
Stomp._transportFactory.proxy_user = proxy_user
Stomp._transportFactory.proxy_password = proxy_password
self._client = Stomp(self._stomp_config)
self._subscribed = {}
self.server_heartbeat = None
self.client_heartbeat = None
self.ALLOWANCE = 2 # multiplier for heartbeat timeouts
@property
def connected(self):
if self._client.session:
return self._client.session.state == StompSession.CONNECTED
else:
return False
@property
def subscribed(self):
return self._subscribed.keys()
@property
def stomp_logger(self):
return LOG_CATEGORY
@handler("disconnect")
def _disconnect(self, receipt=None):
if self.connected:
self._client.disconnect(receipt=receipt)
self._client.close(flush=True)
self.fire(disconnected(reconnect=False))
self._subscribed = {}
return "disconnected"
def start_heartbeats(self):
LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat, self._client.serverHeartBeat)
if self._client.clientHeartBeat:
if self.client_heartbeat:
# Timer already exists, just reset it
self.client_heartbeat.reset()
else:
LOG.info("Client will send heartbeats to server")
# Send heartbeats at 80% of agreed rate
self.client_heartbeat = Timer((self._client.clientHeartBeat / 1000.0) * 0.8,
client_heartbeat(), persist=True)
self.client_heartbeat.register(self)
else:
LOG.info("No Client heartbeats will be sent")
if self._client.serverHeartBeat:
if self.server_heartbeat:
# Timer already exists, just reset it
self.server_heartbeat.reset()
else:
LOG.info("Requested heartbeats from server.")
# Allow a grace period on server heartbeats
self.server_heartbeat = Timer((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE,
server_heartbeat(), persist=True)
self.server_heartbeat.register(self)
else:
LOG.info("Expecting no heartbeats from Server")
@handler("connect")
def connect(self, event, host=None, *args, **kwargs):
""" connect to Stomp server """
LOG.info("Connect to Stomp...")
try:
self._client.connect(heartBeats=self._heartbeats,
host=host,
versions=self._accept_versions,
connectTimeout=self._connect_timeout,
connectedTimeout=self._connected_timeout)
LOG.info("State after Connection Attempt: %s", self._client.session.state)
if self.connected:
LOG.info("Connected to %s", self._stomp_server)
self.fire(connected())
self.start_heartbeats()
return "success"
except StompConnectionError:
LOG.debug(traceback.format_exc())
self.fire(connection_failed(self._stomp_server))
event.success = False
return "fail"
@handler("server_heartbeat")
def check_server_heartbeat(self, event):
""" Confirm that heartbeat from server hasn't timed out """
now = time.time()
last = self._client.lastReceived or 0
if last:
elapsed = now - last
else:
elapsed = -1
LOG.debug("Last received data %d seconds ago", elapsed)
if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE + last) < now:
LOG.error("Server heartbeat timeout. %d seconds since last heartbeat. Disconnecting.", elapsed)
event.success = False
self.fire(heartbeat_timeout())
if self.connected:
self._client.disconnect()
# TODO: Try to auto-reconnect?
@handler("client_heartbeat")
def send_heartbeat(self, event):
if self.connected:
LOG.debug("Sending heartbeat")
try:
self._client.beat()
except StompConnectionError:
event.success = False
self.fire(disconnected())
@handler("generate_events")
def generate_events(self, event):
if not self.connected:
return
try:
if self._client.canRead(1):
frame = self._client.receiveFrame()
LOG.debug("Recieved frame %s", frame)
self.fire(message(frame))
except StompConnectionError:
self.fire(disconnected())
@handler("send")
def send(self, event, destination, body, headers=None, receipt=None):
LOG.debug("send()")
if not self.connected:
LOG.error("Can't send when Stomp is disconnected")
self.fire(on_stomp_error(None, Exception("Message send attempted with stomp disconnected")))
event.success = False
return
try:
self._client.send(destination, body=body.encode('utf-8'), headers=headers, receipt=receipt)
LOG.debug("Message sent")
except StompConnectionError as err:
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(None, err))
@handler("subscribe")
def _subscribe(self, event, destination, ack=ACK_CLIENT_INDIVIDUAL):
if ack not in ACK_MODES:
raise ValueError("Invalid client ack mode specified")
LOG.info("Subscribe to message destination %s", destination)
try:
# Set ID to match destination name for easy reference later
frame, token = self._client.subscribe(destination,
headers={StompSpec.ACK_HEADER: ack,
'id': destination})
self._subscribed[destination] = token
except StompConnectionError as err:
event.success = False
self.fire(disconnected())
except StompError as err:
event.success = False
LOG.debug(traceback.format_exc())
self.fire(on_stomp_error(None, err))
@handler("unsubscribe")
def _unsubscribe(self, event, destination):
if destination not in self._subscribed:
LOG.error("Unsubscribe Request Ignored. Not subscribed to %s", destination)
return
try:
token = self._subscribed.pop(destination)
frame = self._client.unsubscribe(token)
LOG.debug("Unsubscribed: %s", frame)
except StompConnectionError as err:
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(frame, err))
@handler("message")
def on_message(self, event, headers, message):
LOG.info("Stomp message received")
@handler("ack")
def ack_frame(self, event, frame):
LOG.debug("ack_frame()")
try:
self._client.ack(frame)
LOG.debug("Ack Sent")
except StompConnectionError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(frame, err))
def get_subscription(self, frame):
""" Get subscription from frame """
LOG.info(self._subscribed)
_, token = self._client.message(frame)
return self._subscribed[token]
class Client(object):
def __init__(self):
self.stompest = None
self.greenlet = None
self.subscriptions = {}
self._last_id = 0
def _next_id(self):
self._last_id += 1
return self._last_id
def connect(self):
if not self.stompest:
CONFIG, EXTRA = _get_config()
self._hostname = EXTRA.get('hostname', None)
self.stompest = Stomp(CONFIG)
if self.stompest.session.state != StompSession.DISCONNECTED:
return
while True:
try:
self.stompest.connect(host=self._hostname)
logger.info('Connected')
break
except StompConnectTimeout:
continue
if not self.greenlet:
self.greenlet = gevent.spawn(self._run)
def _run(self):
while True:
try:
frame = self.stompest.receiveFrame()
self.stompest.ack(frame)
if frame.command == 'ERROR':
logger.error(frame.info())
elif frame.command == 'MESSAGE':
token = self.stompest.message(frame)
if self.subscriptions.get(token):
subscription = self.subscriptions[token]
subscription.call(frame)
else:
logger.error("Received a message for %s (%s) but there was no matching subscription."
% (frame.headers.get(StompSpec.DESTINATION_HEADER, '???'), token))
else:
logger.warning("Unknown frame: %s" % frame.info())
# @todo Handle receipts
except (gevent.GreenletExit, KeyboardInterrupt):
# @todo Include a receipt in the disconnect. And instead of breaking right away wait for the
# receipt frame before disconnecting and consider waiting on any greenlets we started.
self.stompest.disconnect()
break
except StompConnectionError:
# We've been disconnected from the server. Try reconnecting to it.
self.connect()
def on(self, destination, callback):
self.connect()
token = self.stompest.subscribe(destination, {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
StompSpec.ID_HEADER: self._next_id(),
})
subscription = Subscription(
conn=self,
destination=destination,
token=token,
callback=callback
)
self.subscriptions[subscription.token] = subscription;
# @todo consider adding optional support for additional custom headers
def send(self, cmd, destination):
self.connect()
body = json.dumps(cmd)
headers = {}
headers[StompSpec.CONTENT_TYPE_HEADER] = 'application/json;charset=UTF-8'
self.stompest.send(destination, body, headers)
def join(self):
try:
self.connect()
except (gevent.GreenletExit, KeyboardInterrupt):
return
try:
gevent.joinall([self.greenlet])
except KeyboardInterrupt:
self.greenlet.kill(block=True)
def calculate():
pathwayConfig = app.config[Pathway.CONFIG]
try:
ts = str(time.time())
parameters = dict(request.form)
for field in parameters:
parameters[field] = parameters[field][0]
parameters['idstr'] = ts
filelist = request.files
studyList = []
num_studies = int(parameters['num_studies'])
for i in xrange(1,num_studies+1):
studyKey = "study_" + str(i)
studyObj = {}
studyObj['lambda'] = parameters['lambda_' + str(i)]
del parameters['lambda_'+str(i)]
studyObj['sample_sizes'] = []
for resourceInd in range(1,int(parameters['num_resource_' + str(i)])+1):
studyObj['sample_sizes'].append(parameters['sample_size_' + str(i) + '_' + str(resourceInd)])
del parameters['sample_size_' + str(i) + '_' + str(resourceInd)]
del parameters['num_resource_' + str(i)]
studyFile = filelist[studyKey]
if studyFile.filename:
filename = os.path.join(os.getcwd(),app.config['UPLOAD_FOLDER'],ts + '-' + str(i) + '.study')
studyObj['filename'] = filename
studyFile.save(filename)
else:
return Pathway.buildFailure("The file seems to be missing from Study #" + i + ".")
studyList.append(studyObj)
del parameters['num_studies']
parameters['studies'] = studyList
if parameters['pathway_type'] == 'file_pathway':
pathFile = filelist['file_pathway']
if pathFile.filename:
filename = os.path.join(app.config['UPLOAD_FOLDER'],ts + '.pathway')
parameters['pathway'] = filename
pathFile.save(filename)
else:
return Pathway.buildFailure("The pathway file seems to be missing.")
elif parameters['pathway_type'] == 'database_pathway':
parameters['pathway'] = os.path.join(app.config['PATHWAY_FOLDER'],parameters['database_pathway'])
else:
return Pathway.buildFailure("The pathway file seems to be missing.")
if "database_pathway" in parameters:
del parameters['database_pathway']
if "selectAll" in parameters:
del parameters['selectAll']
if "selectItem" in parameters:
del parameters['selectItem']
superpop = {}
subpop = {}
for population in parameters['populations'].split(","):
population = population.split('|')
if os.path.isfile(os.path.join(app.config['POPULATION_FOLDER'],population[0],population[1]+'.txt')):
superpop[population[0]] = 1
subpop_file = os.path.join(app.config['POPULATION_FOLDER'],population[0],population[1]+'.txt')
with open(subpop_file, 'r') as subpop_file:
for line in subpop_file:
subpop[line.strip()] = 1
else:
return Pathway.buildFailure("An invalid population was submitted.")
if (len(superpop) > 1):
return Pathway.buildFailure("An invalid population was submitted.")
del parameters['populations']
for population in superpop:
parameters['plink'] = app.config['PLINK_PATTERN'].replace("$pop",population)
parameters['population'] = []
for population in subpop:
parameters['population'].append(population)
parameters['outdir'] = app.config['OUT_FOLDER']
parameters['refinep'] = parameters.get('refinep',"").lower() in ['true','t','1']
parameters['gene_subset'] = parameters.get('gene_subset',"").lower() in ['true','t','1']
jsonout = {"submittedTime": parameters['idstr'], "payload": parameters}
with open(os.path.join(app.config['OUT_FOLDER'],str(parameters['idstr'])+'.json'),'w') as outfile:
json.dump(jsonout,outfile)
client = Stomp(pathwayConfig[Pathway.QUEUE_CONFIG])
client.connect()
client.send(pathwayConfig.getAsString(Pathway.QUEUE_NAME), json.dumps(parameters))
client.disconnect()
return Pathway.buildSuccess("The request has been received. An email will be sent when the calculation has completed.")
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("EXCEPTION------------------------------", exc_type, fname, exc_tb.tb_lineno)
return Pathway.buildFailure(str(e))
clientConsumer = Stomp(config)
clientConsumer.connect(host=host)
# Subscribe to the topic where the answers are posted to
clientConsumer.subscribe(destination='/topic/docstore-reply', headers={'id': 'required-for-STOMP-1.1'})
# Send a query
queries = range(1,15)
headers = {
'transformation': 'TEXT',
'event': 'QueryDocelem',
'reply-to': '/topic/docstore-reply'
}
client.send(destination=destination, body="scai.fhg.de/voc/ccg", headers=headers)
for i in queries:
client.send(destination=destination, body="scai.fhg.de/concept/" + str(i), headers=headers)
print u"# ErasmusMC ontology file".encode('utf-8')
print u"VR 0.0".encode('utf-8')
print u"ON CCG_Ontology".encode('utf-8')
print u"--".encode('utf-8')
@timeout(1)
def receiveFrames():
frame = clientConsumer.receiveFrame()
xml = ElementTree.fromstring(frame.body)
modelText = xml.find('docelem').find('model').text
class StompClient(object):
def __init__(self, config):
"""Init the Stompest wrapper client.
:type config: dict
:param config: The configuration for the STOM client.
I.e. {'host': 'tcp://127.0.0.1',
'queue': '/queue/test',
'transaction': True,
'username': '******',
'password': '******'}
The transaction attribute defines if messages should be published
in transactions.
"""
self.host = config['host']
self.queue = config['queue']
self.transactions_enabled = config['transaction']
self.transaction = None
auth_header = {}
if 'username' in config and 'password' in config:
auth_header.update(
{StompSpec.LOGIN_HEADER: config['username'],
StompSpec.PASSCODE_HEADER: config['password']})
self.client = Stomp(StompConfig(self.host))
try:
self.client.connect(headers=auth_header)
except (error.StompConnectTimeout, error.StompProtocolError) as e:
raise ClientErrors.ConnectionError(
"Could not connect to broker: %s" % e)
def close(self):
"""Close the connection."""
try:
self.client.disconnect()
except error.StompConnectionError as e:
raise ClientErrors.ConnectionError(
"Could not close connection: %s" % e)
def publish(self, messages, omit_transaction=False, durable=True):
"""Publish a message to the queue.
:type message: string or list of strings.
:param message: The message(s) to publish.
:type omit_transaction: bool
:param omit_transaction: Set to True if you would like to publish a
message outside a transaction.
:type durable: bool
:param durable: If this message should be durable.
"""
if omit_transaction:
header = None
elif self.transactions_enabled:
if not self.transaction:
self.transaction = str(uuid.uuid4())
try:
self.client.begin(transaction=self.transaction)
except error.StompProtocolError as e:
raise ClientErrors.ProtocolError(
"Could not start transaction: %s" % e)
header = {StompSpec.TRANSACTION_HEADER: self.transaction,
'durable': 'true' if durable else 'false'}
else:
header = None
if isinstance(messages, (basestring, dict, scim.Event)):
messages = [messages]
for msg in messages:
try:
if isinstance(msg, dict):
del msg['routing-key']
msg = json.dumps(msg)
elif isinstance(msg, scim.Event):
msg = json.dumps(msg.get_payload())
self.client.send(self.queue,
msg,
header)
except error.StompConnectionError as e:
raise ClientErrors.ConnectionError(
"Could not publish '%s' to broker: %s" % (msg, e))
def commit(self):
"""Commit the current transaction."""
if self.transaction:
self.client.commit(transaction=self.transaction)
self.transaction = None
def rollback(self):
"""Roll back (ABORT) the current transaction."""
if self.transaction:
self.client.abort(transaction=self.transaction)
self.transaction = None
class Client(object):
def __init__(self):
self.stompest = None
self.greenlet = None
self.subscriptions = {}
self._last_id = 0
def _next_id(self):
self._last_id += 1
return self._last_id
def connect(self):
if not self.stompest:
CONFIG, EXTRA = _get_config()
self._hostname = EXTRA.get('hostname', None)
self.stompest = Stomp(CONFIG)
if self.stompest.session.state != StompSession.DISCONNECTED:
return
while True:
try:
self.stompest.connect(host=self._hostname)
logger.info('Connected')
break
except StompConnectTimeout:
continue
if not self.greenlet:
self.greenlet = gevent.spawn(self._run)
def _run(self):
while True:
try:
frame = self.stompest.receiveFrame()
self.stompest.ack(frame)
if frame.command == 'ERROR':
logger.error(frame.info())
elif frame.command == 'MESSAGE':
token = self.stompest.message(frame)
if self.subscriptions.get(token):
subscription = self.subscriptions[token]
subscription.call(frame)
else:
logger.error(
"Received a message for %s (%s) but there was no matching subscription."
% (frame.headers.get(StompSpec.DESTINATION_HEADER,
'???'), token))
else:
logger.warning("Unknown frame: %s" % frame.info())
# @todo Handle receipts
except (gevent.GreenletExit, KeyboardInterrupt):
# @todo Include a receipt in the disconnect. And instead of breaking right away wait for the
# receipt frame before disconnecting and consider waiting on any greenlets we started.
self.stompest.disconnect()
break
except StompConnectionError:
# We've been disconnected from the server. Try reconnecting to it.
self.connect()
def on(self, destination, callback):
self.connect()
token = self.stompest.subscribe(
destination, {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
StompSpec.ID_HEADER: self._next_id(),
})
subscription = Subscription(conn=self,
destination=destination,
token=token,
callback=callback)
self.subscriptions[subscription.token] = subscription
# @todo consider adding optional support for additional custom headers
def send(self, cmd, destination):
self.connect()
body = json.dumps(cmd)
headers = {}
headers[
StompSpec.CONTENT_TYPE_HEADER] = 'application/json;charset=UTF-8'
self.stompest.send(destination, body, headers)
def join(self):
try:
self.connect()
except (gevent.GreenletExit, KeyboardInterrupt):
return
try:
gevent.joinall([self.greenlet])
except KeyboardInterrupt:
self.greenlet.kill(block=True)
#!/usr/bin/python
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, 'Hello'.encode())
client.send(QUEUE, 'YAYYYYYY'.encode())
client.disconnect()
class StompClient(BaseComponent):
""" Send and Receive messages from a STOMP queue """
channel = "stomp"
def init(self, host, port, username=None, password=None,
connect_timeout=3, connected_timeout=3,
version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"],
heartbeats=(0, 0), ssl_context=None,
use_ssl=True,
key_file=None,
cert_file=None,
ca_certs=None,
ssl_version=ssl.PROTOCOL_SSLv23,
key_file_password=None,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
channel=channel):
""" Initialize StompClient. Called after __init__ """
self.channel = channel
if proxy_host:
LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port)
else:
LOG.info("Connect to %s:%s", host, port)
if use_ssl and not ssl_context:
ssl_params = dict(key_file=key_file,
cert_file=cert_file,
ca_certs=ca_certs,
ssl_version=ssl_version,
password=key_file_password)
LOG.info("Request to use old-style socket wrapper: %s", ssl_params)
ssl_context = ssl_params
if use_ssl:
uri = "ssl://%s:%s" % (host, port)
else:
uri = "tcp://%s:%s" % (host, port)
# Configure failover options so it only tries to connect once
self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri
self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context,
version=version,
login=username,
passcode=password)
self._heartbeats = heartbeats
self._accept_versions = accept_versions
self._connect_timeout = connect_timeout
self._connected_timeout = connected_timeout
Stomp._transportFactory = EnhancedStompFrameTransport
Stomp._transportFactory.proxy_host = proxy_host
Stomp._transportFactory.proxy_port = proxy_port
Stomp._transportFactory.proxy_user = proxy_user
Stomp._transportFactory.proxy_password = proxy_password
self._client = Stomp(self._stomp_config)
self._subscribed = {}
self.server_heartbeat = None
self.client_heartbeat = None
self.ALLOWANCE = 2 # multiplier for heartbeat timeouts
@property
def connected(self):
if self._client.session:
return self._client.session.state == StompSession.CONNECTED
else:
return False
@property
def subscribed(self):
return self._subscribed.keys()
@property
def stomp_logger(self):
return LOG_CATEGORY
@handler("disconnect")
def _disconnect(self, receipt=None):
if self.connected:
self._client.disconnect(receipt=receipt)
self._client.close(flush=True)
self.fire(disconnected(reconnect=False))
self._subscribed = {}
return "disconnected"
def start_heartbeats(self):
LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat, self._client.serverHeartBeat)
if self._client.clientHeartBeat:
if self.client_heartbeat:
# Timer already exists, just reset it
self.client_heartbeat.reset()
else:
LOG.info("Client will send heartbeats to server")
# Send heartbeats at 80% of agreed rate
self.client_heartbeat = Timer((self._client.clientHeartBeat / 1000.0) * 0.8,
client_heartbeat(), persist=True)
self.client_heartbeat.register(self)
else:
LOG.info("No Client heartbeats will be sent")
if self._client.serverHeartBeat:
if self.server_heartbeat:
# Timer already exists, just reset it
self.server_heartbeat.reset()
else:
LOG.info("Requested heartbeats from server.")
# Allow a grace period on server heartbeats
self.server_heartbeat = Timer((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE,
server_heartbeat(), persist=True)
self.server_heartbeat.register(self)
else:
LOG.info("Expecting no heartbeats from Server")
@handler("connect")
def connect(self, event, host=None, *args, **kwargs):
""" connect to Stomp server """
LOG.info("Connect to Stomp...")
try:
self._client.connect(heartBeats=self._heartbeats,
host=host,
versions=self._accept_versions,
connectTimeout=self._connect_timeout,
connectedTimeout=self._connected_timeout)
LOG.info("State after Connection Attempt: %s", self._client.session.state)
if self.connected:
LOG.info("Connected to %s", self._stomp_server)
self.fire(connected())
self.start_heartbeats()
return "success"
except StompConnectionError:
LOG.debug(traceback.format_exc())
self.fire(connection_failed(self._stomp_server))
event.success = False
return "fail"
@handler("server_heartbeat")
def check_server_heartbeat(self, event):
""" Confirm that heartbeat from server hasn't timed out """
now = time.time()
last = self._client.lastReceived or 0
if last:
elapsed = now - last
else:
elapsed = -1
LOG.debug("Last received data %d seconds ago", elapsed)
if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE + last) < now:
LOG.error("Server heartbeat timeout. %d seconds since last heartbeat. Disconnecting.", elapsed)
event.success = False
self.fire(heartbeat_timeout())
if self.connected:
self._client.disconnect()
# TODO: Try to auto-reconnect?
@handler("client_heartbeat")
def send_heartbeat(self, event):
if self.connected:
LOG.debug("Sending heartbeat")
try:
self._client.beat()
except StompConnectionError:
event.success = False
self.fire(disconnected())
@handler("generate_events")
def generate_events(self, event):
if not self.connected:
return
try:
if self._client.canRead(1):
frame = self._client.receiveFrame()
LOG.debug("Recieved frame %s", frame)
self.fire(message(frame))
except StompConnectionError:
self.fire(disconnected())
@handler("send")
def send(self, event, destination, body, headers=None, receipt=None):
LOG.debug("send()")
if not self.connected:
LOG.error("Can't send when Stomp is disconnected")
self.fire(on_stomp_error(None, Exception("Message send attempted with stomp disconnected")))
event.success = False
return
try:
self._client.send(destination, body=body.encode('utf-8'), headers=headers, receipt=receipt)
LOG.debug("Message sent")
except StompConnectionError:
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(None, err))
@handler("subscribe")
def _subscribe(self, event, destination, ack=ACK_CLIENT_INDIVIDUAL):
if ack not in ACK_MODES:
raise ValueError("Invalid client ack mode specified")
LOG.info("Subscribe to message destination %s", destination)
try:
# Set ID to match destination name for easy reference later
frame, token = self._client.subscribe(destination,
headers={StompSpec.ACK_HEADER: ack,
'id': destination})
self._subscribed[destination] = token
except StompConnectionError:
event.success = False
self.fire(disconnected())
except StompError as err:
event.success = False
LOG.debug(traceback.format_exc())
self.fire(on_stomp_error(None, err))
@handler("unsubscribe")
def _unsubscribe(self, event, destination):
if destination not in self._subscribed:
LOG.error("Unsubscribe Request Ignored. Not subscribed to %s", destination)
return
try:
token = self._subscribed.pop(destination)
frame = self._client.unsubscribe(token)
LOG.debug("Unsubscribed: %s", frame)
except StompConnectionError:
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(frame, err))
@handler("message")
def on_message(self, event, headers, message):
LOG.info("Stomp message received")
@handler("ack")
def ack_frame(self, event, frame):
LOG.debug("ack_frame()")
try:
self._client.ack(frame)
LOG.debug("Ack Sent")
except StompConnectionError:
LOG.error("Error sending ack")
event.success = False
self.fire(disconnected())
except StompError as err:
LOG.error("Error sending ack")
event.success = False
self.fire(on_stomp_error(frame, err))
def get_subscription(self, frame):
""" Get subscription from frame """
LOG.info(self._subscribed)
_, token = self._client.message(frame)
return self._subscribed[token]
def send_stomp(message):
config = StompConfig(stomp_uri)
client = Stomp(config)
client.connect()
client.send(stomp_destination, message)
client.disconnect()
import json
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://*****:*****@mail.nih.gov','timeStamp':'2015-06-25','outputDir':'/home/user/python/testOutDir'}))
client.disconnect()
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:7777', version='1.2')
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(connectedTimeout=4)
ID = 'bingo'
client.send(
'/my/test/destination', # destination
'THIS-IS-A-SEND-BODY', # body
headers={'receipt': ID}) # headers
answer = client.receiveFrame()
receiptID = answer.headers["receipt-id"]
returnValue = 0
if receiptID != ID:
print "Receipt header wrong:" + receiptID
returnValue = 1
else:
print "Correct receipt id received: " + receiptID
client.disconnect()
sys.exit(returnValue)
class JMSClient(object):
"""Class JMSClient
"""
_mh = None
_client = None
_host = None
_port = None
_user = None
_passw = None
_verbose = None
_is_connected = None
def __init__(self, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
verbose (bool): verbose mode
"""
try:
self._mh = MasterHead.get_head()
self._verbose = verbose
if (self._verbose):
basicConfig()
getLogger().setLevel(DEBUG)
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
@property
def client(self):
""" STOMP client property getter """
return self._client
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def verbose(self):
""" verbose mode property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def connect(self, host, port=61613, user=None, passw=None, timeout=10):
"""Method connects to server
Args:
host (str): hostname
port (str): port
user (str): username
passw (str): password
timeout (int): timeout
Returns:
bool: result
Raises:
event: jms_before_connect
event: jms_after_connected
"""
try:
msg = 'host:{0}, port:{1}, user:{2}, passw:{3}, timeout:{4}'.format(
host, port, user, passw, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connecting', msg), self._mh.fromhere())
ev = event.Event(
'jms_before_connect', host, port, user, passw, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
timeout = ev.argv(4)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client = Stomp(StompConfig('tcp://{0}:{1}'.format(self._host, self._port),
login=self._user, passcode=self._passw))
self._client.connect(
connectTimeout=timeout, connectedTimeout=timeout)
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connected'), self._mh.fromhere())
ev = event.Event('jms_after_connect')
self._mh.fire_event(ev)
return True
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_disconnecting'), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return False
else:
self._client.disconnect()
self._client.close()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_disconnected'), self._mh.fromhere())
return True
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def send(self, destination_name, message, destination_type='queue', headers={}):
"""Method sends message
JMS headers - JMSCorrelationID, JMSExpiration, JMSDeliveryMode, JMSPriority,
JMSReplyTo, JMSType
Args:
destination_name (str): queue|topic name
message (str): message
destination_type (str): queue|topic
headers (dict): JMS headers, key - title, value - string
Returns:
bool: result
Raises:
event: jms_before_send
event: jms_after_send
"""
try:
msg = 'destination_name:{0}, message:{1}, destination_type:{2}, headers:{3}'.format(
destination_name, message, destination_type, headers)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_sending_msg', msg), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return False
ev = event.Event(
'jms_before_send', destination_name, message, destination_type, headers)
if (self._mh.fire_event(ev) > 0):
destination_name = ev.argv(0)
message = ev.argv(1)
destination_type = ev.argv(2)
headers = ev.argv(3)
if (ev.will_run_default()):
headers_new = {}
for key, value in headers.items():
if (key in mapping):
headers_new[mapping[key]] = value
self._client.send('/{0}/{1}'.format(destination_type, destination_name), message if (
version_info[0] == 2) else message.encode('utf-8'), headers_new)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_msg_sent'), self._mh.fromhere())
ev = event.Event('jms_after_send')
self._mh.fire_event(ev)
return True
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def receive(self, destination_name, cnt=1):
"""Method receives messages
Args:
destination_name (str): queue name
cnt (int): count of messages
Returns:
list: messages as dictionary {'message', JMS headers}
Raises:
event: jms_before_receive
event: jms_after_receive
"""
try:
msg = 'destination_name:{0}, count:{1}'.format(
destination_name, cnt)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_receiving_msg', msg), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return None
ev = event.Event('jms_before_receive', destination_name, cnt)
if (self._mh.fire_event(ev) > 0):
destination_name = ev.argv(0)
cnt = ev.argv(1)
if (ev.will_run_default()):
token = self._client.subscribe('/queue/{0}'.format(destination_name),
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
msgs = []
i = 0
while (i < cnt and self._client.canRead(1)):
frame = self._client.receiveFrame()
if (frame.command != 'MESSAGE'):
break
self._client.ack(frame)
msgs.append(frame)
i = i + 1
self._client.unsubscribe(token)
messages = []
for msg in msgs:
message = {}
message['message'] = msg.body.decode()
for header in msg.rawHeaders:
if (header[0] in mapping.values()):
message[
list(mapping.keys())[list(mapping.values()).index(header[0])]] = header[1]
messages.append(message)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_msg_received', len(messages)), self._mh.fromhere())
ev = event.Event('jms_after_receive')
self._mh.fire_event(ev)
return messages
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return None
def browse(self, destination_name, cnt=100, jms_correlation_id=None, jms_type=None):
"""Method browses queue
Args:
destination_name (str): queue name
cnt (int): count of messages
jms_correlation_id (str): requested JMSCorrelationID
jms_type (str): requested JMSType
Returns:
list: messages as dictionary {'message', JMS headers}
Raises:
event: jms_before_browse
event: jms_after_browse
"""
try:
msg = 'destination_name:{0}, count:{1}, jms_correlation_id:{2}, jms_type:{3}'.format(
destination_name, cnt, jms_correlation_id, jms_type)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_browsing', msg), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return None
ev = event.Event(
'jms_before_browse', destination_name, cnt, jms_correlation_id, jms_type)
if (self._mh.fire_event(ev) > 0):
destination_name = ev.argv(0)
cnt = ev.argv(1)
jms_correlation_id = ev.argv(2)
jms_type = ev.argv(3)
if (ev.will_run_default()):
token = self._client.subscribe('/queue/{0}'.format(destination_name),
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
msgs = []
i = 0
while (i < cnt and self._client.canRead(1)):
frame = self._client.receiveFrame()
correlation_id = None
type = None
for header in frame.rawHeaders:
if (header[0] == 'correlation-id'):
correlation_id = header[1]
elif (header[0] == 'type'):
type = header[1]
if ((jms_correlation_id == None or jms_correlation_id == correlation_id) and
(jms_type == None or jms_type == type)):
msgs.append(frame)
i = i + 1
self._client.unsubscribe(token)
messages = []
for msg in msgs:
message = {}
message['message'] = msg.body.decode()
for header in msg.rawHeaders:
if (header[0] in mapping.values()):
message[
list(mapping.keys())[list(mapping.values()).index(header[0])]] = header[1]
messages.append(message)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_msg_received', len(messages)), self._mh.fromhere())
ev = event.Event('jms_after_browse')
self._mh.fire_event(ev)
return messages
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return None
class ByteportStompClient(AbstractByteportClient):
DEFAULT_BROKER_HOST = 'stomp.byteport.se'
STORE_QUEUE_NAME = '/queue/simple_string_dev_message'
SUPPORTED_CHANNEL_TYPES = ['topic', 'queue']
client = None
def __init__(self, namespace, login, passcode, broker_host=DEFAULT_BROKER_HOST, device_uid=None, channel_type='topic'):
'''
Create a ByteportStompClient. This is a thin wrapper to the underlying STOMP-client that connets to the Byteport Broker
If a device_uid is given, a subscription will be made for Messages sent through Byteport.
The channel_type must be either 'topic' or 'queue'. Set top topic if unsure on what to use (use queue if you need to
use multiple consumers for a single device, this is not how most applications are set up).
:param namespace:
:param login: Broker username (Byteport web users are _not_ valid broker users). Ask [email protected] for access.
:param passcode: Broker passcode
:param broker_hosts: [optional] A list of brokers to connect to
:param device_uid: [optional] The device UID to subscribe for messages on
:param channel_type: [optional] Defaults to queue.
:param channel_key: [optional] Must match the configured key in the Byteport Device Manager
'''
self.namespace = str(namespace)
self.device_uid = device_uid
if channel_type not in self.SUPPORTED_CHANNEL_TYPES:
raise Exception("Unsupported channel type: %s" % channel_type)
broker_url = 'tcp://%s:61613' % broker_host
self.CONFIG = StompConfig(broker_url, version=StompSpec.VERSION_1_2)
self.client = Stomp(self.CONFIG)
try:
self.client.connect(headers={'login': login, 'passcode': passcode}, host='/')
logging.info("Connected to Stomp broker at %s using protocol version %s" % (broker_host, self.client.session.version))
# Set up a subscription on the correct queue if a Specific device UID was given
if self.device_uid:
subscribe_headers = dict()
subscribe_headers[StompSpec.ACK_HEADER] = StompSpec.ACK_CLIENT_INDIVIDUAL
subscribe_headers[StompSpec.ID_HEADER] = '0'
device_message_queue_name = '/%s/device_messages_%s.%s' % (channel_type, namespace, device_uid)
self.subscription_token = self.client.subscribe(device_message_queue_name, subscribe_headers)
logging.info("Subscribing to channel %s" % device_message_queue_name)
except StompProtocolError as e:
logging.error("Client socket connected, but probably failed to login. (ProtocolError)")
raise
except StompConnectionError:
logging.error("Failed to connect to Stomp Broker at %s" % broker_host)
raise
def disconnect(self):
if self.subscription_token:
try:
self.client.unsubscribe(self.subscription_token)
except Exception as e:
logging.error(u'Unsubscribe failed, reason %s' % e)
self.client.disconnect()
def __send_json_message(self, json):
self.client.send(self.STORE_QUEUE_NAME, json)
def __send_message(self, uid, data_string, timestamp=None):
if timestamp:
timestamp = self.auto_timestamp(timestamp)
else:
timestamp = int(time.time())
if not uid:
uid = self.device_uid
if not uid:
raise Exception("Can not send data without valid Device UID!")
message = dict()
message['uid'] = str(uid)
message['namespace'] = self.namespace
message['data'] = str(data_string)
message['timestamp'] = str(timestamp)
self.__send_json_message(json.dumps([message]))
def store(self, data=None, device_uid=None, timestamp=None):
if type(data) != dict:
raise ByteportClientException("Data must be of type dict")
for key in data.keys():
self.verify_field_name(key)
delimited_data = ';'.join("%s=%s" % (key, self.utf8_encode_value(val)) for (key, val) in data.iteritems())
self.__send_message(device_uid, delimited_data, timestamp)
class client_activemq():
def __init__(self, worker_threads=2):
self.log = logging.getLogger('client_activemq')
self.log.debug('Initializing client_activemq')
self.host = '127.0.0.1' #manually set here until there is a better place
self.port = 61613 #manually set here until there is a better place
self.worker_threads = worker_threads
self.client = None
self.connection_issue = True
self.continue_running = True
self.subscriptions = {}
self.outbound_msg_queue = []
self.outbound_msg_queue_lazy = []
self.threads = []
self.work_queue = Queue.Queue()
connection_handler = threading.Thread(target=self.connection_handler)
connection_handler.daemon = True
connection_handler.start()
send_event_hopeful_thread = threading.Thread(
target=self.send_event_hopeful_thread)
send_event_hopeful_thread.daemon = True
send_event_hopeful_thread.start()
send_event_lazy_thread = threading.Thread(
target=self.send_event_lazy_thread)
send_event_lazy_thread.daemon = True
send_event_lazy_thread.start()
publish_loop = threading.Thread(target=self.publish_loop)
publish_loop.daemon = True
publish_loop.start()
def init_connection(self):
if (self.client != None and self.client.session.state == 'connected'):
self.log.error('Re-init connection')
try:
self.client.disconnect()
except:
pass
config = StompConfig('tcp://%s:%s' % (self.host, self.port),
version=StompSpec.VERSION_1_1,
check=True)
self.client = Stomp(config)
self.client.connect(heartBeats=(30000, 0),
connectTimeout=1,
connectedTimeout=1)
def build_worker_pool(self, capacity):
for thread in self.threads:
if thread.is_alive() == False:
self.threads.remove(thread)
while len(self.threads) < capacity:
t = threading.Thread(target=self.worker, args=(self.work_queue, ))
t.daemon = True
t.start()
self.threads.append(t)
def connection_handler(self):
#This func will just try to reconnect repeatedly in a thread if we're flagged as having an issue.
while (True):
if (self.connection_issue == True):
try:
self.init_connection()
self.connection_issue = False
time.sleep(0.1)
for subscription in self.subscriptions:
self.log.info('Re-subscribing upon reconnection: %s' %
subscription)
self.subscribe(
subscription,
self.subscriptions[subscription]['callback_class'],
self.subscriptions[subscription]['callback'], True,
self.subscriptions[subscription]['selector'])
except Exception as e:
self.log.critical('Except: %s' % e)
else:
try:
self.client.beat()
except:
self.log.warning('Failed to heartbeat?')
self.build_worker_pool(self.worker_threads)
time.sleep(10)
def subscribe(self,
queue,
callback_class,
callback,
resub=False,
selector=None):
#This needs to exist so we can keep track of what subs we have and re-sub on reconnect
if queue in self.subscriptions and not resub:
self.log.info('Ignoring existing subscription %s' % queue)
return True #we're already subscribed
this_uuid = '%s' % uuid.uuid4()
if (self.connection_issue == True):
self.log.info(
'Cannot process subscription request as were not properly connected'
)
self.subscriptions[queue] = {
'uuid': this_uuid,
'callback': callback,
'callback_class': callback_class,
'selector': selector
}
return None
self.log.info('subscribe(%s %s %s %s)' %
(queue, callback_class, callback, resub))
try:
self.subscriptions[queue] = {
'uuid': this_uuid,
'callback': callback,
'callback_class': callback_class,
'selector': selector
}
if selector != None:
self.client.subscribe(
queue, {
StompSpec.ID_HEADER: this_uuid,
StompSpec.ACK_HEADER: StompSpec.ACK_AUTO,
'selector': selector
})
else:
self.client.subscribe(
queue, {
StompSpec.ID_HEADER: this_uuid,
StompSpec.ACK_HEADER: StompSpec.ACK_AUTO,
})
except Exception as e:
self.log.fatal('%s' % e)
self.connection_issue = True
def unsubscribe(self, queue):
if queue not in self.subscriptions:
return False #cant unsubscribe, we're not subscribed
if (self.connection_issue == True):
del self.subscriptions[queue]
return None
try:
self.client.unsubscribe(self.subscriptions[queue]['uuid'])
del self.subscriptions[queue]
except Exception as e:
self.log.error('%s' % e)
self.log.error('%s' % traceback.format_exc())
sys.exc_clear()
self.connection_issue = True
def send_event_lazy(self, destination, body, headers={}, persistent=False):
headers['time_queued'] = time.time()
self.outbound_msg_queue_lazy.append({
'destination': destination,
'body': body,
'headers': headers,
'persistent': persistent
})
def send_event_lazy_thread(self):
while self.continue_running:
time.sleep(0.001)
#If it gets there, then great, if not, well we tried!
if (self.connection_issue == True):
continue
while len(self.outbound_msg_queue_lazy
) > 0 and self.connection_issue == False:
try:
item = self.outbound_msg_queue_lazy.pop(0)
if item['persistent'] == True:
item['headers']['persistent'] = 'true'
else:
item['headers']['persistent'] = 'false'
item['headers']['time_sent'] = time.time()
self.client.send(item['destination'],
json.dumps(item['body']), item['headers'])
except Exception as e:
self.log.critical('Except: %s' % e)
self.outbound_msg_queue_lazy.insert(0, item)
self.connection_issue = True
def send_event_hopeful(self, destination, body, persist):
self.outbound_msg_queue.append({
'destination': destination,
'body': body
})
def send_event_hopeful_thread(self):
while self.continue_running:
time.sleep(0.001)
if (self.connection_issue == True):
continue
while len(self.outbound_msg_queue
) > 0 and self.connection_issue == False:
try:
item = self.outbound_msg_queue.pop(0)
self.client.send(item['destination'],
json.dumps(item['body']),
{'persistent': 'false'})
except Exception as e:
self.log.critical('Except: %s' % e)
self.outbound_msg_queue.insert(0, item)
self.connection_issue = True
def worker(self, queue):
while self.continue_running:
try:
item = queue.get()
for x in 1, 2, 3:
try:
item['callback'](item['callback_class'], item['data'],
item['headers'])
break
except Exception as e:
self.log.error('Exception in worker thread: %s' % e)
traceback.print_exc()
sys.exc_clear()
time.sleep(0.01)
queue.task_done()
except:
pass
def publish_loop(self):
self.log.debug('publish_loop() init')
time.sleep(0.2)
while self.continue_running:
if self.connection_issue == False:
try:
if not self.client.canRead(0.01):
continue
try:
frame = self.client.receiveFrame()
data = json.loads(frame.body)
queue = frame.headers['destination']
try:
time_sent = float(frame.headers['time_sent'])
time_queued = float(frame.headers['time_queued'])
send_latency = (time.time() - time_sent)
queue_latency = (time_sent - time_queued)
if queue_latency > 0.1:
print 'Queue Latency: %s' % (queue_latency)
if send_latency > 0.1:
print 'Send Latency: %s' % (send_latency)
except:
pass
self.work_queue.put({
'callback':
self.subscriptions[queue]['callback'],
'callback_class':
self.subscriptions[queue]['callback_class'],
'data':
data,
'headers':
frame.headers,
})
#self.client.ack(frame)
except Exception as e:
raise
self.log.critical('Except: %s' % e)
#self.client.nack(frame)
except Exception as e:
self.log.fatal('except: %s' % e)
self.connection_issue = True
#read the csv file values on column 5
#the csv file is created by running create_csv.py
fields = ['id', 'num5']
df = pd.read_csv('tests.csv', skipinitialspace=True, usecols=fields)
x = df.name.values
q = str(x)
#check to see if values C or G exist on column 5
#if yes create test db and store row values
if 'C' or 'G' in q:
conn = sqlite3.connect("test.sqlite")
conn.execute("CREATE TABLE if not exists Data (id TEXT, col5 TEXT)")
df = pd.read_csv("tests.csv")
df.to_sql("Data", conn, if_exists='append', index=False)
#check to see if only values C and G exist on column 5
#if yes create connect to apacheMq and store values
elif 'C' and 'G'in q:
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
#client.send(QUEUE, json.dumps{(q)}.encode())
client.send(QUEUE, q.encode())
#client.send(QUEUE, 'test 2'.encode())
client.disconnect()
else:
print ('No matches found')
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, 'test message 1'.encode())
client.send(QUEUE, 'test message 2'.encode())
client.disconnect()
class StompClient(BaseComponent):
channel = "stomp"
def init(
self,
host,
port,
username=None,
password=None,
connect_timeout=3,
connected_timeout=3,
version=StompSpec.VERSION_1_2,
accept_versions=["1.0", "1.1", "1.2"],
heartbeats=(0, 0),
ssl_context=None,
use_ssl=True,
key_file=None,
cert_file=None,
ca_certs=None,
ssl_version=ssl.PROTOCOL_SSLv23, # means 'latest available'
key_file_password=None,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
channel=channel,
stomp_params=None):
""" Initialize StompClient. Called after __init__ """
self.channel = channel
if proxy_host:
LOG.info("Connect to %s:%s through proxy %s:%d", host, port,
proxy_host, proxy_port)
else:
LOG.info("Connect to %s:%s", host, port)
if use_ssl and not ssl_context:
ssl_params = dict(key_file=key_file,
cert_file=cert_file,
ca_certs=ca_certs,
ssl_version=ssl_version,
password=key_file_password)
LOG.info("Request to use old-style socket wrapper: %s", ssl_params)
ssl_context = ssl_params
if use_ssl:
uri = "ssl://%s:%s" % (host, port)
else:
uri = "tcp://%s:%s" % (host, port)
# Configure failover options so it only tries based on settings
# build any parameters passed
# every connection has at least these two: maxReconnectAttempts, startupMaxReconnectAttempts
items = [item.split("=", 2)
for item in stomp_params.split(",")] if stomp_params else None
connection_params = {
item[0].strip(): item[1].strip()
for item in items
} if items else {}
if "maxReconnectAttempts" not in connection_params:
connection_params[
'maxReconnectAttempts'] = DEFAULT_MAX_RECONNECT_ATTEMPTS
if "startupMaxReconnectAttempts" not in connection_params:
connection_params[
'startupMaxReconnectAttempts'] = DEFAULT_STARTUP_MAX_RECONNECT_ATTEMPTS
self._stomp_server = "failover:({0})?{1}".format(
uri, ",".join(
["{}={}".format(k, v) for k, v in connection_params.items()]))
LOG.debug("Stomp uri: {}".format(self._stomp_server))
self._stomp_config = StompConfig(uri=self._stomp_server,
sslContext=ssl_context,
version=version,
login=username,
passcode=password)
self._heartbeats = heartbeats
self._accept_versions = accept_versions
self._connect_timeout = connect_timeout
self._connected_timeout = connected_timeout
Stomp._transportFactory = EnhancedStompFrameTransport
Stomp._transportFactory.proxy_host = proxy_host
Stomp._transportFactory.proxy_port = proxy_port
Stomp._transportFactory.proxy_user = proxy_user
Stomp._transportFactory.proxy_password = proxy_password
self._client = Stomp(self._stomp_config)
self._subscribed = {}
self.server_heartbeat = None
self.client_heartbeat = None
self.last_heartbeat = 0
self.ALLOWANCE = 2 # multiplier for heartbeat timeouts
@property
def connected(self):
if self._client.session:
return self._client.session.state == StompSession.CONNECTED
else:
return False
@property
def socket_connected(self):
try:
if self._client._transport:
return True
except:
pass
return False
@property
def subscribed(self):
return self._subscribed.keys()
@property
def stomp_logger(self):
return LOG_CATEGORY
@handler("Disconnect")
def _disconnect(self, receipt=None, flush=True, reconnect=False):
try:
if flush:
self._subscribed = {}
if self.connected:
self._client.disconnect(receipt=receipt)
except Exception as e:
LOG.error("Failed to disconnect client")
try:
self.fire(Disconnected(reconnect=reconnect))
self._client.close(flush=flush)
except Exception as e:
LOG.error("Failed to close client connection")
return "disconnected"
def start_heartbeats(self):
LOG.info("Client HB: %s Server HB: %s", self._client.clientHeartBeat,
self._client.serverHeartBeat)
if self._client.clientHeartBeat:
if self.client_heartbeat:
# Timer already exists, just reset it
self.client_heartbeat.reset()
else:
LOG.info("Client will send heartbeats to server")
# Send heartbeats at 80% of agreed rate
self.client_heartbeat = Timer(
(self._client.clientHeartBeat / 1000.0) * 0.8,
ClientHeartbeat(),
persist=True)
self.client_heartbeat.register(self)
else:
LOG.info("No Client heartbeats will be sent")
if self._client.serverHeartBeat:
if self.server_heartbeat:
# Timer already exists, just reset it
self.server_heartbeat.reset()
else:
LOG.info("Requested heartbeats from server.")
# Allow a grace period on server heartbeats
self.server_heartbeat = Timer(
(self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE,
ServerHeartbeat(),
persist=True)
self.server_heartbeat.register(self)
else:
LOG.info("Expecting no heartbeats from Server")
@handler("Connect")
def connect(self, event, host=None, *args, **kwargs):
""" connect to Stomp server """
LOG.info("Connect to Stomp...")
try:
self._client.connect(heartBeats=self._heartbeats,
host=host,
versions=self._accept_versions,
connectTimeout=self._connect_timeout,
connectedTimeout=self._connected_timeout)
LOG.debug("State after Connection Attempt: %s",
self._client.session.state)
if self.connected:
LOG.info("Connected to %s", self._stomp_server)
self.fire(Connected())
self.start_heartbeats()
return "success"
except StompConnectionError as err:
LOG.debug(traceback.format_exc())
self.fire(ConnectionFailed(self._stomp_server))
event.success = False
# This logic is added to trap the situation where resilient-circuits does not reconnect from a loss of connection
# with the resilient server. In these cases, this error is not survivable and it's best to kill resilient-circuits.
# If resilient-circuits is running as a service, it will restart and state would clear for a new stomp connection.
except StompProtocolError as err:
LOG.error(traceback.format_exc())
LOG.error("Exiting due to unrecoverable error")
sys.exit(1) # this will exit resilient-circuits
return "fail"
@handler("ServerHeartbeat")
def check_server_heartbeat(self, event):
""" Confirm that heartbeat from server hasn't timed out """
now = time.time()
self.last_heartbeat = self._client.lastReceived or self.last_heartbeat
if self.last_heartbeat:
elapsed = now - self.last_heartbeat
else:
elapsed = -1
if ((self._client.serverHeartBeat / 1000.0) * self.ALLOWANCE +
self.last_heartbeat) < now:
LOG.error(
"Server heartbeat timeout. %d seconds since last heartbeat.",
elapsed)
event.success = False
self.fire(HeartbeatTimeout())
@handler("ClientHeartbeat")
def send_heartbeat(self, event):
if self.connected:
LOG.debug("Sending heartbeat")
try:
self._client.beat()
except (StompConnectionError, StompError) as err:
event.success = False
self.fire(OnStompError(None, err))
@handler("generate_events")
def generate_events(self, event):
event.reduce_time_left(0.1)
if not self.connected:
return
try:
if self._client.canRead(0):
frame = self._client.receiveFrame()
LOG.debug("Recieved frame %s", frame)
if frame.command == StompSpec.ERROR:
self.fire(OnStompError(frame, None))
else:
self.fire(Message(frame))
except (StompConnectionError, StompError) as err:
LOG.error("Failed attempt to generate events.")
self.fire(OnStompError(None, err))
@handler("Send")
def send(self, event, destination, body, headers=None, receipt=None):
LOG.debug("send()")
try:
self._client.send(destination,
body=body.encode('utf-8'),
headers=headers,
receipt=receipt)
LOG.debug("Message sent")
except (StompConnectionError, StompError) as err:
LOG.error("Error sending frame")
event.success = False
self.fire(OnStompError(None, err))
raise # To fire Send_failure event
@handler("Subscribe")
def _subscribe(self,
event,
destination,
additional_headers=None,
ack=ACK_CLIENT_INDIVIDUAL):
if ack not in ACK_MODES:
raise ValueError("Invalid client ack mode specified")
if destination in self._client.session._subscriptions:
LOG.debug("Ignoring subscribe request to %s. Already subscribed.",
destination)
LOG.info("Subscribe to message destination %s", destination)
try:
headers = {StompSpec.ACK_HEADER: ack, 'id': destination}
if additional_headers:
headers.update(additional_headers)
# Set ID to match destination name for easy reference later
frame, token = self._client.subscribe(destination, headers)
self._subscribed[destination] = token
except (StompConnectionError, StompError) as err:
LOG.error("Failed to subscribe to queue.")
event.success = False
LOG.debug(traceback.format_exc())
self.fire(OnStompError(None, err))
# This logic is added to trap the situation where resilient-circuits does not reconnect from a loss of connection
# with the resilient server. In these cases, this error is not survivable and it's best to kill resilient-circuits.
# If resilient-circuits is running as a service, it will restart and state would clear for a new stomp connection.
except StompProtocolError as err:
LOG.error(traceback.format_exc())
LOG.error("Exiting due to unrecoverable error")
sys.exit(1) # this will exit resilient-circuits
@handler("Unsubscribe")
def _unsubscribe(self, event, destination):
if destination not in self._subscribed:
LOG.error("Unsubscribe Request Ignored. Not subscribed to %s",
destination)
return
try:
token = self._subscribed.pop(destination)
frame = self._client.unsubscribe(token)
LOG.debug("Unsubscribed: %s", frame)
except (StompConnectionError, StompError) as err:
event.success = False
LOG.error("Unsubscribe Failed.")
self.fire(OnStompError(frame, err))
@handler("Message")
def on_message(self, event, headers, message, queue):
LOG.debug("Stomp message received")
@handler("Ack")
def ack_frame(self, event, frame):
LOG.debug("ack_frame()")
try:
self._client.ack(frame)
LOG.debug("Ack Sent")
except (StompConnectionError, StompError) as err:
LOG.error("Error sending ack")
event.success = False
self.fire(OnStompError(frame, err))
raise # To fire Ack_failure event
def get_subscription(self, frame):
""" Get subscription from frame """
_, token = self._client.message(frame)
return self._subscribed[token]
'''
@author: Saravana Perumal Shanmugam
'''
from stompest.config import StompConfig
from stompest.sync import Stomp
from datetime import date
import time
import logging
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = 'jms.queue.prodcons'
if __name__ == '__main__':
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
client = Stomp(CONFIG)
client.connect()
for index in range(10):
client.send(QUEUE, 'A Message ' + str(index) + ' At ' + str(date.today()))
time.sleep(1)
client.disconnect()
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:7777', version='1.2')
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(connectedTimeout=4)
ID = 'bingo'
client.send('/my/test/destination', # destination
'THIS-IS-A-SEND-BODY', # body
headers={'receipt': ID}) # headers
answer = client.receiveFrame()
receiptID = answer.headers["receipt-id"]
returnValue = 0
if receiptID != ID:
print "Receipt header wrong:" + receiptID
returnValue = 1
else:
print "Correct receipt id received: " + receiptID
client.disconnect()
sys.exit(returnValue)
port = int(os.getenv('ACTIVEMQ_PORT') or 61613)
destination = sys.argv[1:2] or ['/topic/event']
destination = destination[0]
messages = 10000
data = 'Hello World from Python'
config = StompConfig('tcp://%s:%d' % (host, port),
login=user,
passcode=password,
version='1.1')
client = Stomp(config)
client.connect(host='mybroker')
count = 0
start = time.time()
for _ in xrange(messages):
send_data = "".join((data, " ", str(count)))
client.send(destination=destination,
body=send_data,
headers={'persistent': 'false'})
# print("Send msg [ %s ] to destination [ %s ]" % (send_data, destination))
count += 1
diff = time.time() - start
print 'Sent %s frames in %f seconds' % (count, diff)
client.disconnect(receipt='bye')
client.receiveFrame()
client.close()
frame = client.receiveFrame()
try:
payload = json.loads(frame.body)
except json.JSONDecoder:
print("Failed to decode message body: frame.body was {}".format(
frame.body))
with open('/opt/apache-activemq/sendfail/malformed', 'a+') as mf:
mf.write('{0}\n{1}'.format(frame.headers, frame.body))
mf.close()
continue
headers = frame.headers
print('Sending the following message to {} : '.format(to_queue))
print('body: {}'.format(payload))
print('headers: {}'.format(frame.headers))
try:
client.ack(frame)
except stompest.error.StompConnectionError:
print('Unable to send ACK to {}'.format(broker))
try:
headers['persistent'] = 'true'
client.send(to_queue, body=json.dumps(payload), headers=headers)
except stompest.error.StompError:
print(
'Sending message to {} failed. Writing message locally at /opt/apache-activemq/sendfail'
.format(broker))
with open('/opt/apache-activemq/sendfail/{}\n'.format(from_queue),
'a+') as sf:
sf.write(json.dumps(payload))
client.disconnect()
import json
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://*****:*****@mail.nih.gov',
'timeStamp': '2015-06-25',
'outputDir': '/home/user/python/testOutDir'
}))
client.disconnect()
# -*- coding:utf-8 -*-
import logging
from stompest.config import StompConfig
from stompest.sync import Stomp
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
uri = 'failover:(tcp://x:61613,tcp://y:61613,tcp://z:61613)?randomize=false,startupMaxReconnectAttempts=3,initialReconnectDelay=7,maxReconnectDelay=8,maxReconnectAttempts=0'
CONFIG = StompConfig(uri)
QUEUE = '/queue/liuyang-test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, 'test message 1')
client.send(QUEUE, 'test message 2')
client.disconnect()
client = Stomp(config)
client.connect(host=host)
body = """
<corpus>
<docelems>
<docelem>
<uiid>scai.fhg.de/voc/ccg</uiid>
<model>NS Voc\nID CCG\nNA Cologne Center for Genomics</model>
</docelem>
<docelem>
<uiid>scai.fhg.de/semtype/1000</uiid>
<model>NS SemType\nID 1000\nNA chemical compound\nDF A chemical compound (or just compound if used in the context of chemistry) is an entity consisting of two or more different atoms which associate via chemical bonds.</model>
</docelem>
<docelem>
<uiid>scai.fhg.de/concept/1</uiid>
<model>ID 1\nNA Ceritinib\nTM Ceritinib\nTM LDK378\nTM ZYKADIA\t@match=ci\nTM 1032900-25-6\nVO CCG</model>
</docelem>
</docelems>
</corpus>
"""
client.send(destination=destination, body=body, headers={'transformation': 'TEXT', 'event': 'FoundCorpus'})
print "Updating Ontology"
client.disconnect(receipt='bye')
client.receiveFrame()
client.close()
sys.exit(0)
def enqueue(queue_url, queue_name, data):
""" Sends a message to a queue """
client = Stomp(StompConfig(queue_url))
client.connect()
client.send(queue_name, data)
client.disconnect()
from stompest.config import StompConfig
from stompest.sync import Stomp
user = os.getenv('APOLLO_USER') or 'admin'
password = os.getenv('APOLLO_PASSWORD') or 'password'
host = os.getenv('APOLLO_HOST') or 'localhost'
port = int(os.getenv('APOLLO_PORT') or 61613)
destination = sys.argv[1:2] or ['/topic/event']
destination = destination[0]
messages = 10000
data = 'Hello World from Python'
config = StompConfig('tcp://%s:%d' % (host, port), login=user, passcode=password, version='1.1')
client = Stomp(config)
client.connect(host='mybroker')
count = 0
start = time.time()
for _ in xrange(messages):
client.send(destination=destination, body=data, headers={'persistent': 'false'})
count += 1
diff = time.time() - start
print 'Sent %s frames in %f seconds' % (count, diff)
client.disconnect(receipt='bye')
client.receiveFrame()
client.close()
class PostProcessAdmin:
def __init__(self, data, conf):
logging.debug("json data: %s [%s]" % (str(data), type(data)))
if not type(data) == dict:
raise ValueError, "PostProcessAdmin expects a data dictionary"
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
# List of error messages to be handled as information
self.exceptions = self.conf.exceptions
stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user, self.conf.amq_pwd)
self.client = Stomp(stompConfig)
self.data_file = None
self.facility = None
self.instrument = None
self.proposal = None
self.run_number = None
def _process_data(self, data):
"""
Retrieve run information from the data dictionary
provided with an incoming message.
@param data: data dictionary
"""
if data.has_key('data_file'):
self.data_file = str(data['data_file'])
if os.access(self.data_file, os.R_OK) == False:
raise ValueError("Data file does not exist or is not readable: %s" % self.data_file)
else:
raise ValueError("data_file is missing: %s" % self.data_file)
if data.has_key('facility'):
self.facility = str(data['facility']).upper()
else:
raise ValueError("Facility is missing")
if data.has_key('instrument'):
self.instrument = str(data['instrument']).upper()
else:
raise ValueError("Instrument is missing")
if data.has_key('ipts'):
self.proposal = str(data['ipts']).upper()
else:
raise ValueError("IPTS is missing")
if data.has_key('run_number'):
self.run_number = str(data['run_number'])
else:
raise ValueError("Run number is missing")
def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " + summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data))
return
# Run the reduction
out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
# Determine error condition
success, status_data = job_handling.determine_success_local(self.conf, out_err)
self.data.update(status_data)
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data))
else:
self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
def catalog_raw(self):
"""
Catalog a nexus file containing raw data
"""
self._process_data(self.data)
try:
from ingest_nexus import IngestNexus
self.send('/queue/' + self.conf.catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
ingestNexus = IngestNexus(self.data_file)
ingestNexus.execute()
ingestNexus.logout()
self.send('/queue/' + self.conf.catalog_complete, json.dumps(self.data))
except:
logging.error("catalog_raw: %s" % sys.exc_value)
self.data["error"] = "Catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.catalog_error, json.dumps(self.data))
def catalog_reduced(self):
"""
Catalog reduced data files for a given run
"""
self._process_data(self.data)
try:
from ingest_reduced import IngestReduced
self.send('/queue/' + self.conf.reduction_catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
# Send image to the web monitor
if len(self.conf.web_monitor_url.strip()) > 0:
monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd}
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
url_template = string.Template(self.conf.web_monitor_url)
url = url_template.substitute(instrument=self.instrument, run_number=self.run_number)
pattern = self.instrument + "_" + self.run_number + "*"
for dirpath, dirnames, filenames in os.walk(proposal_shared_dir):
listing = glob.glob(os.path.join(dirpath, pattern))
for filepath in listing:
f, e = os.path.splitext(filepath)
if e.startswith(os.extsep):
e = e[len(os.extsep):]
if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat") or filepath.endswith("plot_data.json"):
files = {'file': open(filepath, 'rb')}
# Post the file if it's small enough
if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size:
request = requests.post(url, data=monitor_user, files=files, verify=False)
logging.info("Submitted %s [status: %s]" % (filepath,
request.status_code))
ingestReduced = IngestReduced(self.facility, self.instrument, self.proposal, self.run_number)
ingestReduced.execute()
ingestReduced.logout()
self.send('/queue/' + self.conf.reduction_catalog_complete , json.dumps(self.data))
except:
logging.error("catalog_reduced: %s" % sys.exc_value)
self.data["error"] = "Reduction catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.reduction_catalog_error , json.dumps(self.data))
def create_reduction_script(self):
"""
Create a new reduction script from a template
"""
try:
import reduction_script_writer
writer = reduction_script_writer.ScriptWriter(self.data["instrument"])
writer.process_request(self.data,
configuration=self.conf,
send_function=self.send)
except:
logging.error("create_reduction_script: %s" % sys.exc_value)
def send(self, destination, data):
"""
Send an AMQ message
@param destination: AMQ queue to send to
@param data: payload of the message
"""
logging.info("%s: %s" % (destination, data))
self.client.connect()
self.client.send(destination, data)
self.client.disconnect()
client.subscribe(from_queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
while client.canRead(timeout=tmout):
frame = client.receiveFrame()
try:
payload = json.loads(frame.body)
except json.JSONDecoder:
print("Failed to decode message body: frame.body was {}".format(frame.body))
with open('/opt/apache-activemq/sendfail/malformed', 'a+') as mf:
mf.write('{0}\n{1}'.format(frame.headers, frame.body))
mf.close()
continue
headers = frame.headers
print('Sending the following message to {} : '.format(to_queue))
print('body: {}'.format(payload))
print('headers: {}'.format(frame.headers))
try:
client.ack(frame)
except stompest.error.StompConnectionError:
print('Unable to send ACK to {}'.format(broker))
try:
headers['persistent'] = 'true'
client.send(to_queue, body=json.dumps(payload), headers=headers)
except stompest.error.StompError:
print('Sending message to {} failed. Writing message locally at /opt/apache-activemq/sendfail'.format(broker))
with open('/opt/apache-activemq/sendfail/{}\n'.format(from_queue), 'a+') as sf:
sf.write(json.dumps(payload))
client.disconnect()
from stompest.config import StompConfig
from stompest.sync import Stomp
client = Stomp(StompConfig('tcp://activemq:61613'))
client.connect()
client.send('/queue/test', 'request sent')
client.disconnect()
pp.create_reduction_script()
# Check for registered processors
if type(configuration.processors) == list:
for p in configuration.processors:
toks = p.split('.')
if len(toks) == 2:
processor_module = __import__("postprocessing.processors.%s" % toks[0], globals(), locals(), [toks[1], ], -1)
try:
processor_class = eval("processor_module.%s" % toks[1])
if namespace.queue == processor_class.get_input_queue_name():
# Instantiate and call the processor
proc = processor_class(data, configuration, send_function=pp.send)
proc()
except:
logging.error("PostProcessAdmin: Processor error: %s" % sys.exc_value)
else:
logging.error("PostProcessAdmin: Processors can only be specified in the format module.Processor_class")
except:
# If we have a proper data dictionary, send it back with an error message
if type(data) == dict:
data["error"] = str(sys.exc_value)
stomp = Stomp(StompConfig(configuration.failover_uri, configuration.amq_user, configuration.amq_pwd))
stomp.connect()
stomp.send(configuration.postprocess_error, json.dumps(data))
stomp.disconnect()
raise
except:
logging.error("PostProcessAdmin: %s" % sys.exc_value)
# 'transformation': 'TEXT',
# 'event': 'QueryDocelem',
# 'reply-to': '/topic/docstore-reply'
# }
body = """<query>
<annot>scai.fraunhofer.de/prominer-entry/hs00001</annot>
<annot>scai.fraunhofer.de/prominer-entry/hs00002</annot>
</query>"""
headers = {
'transformation': 'TEXT',
'event': 'QueryAnnotationIndex',
'reply-to': '/topic/docstore-reply'
}
print "Sending to " + destination
client.send(destination=destination, body=body, headers=headers)
clientConsumer.subscribe(destination='/topic/docstore-reply', headers={'id': 'required-for-STOMP-1.1'})
while True:
startReceive = time.time()
frame = clientConsumer.receiveFrame()
diffReceive = time.time() - startReceive
print frame
print diffReceive
time.sleep(1)
print "Sending to " + destination
client.send(destination=destination, body=body, headers=headers)
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1)
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(heartBeats=(0, 10000))
client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
client.send(QUEUE, 'test message 1')
client.send(QUEUE, 'test message 2')
while True:
frame = client.receiveFrame()
print 'Got %s' % frame.info()
client.ack(frame)
client.disconnect()
import ssl
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
context = ssl.create_default_context()
# Disable cert validation for demo only
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
CONFIG = StompConfig(uri='ssl://x.x.x.x:61614',
login='******',
passcode='password',
check=False,
sslContext=context)
QUEUE = '/queue/my-queue'
RQUEUE = '/queue/my-reply'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.subscribe(QUEUE,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
frame = client.receiveFrame()
print('Got %s' % frame.info())
client.ack(frame)
client.send(RQUEUE, b'hello', headers={'correlation-id': 0})
client.disconnect()
