def main():
parser = argparse.ArgumentParser()
parser.add_argument("-d", nargs=1, required=True, help="destination", metavar="dest")
parser.add_argument("-e", action="store_true", default=False, help="persistent msgs flag")
parser.add_argument("-m", nargs=1, required=True, help="msg", metavar="msg")
parser.add_argument("-n", nargs=1, default=False, help="number of msgs", metavar="int")
parser.add_argument("-p", nargs=1, default=False, help="msg prefix", metavar="msg prefix")
parser.add_argument("-s", nargs=1, required=True, help="broker", metavar="broker")
parser.add_argument("-t", nargs=1, default=False, help="send msg every sec", metavar="float")
args = parser.parse_args()
broker = "tcp://%s:6163" % (args.s[0])
config = StompConfig(broker)
client = Stomp(config)
client.connect()
rd = "".join(random.sample("abcdefghijklmno", 2))
try:
i = 0
if args.n:
while i < int(args.n[0]):
send_msg(args, client, i, rd)
i += 1
else:
while True:
send_msg(args, client, i, rd)
i += 1
except KeyboardInterrupt:
client.disconnect()
raise SystemExit(1)
def __init__(self,
namespace,
login,
passcode,
broker_hosts=DEFAULT_BROKER_HOSTS):
self.namespace = str(namespace)
for broker_host in broker_hosts:
broker_url = 'tcp://%s:61613' % broker_host
self.CONFIG = StompConfig(broker_url,
version=StompSpec.VERSION_1_2)
self.client = Stomp(self.CONFIG)
try:
# Convention: set vhost to the namespace. This will require a message-boss consuming on this vhost!!!
vhost = namespace
self.client.connect(headers={
'login': login,
'passcode': passcode
},
host=vhost)
print "Connected to %s using protocol version %s" % (
broker_host, self.client.session.version)
except StompConnectionError:
pass
def __init__(self, config):
"""Init the Stompest wrapper client.
:type config: dict
:param config: The configuration for the STOM client.
I.e. {'host': 'tcp://127.0.0.1',
'queue': '/queue/test',
'transaction': True,
'username': '******',
'password': '******'}
The transaction attribute defines if messages should be published
in transactions.
"""
self.host = config['host']
self.queue = config['queue']
self.transactions_enabled = config['transaction']
self.transaction = None
auth_header = {}
if 'username' in config and 'password' in config:
auth_header.update(
{StompSpec.LOGIN_HEADER: config['username'],
StompSpec.PASSCODE_HEADER: config['password']})
self.client = Stomp(StompConfig(self.host))
try:
self.client.connect(headers=auth_header)
except (error.StompConnectTimeout, error.StompProtocolError) as e:
raise ClientErrors.ConnectionError(
"Could not connect to broker: %s" % e)
def call_route(request_queue, response_queue, request):
"""
"""
config = {
"stomp": {
"server": '192.168.0.3',
"port": 61613,
"timeout": 15,
}
}
stomp_config = StompConfig("tcp://%s:%d" % (config['stomp']['server'], config['stomp']['port']), version=StompSpec.VERSION_1_0)
stomp = Stomp(stomp_config)
stomp.connect()
jms_id = str(uuid4())
token = stomp.subscribe(response_queue, {'JMSCorrelationID': jms_id})
stomp.send(request_queue, json.dumps(request), {'JMSCorrelationID': jms_id})
response = None
if stomp.canRead(config['stomp']['timeout']):
response = stomp.receiveFrame()
stomp.unsubscribe(token)
return response
class StompHandler(logging.Handler):
'''
Send messages to a remote queue broker using the STOMP protocol.
This module is named and configured separately from
standard logging for clarity about its scope: while logging has an
informational purpose, this module has a functional one.
'''
def __init__(self, conf):
self.conf = conf
if self.conf['protocol'] == '11':
protocol_v = StompSpec.VERSION_1_1
elif self.conf['protocol'] == '12':
protocol_v = StompSpec.VERSION_1_2
else:
protocol_v = StompSpec.VERSION_1_0
client_config = StompConfig(
'tcp://{}:{}'.format(self.conf['host'], self.conf['port']),
login=self.conf['username'],
passcode=self.conf['password'],
version=protocol_v
)
self.conn = Stomp(client_config)
self.conn.connect()
return super().__init__()
def emit(self, record):
'''
Send the message to the destination endpoint.
'''
self.conn.send(destination=self.conf['destination'],
body=bytes(self.format(record), 'utf-8'))
def init(self, host, port, username=None, password=None,
connect_timeout=3, connected_timeout=3,
version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"],
heartbeats=(0, 0), ssl_context=None,
use_ssl=True,
key_file=None,
cert_file=None,
ca_certs=None,
ssl_version=ssl.PROTOCOL_SSLv23,
key_file_password=None,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
channel=channel):
""" Initialize StompClient. Called after __init__ """
self.channel = channel
if proxy_host:
LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port)
else:
LOG.info("Connect to %s:%s", host, port)
if use_ssl and not ssl_context:
ssl_params = dict(key_file=key_file,
cert_file=cert_file,
ca_certs=ca_certs,
ssl_version=ssl_version,
password=key_file_password)
LOG.info("Request to use old-style socket wrapper: %s", ssl_params)
ssl_context = ssl_params
if use_ssl:
uri = "ssl://%s:%s" % (host, port)
else:
uri = "tcp://%s:%s" % (host, port)
# Configure failover options so it only tries to connect once
self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri
self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context,
version=version,
login=username,
passcode=password)
self._heartbeats = heartbeats
self._accept_versions = accept_versions
self._connect_timeout = connect_timeout
self._connected_timeout = connected_timeout
Stomp._transportFactory = EnhancedStompFrameTransport
Stomp._transportFactory.proxy_host = proxy_host
Stomp._transportFactory.proxy_port = proxy_port
Stomp._transportFactory.proxy_user = proxy_user
Stomp._transportFactory.proxy_password = proxy_password
self._client = Stomp(self._stomp_config)
self._subscribed = {}
self.server_heartbeat = None
self.client_heartbeat = None
self.ALLOWANCE = 2 # multiplier for heartbeat timeouts
def _get_timeouting_connect_mock(self):
stomp = Stomp(CONFIG)
stomp._transportFactory = mock.Mock()
transport = stomp._transportFactory.return_value = mock.Mock()
transport.host = 'mock'
transport.port = 0
transport.canRead.return_value = False
return stomp
def configureClient(self, _login, _passcode):
CONFIG = StompConfig("tcp://datafeeds.networkrail.co.uk:61618", login=_login, passcode=_passcode)
client = Stomp(CONFIG)
client.connect()
client.subscribe("/topic/TD_KENT_MCC_SIG_AREA")
self._client = client
def get_stomp_conn():
mqcfg = settings.MESSAGE_QUEUE
config = StompConfig(mqcfg['url'],
login=mqcfg['username'],
passcode=mqcfg['password'])
stomp = Stomp(config)
stomp.connect(host=mqcfg['broker'])
return stomp
def _get_connect_mock(self, receive=None, config=None):
stomp = Stomp(config or CONFIG)
stomp._transportFactory = Mock()
transport = stomp._transportFactory.return_value = Mock()
transport.host = 'mock'
transport.port = 0
if receive:
transport.receive.return_value = receive
return stomp
def recv_stomp ():
config = StompConfig (stomp_uri)
client = Stomp (config)
client.connect ()
client.subscribe (stomp_source, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
frame = client.receiveFrame ()
print "Received: {}".format (frame.info ())
client.ack (frame)
client.disconnect ()
def test(size):
config = StompConfig("tcp://%s:%d" % (ACTIVEMQ_HOST, ACTIVEMQ_PORT))
client = Stomp(config)
client.connect()
msg = "0" * size
for i in range(0, COUNT):
client.send(TOPIC, msg)
client.send(TOPIC, "quit")
client.disconnect()
def _get_connect_mock(self, receive=None, config=None):
stomp = Stomp(config or CONFIG)
stomp._transportFactory = Mock()
transport = stomp._transportFactory.return_value = Mock()
transport.host = 'mock'
transport.port = 0
if receive:
transport.receive.return_value = receive
return stomp
def establishConnection(self, address='localhost', port=61613):
dataTest = 'Hello World!'
print('endereco: ' + address)
print('porta: ' + str(port))
print('dados: ' + dataTest)
config = StompConfig('tcp://{}:{}'.format(address, port))
self.client = Stomp(config)
self.client.connect()
self.client.send(self.queue, dataTest.encode())
def _conn(cfg_uri, queue, _info):
from stompest.config import StompConfig
from stompest.sync import Stomp
_info("Init Stomp obj: [%s-%s]" % (cfg_uri, queue))
client = Stomp(StompConfig(cfg_uri))
_info("connecting... %s" % cfg_uri)
client.connect()
_info("connected %s" % cfg_uri)
return client
def _conn(cfg_uri, queue, _info):
from stompest.config import StompConfig
from stompest.sync import Stomp
_info('Init Stomp obj: [%s-%s]' % (cfg_uri, queue))
client = Stomp(StompConfig(cfg_uri))
_info('connecting... %s' % cfg_uri)
client.connect()
_info('connected %s' % cfg_uri)
return client
def __init__(self, tb, host='127.0.0.1', port=61613):
self.tb = tb
self.config = StompConfig('tcp://%s:%s' % (host, port))
self.client = Stomp(self.config)
self.client.connect()
threading.Thread.__init__(self)
self.setDaemon(1)
self.start()
def run(self):
client = Stomp(self.config)
client.connect()
headers = {
# client-individual mode is necessary for concurrent processing
# (requires ActiveMQ >= 5.2)
StompSpec.ACK_HEADER:
StompSpec.ACK_CLIENT_INDIVIDUAL,
# the maximal number of messages the broker will let you work on at
# the same time
'activemq.prefetchSize':
'100',
}
client.subscribe(config['queue']['BotNet'], headers)
while True:
frame = client.receiveFrame()
data = json.loads(frame.body)
body = data.get('body')
if body and data.get('action') == 'update' and \
body.get('func_name') == 'updater.receive' and \
body.get('func_args'):
args = body.get('func_args')
if args.get('data'):
doc_pin = args['data']['objects'][0]['id']
Storage().remove(doc_pin)
client.ack(frame)
def test_3_timeout(self):
timeout = 0.2
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61613)?startupMaxReconnectAttempts=1,randomize=false', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0))
client.connect(host=VIRTUALHOST, connectTimeout=timeout)
client.disconnect()
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61611)?startupMaxReconnectAttempts=1,backOffMultiplier=3', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0))
self.assertRaises(StompConnectionError, client.connect, host=VIRTUALHOST, connectTimeout=timeout)
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61613)?randomize=false', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0)) # default is startupMaxReconnectAttempts = 0
self.assertRaises(StompConnectionError, client.connect, host=VIRTUALHOST, connectTimeout=timeout)
def connect(self, mq_name, selector=None):
self.rsp_queue = config.mq.get(mq_name)
self.client = Stomp(StompConfig(config.mq.get('stomp')))
self.client.connect()
spec = {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}
if selector:
spec[StompSpec.SELECTOR_HEADER] = "%s='%s'" % selector
self.client.subscribe(self.rsp_queue, spec)
print 'Consumer [%s] waiting for messages ...' % self.rsp_queue
def __init__(self, selk, selv, config=None, queue=None):
'''
Constructor
'''
if config is not None:
self.CONFIG = config
if queue is not None:
self.QUEUE = queue
self.client = Stomp(self.CONFIG)
self.client.connect()
self.selk = selk
self.selv = selv
def init_connection(self):
if (self.client != None and self.client.session.state == 'connected'):
try:
self.client.disconnect()
except:
pass
config = StompConfig('tcp://%s:%s' % (self.host, self.port),
version=StompSpec.VERSION_1_1,
check=True)
self.client = Stomp(config)
self.client.connect(heartBeats=(30000, 0),
connectTimeout=1,
connectedTimeout=1)
def place_order(body):
conn = get_connection()
cursor = conn.cursor()
sql = "Insert into orders (total_amount, created_on, status, email) values (%s, %s,%s,%s )"
values = (body['total_amount'], body['created_on'], 'Ordered',
body['email'])
rows = cursor.execute(sql, values)
order_id = cursor.lastrowid
for row in body['products']:
sql = "Insert into order_details (product_id, order_id, product_name, quantity, unit_cost) values (%s, %s, %s, %s, %s)"
values = (row['product_id'], order_id, row['product_name'],
row['quantity'], row['unit_cost'])
rows = cursor.execute(sql, values)
conn.commit()
queue = '/queue/ordered'
amq_conf = None
# if isOpen('activemq-service.default', 61613):
amq_conf = StompConfig('tcp://activemq-service.default:61613')
#else:
# amq_conf = StompConfig('tcp://localhost:30012')
try:
client = Stomp(amq_conf)
client.connect()
client.send(queue, str(order_id).encode())
client.disconnect()
except:
print("something went wrong")
conn.close()
return {'order_id': order_id}
def queueFile(parameters):
bucket = app.config['s3.bucket']
username = app.config['s3.username']
password = app.config['s3.password']
input_folder = app.config['s3.input_folder'] or '/comets/input/'
if username and password:
s3 = boto3.resource('s3',
aws_access_key_id=username,
aws_secret_access_key=password)
else:
s3 = boto3.resource('s3')
s3.meta.client.upload_file(
parameters['filepath'],
bucket,
input_folder + parameters['filename'],
)
forQueue = json.dumps(parameters)
client = Stomp(
StompConfig('tcp://' + app.config['queue.host'] + ':' +
str(app.config['queue.port'])))
client.connect()
client.send('/queue/Comets', forQueue,
{'correlation-id': parameters['filename']})
client.disconnect()
def send(self):
"""
Create a new stomp configuration client, connect and
then serializes message by message posting
them to your consumers in TOPIC standard
and disconnect.
"""
try:
configuration = StompConfig(uri=self._broker_uri)
client = Stomp(configuration)
client.connect(connectTimeout=self._broker_timeout)
for message in self._queue:
serialized_message = json.dumps(message, ensure_ascii=False)
client.send(self._queue_destination, serialized_message)
client.disconnect()
except Exception, e:
self.log.error(
u'QueueManagerError - Error on sending objects from queue.')
self.log.debug(e)
raise Exception(
'QueueManagerError - Error on sending objects to queue.')
def __init__(self, namespace, login, passcode, broker_host=DEFAULT_BROKER_HOST, device_uid=None, channel_type='topic'):
'''
Create a ByteportStompClient. This is a thin wrapper to the underlying STOMP-client that connets to the Byteport Broker
If a device_uid is given, a subscription will be made for Messages sent through Byteport.
The channel_type must be either 'topic' or 'queue'. Set top topic if unsure on what to use (use queue if you need to
use multiple consumers for a single device, this is not how most applications are set up).
:param namespace:
:param login: Broker username (Byteport web users are _not_ valid broker users). Ask [email protected] for access.
:param passcode: Broker passcode
:param broker_hosts: [optional] A list of brokers to connect to
:param device_uid: [optional] The device UID to subscribe for messages on
:param channel_type: [optional] Defaults to queue.
:param channel_key: [optional] Must match the configured key in the Byteport Device Manager
'''
self.namespace = str(namespace)
self.device_uid = device_uid
if channel_type not in self.SUPPORTED_CHANNEL_TYPES:
raise Exception("Unsupported channel type: %s" % channel_type)
broker_url = 'tcp://%s:61613' % broker_host
self.CONFIG = StompConfig(broker_url, version=StompSpec.VERSION_1_2)
self.client = Stomp(self.CONFIG)
try:
self.client.connect(headers={'login': login, 'passcode': passcode}, host='/')
logging.info("Connected to Stomp broker at %s using protocol version %s" % (broker_host, self.client.session.version))
# Set up a subscription on the correct queue if a Specific device UID was given
if self.device_uid:
subscribe_headers = dict()
subscribe_headers[StompSpec.ACK_HEADER] = StompSpec.ACK_CLIENT_INDIVIDUAL
subscribe_headers[StompSpec.ID_HEADER] = '0'
device_message_queue_name = '/%s/device_messages_%s.%s' % (channel_type, namespace, device_uid)
self.subscription_token = self.client.subscribe(device_message_queue_name, subscribe_headers)
logging.info("Subscribing to channel %s" % device_message_queue_name)
except StompProtocolError as e:
logging.error("Client socket connected, but probably failed to login. (ProtocolError)")
raise
except StompConnectionError:
logging.error("Failed to connect to Stomp Broker at %s" % broker_host)
raise
def __init__(self, config):
"""Init the Stompest wrapper client.
:type config: dict
:param config: The configuration for the STOM client.
I.e. {'host': 'tcp://127.0.0.1',
'queue': '/queue/test',
'transaction': True,
'username': '******',
'password': '******'}
The transaction attribute defines if messages should be published
in transactions.
"""
self.host = config['host']
self.queue = config['queue']
self.transactions_enabled = config['transaction']
self.transaction = None
auth_header = {}
if 'username' in config and 'password' in config:
auth_header.update(
{StompSpec.LOGIN_HEADER: config['username'],
StompSpec.PASSCODE_HEADER: config['password']})
self.client = Stomp(StompConfig(self.host))
try:
self.client.connect(headers=auth_header)
except (error.StompConnectTimeout, error.StompProtocolError) as e:
raise ClientErrors.ConnectionError(
"Could not connect to broker: %s" % e)
class AMQPublisher:
ip = Config.get("bus_ip")
port = Config.get("bus_port_stomp")
user = Config.get("bus_login")
pw = Config.get("bus_pass")
def __init__(self):
stompConfig = StompConfig("tcp://" + self.ip + ":" + self.port,
login=self.user,
passcode=self.pw)
self.client = Stomp(stompConfig)
self.client.connect()
def publish(self, topic, message):
self.client.send("/topic/" + topic, message)
def send(data):
mqcfg = settings.MESSAGE_QUEUE
config = StompConfig(mqcfg['url'],
login=mqcfg['username'],
passcode=mqcfg['password'])
stomp = Stomp(config)
stomp.connect(host=mqcfg['broker'])
stomp.send(mqcfg['vote_queue'], json.dumps(data))
stomp.disconnect()
def init(self, host, port, username=None, password=None,
connect_timeout=3, connected_timeout=3,
version=StompSpec.VERSION_1_2, accept_versions=["1.0", "1.1", "1.2"],
heartbeats=(0, 0), ssl_context=None,
use_ssl=True,
key_file=None,
cert_file=None,
ca_certs=None,
ssl_version=ssl.PROTOCOL_SSLv23,
key_file_password=None,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
channel=channel):
""" Initialize StompClient. Called after __init__ """
self.channel = channel
if proxy_host:
LOG.info("Connect to %s:%s through proxy %s:%d", host, port, proxy_host, proxy_port)
else:
LOG.info("Connect to %s:%s", host, port)
if use_ssl and not ssl_context:
ssl_params = dict(key_file=key_file,
cert_file=cert_file,
ca_certs=ca_certs,
ssl_version=ssl_version,
password=key_file_password)
LOG.info("Request to use old-style socket wrapper: %s", ssl_params)
ssl_context = ssl_params
if use_ssl:
uri = "ssl://%s:%s" % (host, port)
else:
uri = "tcp://%s:%s" % (host, port)
# Configure failover options so it only tries to connect once
self._stomp_server = "failover:(%s)?maxReconnectAttempts=1,startupMaxReconnectAttempts=1" % uri
self._stomp_config = StompConfig(uri=self._stomp_server, sslContext=ssl_context,
version=version,
login=username,
passcode=password)
self._heartbeats = heartbeats
self._accept_versions = accept_versions
self._connect_timeout = connect_timeout
self._connected_timeout = connected_timeout
Stomp._transportFactory = EnhancedStompFrameTransport
Stomp._transportFactory.proxy_host = proxy_host
Stomp._transportFactory.proxy_port = proxy_port
Stomp._transportFactory.proxy_user = proxy_user
Stomp._transportFactory.proxy_password = proxy_password
self._client = Stomp(self._stomp_config)
self._subscribed = {}
self.server_heartbeat = None
self.client_heartbeat = None
self.ALLOWANCE = 2 # multiplier for heartbeat timeouts
def __init__(self, sel, config=None, queue=None):
'''
Constructor
'''
if config is not None:
self.CONFIG = config
if queue is not None:
self.QUEUE = queue
self.selector = sel
self.client = Stomp(self.CONFIG)
self.client.connect()
self.client.subscribe(
self.QUEUE, {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
"selector": self.selector
})
def run(self, messages=None):
client = Stomp(self.config)
client.connect()
j = 0
for message in messages:
client.send(queue, message,
receipt='message-asterisk-%d' %
j, headers={'persistent': 'true'})
j = j + 1
client.disconnect(receipt='bye')
def writeJson(value,*argv):
'''Writes the specified value to an output file
Takes:
value-> List or Dict
*argv: Available options:
'stomp'
'post'
'outfile'
Returns:
none
'''
outJson=json.dumps(value)
print outJson
if 'stomp' in argv:
CONFIG = StompConfig(stomp_config.server, stomp_config.login,stomp_config.passcode)
QUEUE = stomp_config.queue
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, outJson)
client.disconnect()
elif 'outfile' in argv:
with open(outfile,'w') as jsonFile:
jsonFile.write(outJson)
elif 'post' in argv:
#TODO Post to php server
pass
def connect(self, host, port=61613, user=None, passw=None, timeout=10):
"""Method connects to server
Args:
host (str): hostname
port (str): port
user (str): username
passw (str): password
timeout (int): timeout
Returns:
bool: result
Raises:
event: jms_before_connect
event: jms_after_connected
"""
try:
msg = 'host:{0}, port:{1}, user:{2}, passw:{3}, timeout:{4}'.format(
host, port, user, passw, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connecting', msg), self._mh.fromhere())
ev = event.Event(
'jms_before_connect', host, port, user, passw, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
timeout = ev.argv(4)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client = Stomp(StompConfig('tcp://{0}:{1}'.format(self._host, self._port),
login=self._user, passcode=self._passw))
self._client.connect(
connectTimeout=timeout, connectedTimeout=timeout)
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connected'), self._mh.fromhere())
ev = event.Event('jms_after_connect')
self._mh.fire_event(ev)
return True
except StompError as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
class ConsumerAMQ(object):
'''
classdocs
'''
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/revideomessagesFE'
FRAMEBODY = "MESSAGE"
def __init__(self, sel, config=None, queue=None):
'''
Constructor
'''
if config is not None:
self.CONFIG = config
if queue is not None:
self.QUEUE = queue
self.selector = sel
self.client = Stomp(self.CONFIG)
self.client.connect()
self.client.subscribe(
self.QUEUE, {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
"selector": self.selector
})
def receive(self):
frame = None
cnt = 5
while True:
frame = self.client.receiveFrame()
#print "frame: " + frame.info()
if frame.command == self.FRAMEBODY:
self.client.ack(frame)
break
frame = None
cnt = cnt - 1
if cnt <= 0:
break
if frame is None:
return frame
else:
return frame.body
def end(self):
self.client.disconnect()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_file')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--logfile')
parser.add_argument('--purge', action='store_true')
parser.add_argument('--loadgen', action='store_true')
parser.add_argument('--sessionid')
parser.add_argument('--httpserver', action='store_true')
args = parser.parse_args()
if args.logfile:
fh = FileHandler(args.logfile)
formatter = logging.Formatter(FORMAT)
fh.setFormatter(formatter)
logger.addHandler(fh)
if args.debug:
logger.setLevel(logging.DEBUG)
config = yaml.load(open(args.config_file))
sessionmaker = config_to_db_session(config, Base)
stomp_endpoint = config['stomp_endpoint']
stomp_login = config['stomp_login']
stomp_password = config['stomp_password']
stomp_config = StompConfig(stomp_endpoint, login=stomp_login, passcode=stomp_password)
if args.purge:
purge_completed_jobs(sessionmaker())
return
elif args.loadgen:
if not args.sessionid:
raise ValueError('parameter sessionid is required for load generator')
jm = StompSync(stomp_config)
jm.connect()
generate_load(jm, args.sessionid)
jm.disconnect()
return
elif args.httpserver:
from jobmanager.www import app
app.config['DB'] = sessionmaker
app.config['params'] = config
app.run()
return
jm = JobManager(sessionmaker, config=stomp_config)
jm.run()
l = task.LoopingCall(jm.run_periodic_checks)
l.start(PERIODIC_CHECK_INTERVAL)
reactor.run()
def __init__(self, conf):
self.conf = conf
if self.conf['protocol'] == '11':
protocol_v = StompSpec.VERSION_1_1
elif self.conf['protocol'] == '12':
protocol_v = StompSpec.VERSION_1_2
else:
protocol_v = StompSpec.VERSION_1_0
client_config = StompConfig(
'tcp://{}:{}'.format(self.conf['host'], self.conf['port']),
login=self.conf['username'],
passcode=self.conf['password'],
version=protocol_v
)
self.conn = Stomp(client_config)
self.conn.connect()
return super().__init__()
def connect(self):
if not self.stompest:
CONFIG, EXTRA = _get_config()
self._hostname = EXTRA.get('hostname', None)
self.stompest = Stomp(CONFIG)
if self.stompest.session.state != StompSession.DISCONNECTED:
return
while True:
try:
self.stompest.connect(host=self._hostname)
logger.info('Connected')
break
except StompConnectTimeout:
continue
if not self.greenlet:
self.greenlet = gevent.spawn(self._run)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', nargs=1, required=True, help='destination', metavar='dest')
parser.add_argument('-e', action='store_true', default=False, help='persistent msgs flag')
parser.add_argument('-a', nargs=1, required=False, help='fixed timestamp', metavar='fixed timestamp')
parser.add_argument('-n', nargs=1, default=False, help='number of msgs', metavar='int')
parser.add_argument('-o', default=6163, type=int, help='port', metavar='port')
parser.add_argument('-z', default=16, type=int, help='size of msg payload', metavar='msg prefix')
parser.add_argument('-p', nargs=1, default=False, help='msg prefix', metavar='msg prefix')
parser.add_argument('-s', nargs=1, required=True, help='broker', metavar='broker')
parser.add_argument('-i', action='store_true', required=False, default=False, help='paired service type')
parser.add_argument('-w', action='store_true', default=False, help='format message wrongly')
parser.add_argument('-t', nargs=1, default=False, help='send msg every sec', metavar='float')
parser.add_argument('-v', action='store_true', default=False, help='verbose')
args = parser.parse_args()
broker = 'tcp://%s:%i' % (args.s[0], args.o)
config = StompConfig(broker)
client = Stomp(config)
client.connect()
try:
i = 0
if args.n:
while i < int(args.n[0]):
msg = gen_msg(args)
if args.v:
print str(i)
print '%.128s' % msg
send_msg(args, client, msg)
i += 1
else:
while True:
msg = gen_msg(args)
if args.v:
print str(i)
print '%.128s' % msg
send_msg(args, client, msg)
i += 1
except KeyboardInterrupt:
client.disconnect()
raise SystemExit(1)
def __init__(self, data, conf):
logging.debug("json data: %s [%s]" % (str(data), type(data)))
if not type(data) == dict:
raise ValueError, "PostProcessAdmin expects a data dictionary"
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
# List of error messages to be handled as information
self.exceptions = self.conf.exceptions
stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user,
self.conf.amq_pwd)
self.client = Stomp(stompConfig)
self.data_file = None
self.facility = None
self.instrument = None
self.proposal = None
self.run_number = None
def _active_queue_configure(self, host, port, user_name, password):
""" TO connect to Active MQ """
host = str(host)
port = str(port)
user_name = str(user_name)
password = str(password)
if not user_name:
raise AssertionError(
"user_Name argument is required.!! Please check and pass user_Name value"
)
if not password:
raise AssertionError(
"password argument is required.!! Please check and pass password value"
)
ActiveMQ_url = "tcp://{}:{}".format(host, port)
ActiveMQ_Client = None
config = StompConfig(ActiveMQ_url, login=user_name, passcode=password)
ActiveMQ_Client = Stomp(config)
ActiveMQ_Client.connect()
return ActiveMQ_Client
def __init__(self, host, port, version, headers={}):
"""
Creates an instance of the StompMQAdapter class.
Args:
host (str): Hostname or IP address for locating the MQ system.
port (int): The port to connect to.
stomp_version (str): The Stomp protocol version ot be used.
headers (dict, optional): Defaults to {}. The headers to be used in the connection.
"""
self.client = Stomp(
StompConfig('tcp://{}:{}'.format(host, port), version=version))
# Assign mandatory ID_HEADER if version above 1.1
if float(version) > 1.1:
self.headers[StompSpec.ID_HEADER] = id(self.client)
# Add all given headers to object headers
for key in headers.keys():
self.headers[key] = headers[key]
def connect_to_amq(tb, queue=False, topic=False):
# Format the connection url
dl_ff_ip, dl_north_rest = tb.dl_northside_rest.split(":")
dl_activemq_stomp_port = 61613
url = "tcp://{}:{}".format(dl_ff_ip, dl_activemq_stomp_port)
# Create stomp config
config = StompConfig(url)
stomp = Stomp(config)
# Connect to activemq
stomp.connect()
if queue:
stomp.subscribe(queue, {StompSpec.ID_HEADER: u'testbench'})
elif topic:
stomp.subscribe('/topic/' + str(topic), {StompSpec.ID_HEADER: u'testbench'})
# return the stomp
return stomp
def connect(self):
if not self.stompest:
CONFIG, EXTRA = _get_config()
self._hostname = EXTRA.get('hostname', None)
self.stompest = Stomp(CONFIG)
if self.stompest.session.state != StompSession.DISCONNECTED:
return
while True:
try:
self.stompest.connect(host=self._hostname)
logger.info('Connected')
break
except StompConnectTimeout:
continue
if not self.greenlet:
self.greenlet = gevent.spawn(self._run)
def __init__(self, data, conf):
logging.debug("json data: %s [%s]" % (str(data), type(data)))
if not type(data) == dict:
raise ValueError, "PostProcessAdmin expects a data dictionary"
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
# List of error messages to be handled as information
self.exceptions = self.conf.exceptions
stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user, self.conf.amq_pwd)
self.client = Stomp(stompConfig)
self.data_file = None
self.facility = None
self.instrument = None
self.proposal = None
self.run_number = None
def run(self):
files = os.listdir(self.path)
for queue_file in files:
archive_file = '{0}/archive/{1}'.format(self.path, queue_file)
if queue_file.startswith('archive'):
pass
else:
with open("{0}/{1}".format(self.path, queue_file), 'r') as qf:
get_lines = list(qf)
for line in get_lines:
dts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
client = Stomp(config=self.config)
client.connect()
print "Sending message {0} to queue {1}".format(line, queue_file)
with open(archive_file, 'a') as af:
af.write("{0} Sent message: {1} to queue {2}\n".format(dts, line, queue_file))
client.send(queue_file, json.dumps(line))
client.disconnect()
def send_message(messageBody, destination=None, queueName=None):
client = None
if destination != None:
client = Stomp(StompConfig(destination))
else:
client = Stomp(StompConfig("tcp://localhost:61613"))
QUEUE = None
if queueName != None:
QUEUE = queueName
else:
QUEUE = "pods2jbpm"
#client = Stomp(CONFIG)
client.connect()
body = messageBody
client.send(QUEUE, body)
client.disconnect()
def send(self):
"""
Create a new stomp configuration client, connect and
then serializes message by message posting
them to your consumers in TOPIC standard
and disconnect.
"""
try:
configuration = StompConfig(uri=self._broker_uri)
client = Stomp(configuration)
client.connect(connectTimeout=self._broker_timeout)
for message in self._queue:
serialized_message = json.dumps(message, ensure_ascii=False)
client.send(self._queue_destination, serialized_message)
client.disconnect()
except Exception, e:
self.log.error(u"QueueManagerError - Error on sending objects from queue.")
self.log.debug(e)
def main(username, password):
hostname = 'ec2-184-72-17-222.us-west-1.compute.amazonaws.com'
port = '61613'
# The names of the channels we will subscribe to
channel1 = '/topic/moa.voevent'
channel2 = '/topic/test.test'
# Connect
client = Stomp(StompConfig('tcp://' + hostname + ':' + port))
client.connect(headers={'login': username, 'passcode': password})
# Subscribe to each channel. For a durable subscription, a subscription
# name is associated with each channel
for (name, channel) in [('topic1', channel1), ('topic2', channel2)]:
client.subscribe(channel, headers={'activemq.subscriptionName' : name})
while True:
try:
frame = client.receiveFrame()
# We got a frame - from which channel did it come from?
wherefrom = frame.headers['destination'] + ': '
moa_id, int_id, ra, dec, jd, concept= parse_voe(frame.body)
outs = wherefrom + ' '.join([moa_id, int_id, ra, dec, jd, concept])
print outs
except etree.XMLSyntaxError, e:
outs = wherefrom + frame.body + ' (doesn\'t look like XML)'
print str(e)
print outs
except KeyboardInterrupt:
print "Disconnecting on keyboard interrupt ..."
break
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1)
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(heartBeats=(0, 10000))
client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
client.send(QUEUE, 'test message 1')
client.send(QUEUE, 'test message 2')
while True:
frame = client.receiveFrame()
print 'Got %s' % frame.info()
client.ack(frame)
client.disconnect()
import ssl
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
context = ssl.create_default_context()
# Disable cert validation for demo only
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
CONFIG = StompConfig('ssl://localhost:61612', sslContext=context)
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
while True:
frame = client.receiveFrame()
print('Got %s' % frame.info())
client.ack(frame)
client.disconnect()
class PostProcessAdmin:
def __init__(self, data, conf):
logging.debug("json data: %s [%s]" % (str(data), type(data)))
if not type(data) == dict:
raise ValueError, "PostProcessAdmin expects a data dictionary"
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
# List of error messages to be handled as information
self.exceptions = self.conf.exceptions
stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user, self.conf.amq_pwd)
self.client = Stomp(stompConfig)
self.data_file = None
self.facility = None
self.instrument = None
self.proposal = None
self.run_number = None
def _process_data(self, data):
"""
Retrieve run information from the data dictionary
provided with an incoming message.
@param data: data dictionary
"""
if data.has_key('data_file'):
self.data_file = str(data['data_file'])
if os.access(self.data_file, os.R_OK) == False:
raise ValueError("Data file does not exist or is not readable: %s" % self.data_file)
else:
raise ValueError("data_file is missing: %s" % self.data_file)
if data.has_key('facility'):
self.facility = str(data['facility']).upper()
else:
raise ValueError("Facility is missing")
if data.has_key('instrument'):
self.instrument = str(data['instrument']).upper()
else:
raise ValueError("Instrument is missing")
if data.has_key('ipts'):
self.proposal = str(data['ipts']).upper()
else:
raise ValueError("IPTS is missing")
if data.has_key('run_number'):
self.run_number = str(data['run_number'])
else:
raise ValueError("Run number is missing")
def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " + summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data))
return
# Run the reduction
out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
# Determine error condition
success, status_data = job_handling.determine_success_local(self.conf, out_err)
self.data.update(status_data)
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data))
else:
self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
def catalog_raw(self):
"""
Catalog a nexus file containing raw data
"""
self._process_data(self.data)
try:
from ingest_nexus import IngestNexus
self.send('/queue/' + self.conf.catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
ingestNexus = IngestNexus(self.data_file)
ingestNexus.execute()
ingestNexus.logout()
self.send('/queue/' + self.conf.catalog_complete, json.dumps(self.data))
except:
logging.error("catalog_raw: %s" % sys.exc_value)
self.data["error"] = "Catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.catalog_error, json.dumps(self.data))
def catalog_reduced(self):
"""
Catalog reduced data files for a given run
"""
self._process_data(self.data)
try:
from ingest_reduced import IngestReduced
self.send('/queue/' + self.conf.reduction_catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
# Send image to the web monitor
if len(self.conf.web_monitor_url.strip()) > 0:
monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd}
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
url_template = string.Template(self.conf.web_monitor_url)
url = url_template.substitute(instrument=self.instrument, run_number=self.run_number)
pattern = self.instrument + "_" + self.run_number + "*"
for dirpath, dirnames, filenames in os.walk(proposal_shared_dir):
listing = glob.glob(os.path.join(dirpath, pattern))
for filepath in listing:
f, e = os.path.splitext(filepath)
if e.startswith(os.extsep):
e = e[len(os.extsep):]
if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat") or filepath.endswith("plot_data.json"):
files = {'file': open(filepath, 'rb')}
# Post the file if it's small enough
if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size:
request = requests.post(url, data=monitor_user, files=files, verify=False)
logging.info("Submitted %s [status: %s]" % (filepath,
request.status_code))
ingestReduced = IngestReduced(self.facility, self.instrument, self.proposal, self.run_number)
ingestReduced.execute()
ingestReduced.logout()
self.send('/queue/' + self.conf.reduction_catalog_complete , json.dumps(self.data))
except:
logging.error("catalog_reduced: %s" % sys.exc_value)
self.data["error"] = "Reduction catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.reduction_catalog_error , json.dumps(self.data))
def create_reduction_script(self):
"""
Create a new reduction script from a template
"""
try:
import reduction_script_writer
writer = reduction_script_writer.ScriptWriter(self.data["instrument"])
writer.process_request(self.data,
configuration=self.conf,
send_function=self.send)
except:
logging.error("create_reduction_script: %s" % sys.exc_value)
def send(self, destination, data):
"""
Send an AMQ message
@param destination: AMQ queue to send to
@param data: payload of the message
"""
logging.info("%s: %s" % (destination, data))
self.client.connect()
self.client.send(destination, data)
self.client.disconnect()
pp.create_reduction_script()
# Check for registered processors
if type(configuration.processors) == list:
for p in configuration.processors:
toks = p.split('.')
if len(toks) == 2:
processor_module = __import__("postprocessing.processors.%s" % toks[0], globals(), locals(), [toks[1], ], -1)
try:
processor_class = eval("processor_module.%s" % toks[1])
if namespace.queue == processor_class.get_input_queue_name():
# Instantiate and call the processor
proc = processor_class(data, configuration, send_function=pp.send)
proc()
except:
logging.error("PostProcessAdmin: Processor error: %s" % sys.exc_value)
else:
logging.error("PostProcessAdmin: Processors can only be specified in the format module.Processor_class")
except:
# If we have a proper data dictionary, send it back with an error message
if type(data) == dict:
data["error"] = str(sys.exc_value)
stomp = Stomp(StompConfig(configuration.failover_uri, configuration.amq_user, configuration.amq_pwd))
stomp.connect()
stomp.send(configuration.postprocess_error, json.dumps(data))
stomp.disconnect()
raise
except:
logging.error("PostProcessAdmin: %s" % sys.exc_value)
from stompest.config import StompConfig
from stompest.sync import Stomp
user = os.getenv('APOLLO_USER') or 'admin'
password = os.getenv('APOLLO_PASSWORD') or 'password'
host = os.getenv('APOLLO_HOST') or 'localhost'
port = int(os.getenv('APOLLO_PORT') or 61613)
destination = sys.argv[1:2] or ['/topic/event']
destination = destination[0]
messages = 10000
data = 'Hello World from Python'
config = StompConfig('tcp://%s:%d' % (host, port), login=user, passcode=password, version='1.1')
client = Stomp(config)
client.connect(host='mybroker')
count = 0
start = time.time()
for _ in xrange(messages):
client.send(destination=destination, body=data, headers={'persistent': 'false'})
count += 1
diff = time.time() - start
print 'Sent %s frames in %f seconds' % (count, diff)
client.disconnect(receipt='bye')
client.receiveFrame()
client.close()
import time
from stompest.config import StompConfig
from stompest.sync import Stomp
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
CONFIG = StompConfig('tcp://localhost:7777', version='1.2')
def toTime(input):
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(input))
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(connectedTimeout=4, heartBeats=(2000,2500))
print "--------------------------------------------------------------------"
print "state : ", client.session.state
print "client HB : ", client.session.clientHeartBeat
print "server HB : ", client.session.serverHeartBeat
print "server : ", client.session.server
print "id : ", client.session.id
print "lastSent : ", toTime(client.session.lastSent)
print "lastReceived : ", toTime(client.session.lastReceived)
client.disconnect()
sys.exit(0)
#!/usr/bin/python
import logging, sys
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
from stompest.config import StompConfig
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:7777', version='1.2')
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(connectedTimeout=4)
ID = 'bingo'
client.send('/my/test/destination', # destination
'THIS-IS-A-SEND-BODY', # body
headers={'receipt': ID}) # headers
answer = client.receiveFrame()
receiptID = answer.headers["receipt-id"]
returnValue = 0
if receiptID != ID:
print "Receipt header wrong:" + receiptID
returnValue = 1
else:
print "Correct receipt id received: " + receiptID
def _get_transport_mock(self, receive=None, config=None):
stomp = Stomp(config or CONFIG)
stomp._transport = Mock()
if receive:
stomp._transport.receive.return_value = receive
return stomp