def queueFile(parameters):
bucket = app.config['s3.bucket']
username = app.config['s3.username']
password = app.config['s3.password']
input_folder = app.config['s3.input_folder'] or '/comets/input/'
if username and password:
s3 = boto3.resource('s3',
aws_access_key_id=username,
aws_secret_access_key=password)
else:
s3 = boto3.resource('s3')
s3.meta.client.upload_file(
parameters['filepath'],
bucket,
input_folder + parameters['filename'],
)
forQueue = json.dumps(parameters)
client = Stomp(
StompConfig('tcp://' + app.config['queue.host'] + ':' +
str(app.config['queue.port'])))
client.connect()
client.send('/queue/Comets', forQueue,
{'correlation-id': parameters['filename']})
client.disconnect()
def place_order(body):
conn = get_connection()
cursor = conn.cursor()
sql = "Insert into orders (total_amount, created_on, status, email) values (%s, %s,%s,%s )"
values = (body['total_amount'], body['created_on'], 'Ordered',
body['email'])
rows = cursor.execute(sql, values)
order_id = cursor.lastrowid
for row in body['products']:
sql = "Insert into order_details (product_id, order_id, product_name, quantity, unit_cost) values (%s, %s, %s, %s, %s)"
values = (row['product_id'], order_id, row['product_name'],
row['quantity'], row['unit_cost'])
rows = cursor.execute(sql, values)
conn.commit()
queue = '/queue/ordered'
amq_conf = None
# if isOpen('activemq-service.default', 61613):
amq_conf = StompConfig('tcp://activemq-service.default:61613')
#else:
# amq_conf = StompConfig('tcp://localhost:30012')
try:
client = Stomp(amq_conf)
client.connect()
client.send(queue, str(order_id).encode())
client.disconnect()
except:
print("something went wrong")
conn.close()
return {'order_id': order_id}
def Producer(account, message):
def documentCreateNotice():
result = dict(body=dict(func_name="documentCreateNotice",
func_args=[{
"doc_type": "importEmail"
}]),
recipient="*",
profile="user",
tag="grendizer")
return json.dumps(result)
def Send_Notify(message):
result = dict(body=dict(func_name="toastr.info", func_args=[message]),
recipient="*",
profile="user",
tag="grendizer")
return json.dumps(result)
def clearNotice():
result = dict(body=dict(func_name="toastr.clear", func_args=[]),
recipient="*",
profile="user",
tag="grendizer")
return json.dumps(result)
send_notify = Send_Notify(message)
clear_notice = clearNotice()
notice = documentCreateNotice()
logger.debug(u"Отправляем сообщение: %s" % message)
client = Stomp(StompConfig(default_uri))
client.connect()
client.send(queue, notice)
client.send(queue, send_notify)
client.send(queue, clear_notice)
client.disconnect()
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self):
if BROKER == 'rabbitmq':
print 'Broker does not support unicode characters. Skipping this test case.'
return
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (
e, version)
return
specialCharactersHeader = u'fen\xeatre:\r\n'
headers = {specialCharactersHeader: u'\xbfqu\xe9 tal?, s\xfc\xdf'}
client.send(self.DESTINATION, body='test message 1', headers=headers)
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(
self.DESTINATION, {
StompSpec.ID_HEADER: 4711,
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL
})
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
client.ack(frame)
self.assertEquals(frame.version, version)
self.assertEquals(frame.headers[specialCharactersHeader],
headers[specialCharactersHeader])
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.disconnect(receipt='4712')
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self):
if BROKER == 'rabbitmq':
print('Broker does not support unicode characters. Skipping this test case.')
return
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version))
return
key = b'fen\xc3\xaatre'.decode('utf-8')
value = b'\xc2\xbfqu\xc3\xa9 tal?'.decode('utf-8')
headers = {key: value}
client.send(self.DESTINATION, body=b'test message 1', headers=headers)
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
client.ack(frame)
self.assertEqual(frame.version, version)
self.assertEqual(frame.headers[key], headers[key])
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.disconnect(receipt='4712')
def test_4_integration_stomp_1_1(self):
if StompSpec.VERSION_1_1 not in commands.versions(VERSION):
print 'This broker does not support STOMP protocol version 1.1'
return
client = Stomp(self.getConfig(StompSpec.VERSION_1_1))
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, 'test message 1')
client.send(self.DESTINATION, 'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.send(self.DESTINATION, 'test message 3', receipt='4711')
self.assertTrue(client.canRead(self.TIMEOUT))
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'}))
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: 'client-individual'})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect(receipt='4712')
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4712'}))
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.disconnect(receipt='4711')
self.assertEquals(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {'receipt-id': '4711'}))
client.close()
self.assertRaises(StompConnectionError, client.canRead, 0)
class PublisherAMQ(object):
'''
classdocs
'''
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/revideomessagesBE'
def __init__(self, selk, selv, config=None, queue=None):
'''
Constructor
'''
if config is not None:
self.CONFIG = config
if queue is not None:
self.QUEUE = queue
self.client = Stomp(self.CONFIG)
self.client.connect()
self.selk = selk
self.selv = selv
def send(self, msg):
#print "Send::: " + self.selector
self.client.send(self.QUEUE, msg, headers={self.selk: self.selv})
def end(self):
self.client.disconnect()
class StompHandler(logging.Handler):
'''
Send messages to a remote queue broker using the STOMP protocol.
This module is named and configured separately from
standard logging for clarity about its scope: while logging has an
informational purpose, this module has a functional one.
'''
def __init__(self, conf):
self.conf = conf
if self.conf['protocol'] == '11':
protocol_v = StompSpec.VERSION_1_1
elif self.conf['protocol'] == '12':
protocol_v = StompSpec.VERSION_1_2
else:
protocol_v = StompSpec.VERSION_1_0
client_config = StompConfig(
'tcp://{}:{}'.format(self.conf['host'], self.conf['port']),
login=self.conf['username'],
passcode=self.conf['password'],
version=protocol_v
)
self.conn = Stomp(client_config)
self.conn.connect()
return super().__init__()
def emit(self, record):
'''
Send the message to the destination endpoint.
'''
self.conn.send(destination=self.conf['destination'],
body=bytes(self.format(record), 'utf-8'))
def test_6_integration_stomp_1_1_encoding_and_escaping_headers(self):
if BROKER == 'rabbitmq':
print 'Broker does not support unicode characters. Skipping this test case.'
return
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)
return
specialCharactersHeader = u'fen\xeatre:\r\n'
headers = {specialCharactersHeader: u'\xbfqu\xe9 tal?, s\xfc\xdf'}
client.send(self.DESTINATION, body='test message 1', headers=headers)
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
client.ack(frame)
self.assertEquals(frame.version, version)
self.assertEquals(frame.headers[specialCharactersHeader], headers[specialCharactersHeader])
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.disconnect(receipt='4712')
def send(self):
"""
Create a new stomp configuration client, connect and
then serializes message by message posting
them to your consumers in TOPIC standard
and disconnect.
"""
try:
configuration = StompConfig(uri=self._broker_uri)
client = Stomp(configuration)
client.connect(connectTimeout=self._broker_timeout)
for message in self._queue:
serialized_message = json.dumps(message, ensure_ascii=False)
client.send(self._queue_destination, serialized_message)
client.disconnect()
except Exception, e:
self.log.error(
u'QueueManagerError - Error on sending objects from queue.')
self.log.debug(e)
raise Exception(
'QueueManagerError - Error on sending objects to queue.')
def call_route(request_queue, response_queue, request):
"""
"""
config = {
"stomp": {
"server": '192.168.0.3',
"port": 61613,
"timeout": 15,
}
}
stomp_config = StompConfig("tcp://%s:%d" % (config['stomp']['server'], config['stomp']['port']), version=StompSpec.VERSION_1_0)
stomp = Stomp(stomp_config)
stomp.connect()
jms_id = str(uuid4())
token = stomp.subscribe(response_queue, {'JMSCorrelationID': jms_id})
stomp.send(request_queue, json.dumps(request), {'JMSCorrelationID': jms_id})
response = None
if stomp.canRead(config['stomp']['timeout']):
response = stomp.receiveFrame()
stomp.unsubscribe(token)
return response
def main():
logging.basicConfig()
logging.getLogger().setLevel(logging.WARN)
client = Stomp(stomp_config)
client.connect()
client.send(stomp_queue, body=stomp_body)
client.subscribe(stomp_queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT, 'activemq.prefetchSize': 1})
if client.canRead(timeout=5):
frame = client.receiveFrame()
print 'Got %s' % frame.info()
client.ack(frame)
frame_body = str(frame.body)
if frame_body == stomp_body:
print "OK: Message received"
status = 'ok'
else:
print "WARNING: Incorrect message body; is %s, should be %s" % (frame_body, stomp_body)
status = 'warning'
else:
print "CRITICAL: Timed out while trying to collect the message"
status = 'critical'
client.disconnect()
client.close(flush=True)
return exit_codes[status]
def _test_4_integration_stomp(self, version):
client = Stomp(self.getConfig(version))
try:
client.connect(host=VIRTUALHOST, versions=[version])
except StompProtocolError as e:
print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version))
return
client.send(self.DESTINATION, b'test message 1')
client.send(self.DESTINATION, b'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
token = client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.unsubscribe(token)
client.send(self.DESTINATION, b'test message 3', receipt='4711')
self.assertTrue(client.canRead(self.TIMEOUT))
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'}))
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 4711, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect(receipt='4712')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712'}))
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.disconnect(receipt='4711')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4711'}))
client.close()
self.assertRaises(StompConnectionError, client.canRead, 0)
def writeJson(value,*argv):
'''Writes the specified value to an output file
Takes:
value-> List or Dict
*argv: Available options:
'stomp'
'post'
'outfile'
Returns:
none
'''
outJson=json.dumps(value)
print outJson
if 'stomp' in argv:
CONFIG = StompConfig(stomp_config.server, stomp_config.login,stomp_config.passcode)
QUEUE = stomp_config.queue
client = Stomp(CONFIG)
client.connect()
client.send(QUEUE, outJson)
client.disconnect()
elif 'outfile' in argv:
with open(outfile,'w') as jsonFile:
jsonFile.write(outJson)
elif 'post' in argv:
#TODO Post to php server
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-d", nargs=1, required=True, help="destination", metavar="dest")
parser.add_argument("-e", action="store_true", default=False, help="persistent msgs flag")
parser.add_argument("-m", nargs=1, required=True, help="msg", metavar="msg")
parser.add_argument("-n", nargs=1, default=False, help="number of msgs", metavar="int")
parser.add_argument("-p", nargs=1, default=False, help="msg prefix", metavar="msg prefix")
parser.add_argument("-s", nargs=1, required=True, help="broker", metavar="broker")
parser.add_argument("-t", nargs=1, default=False, help="send msg every sec", metavar="float")
args = parser.parse_args()
broker = "tcp://%s:6163" % (args.s[0])
config = StompConfig(broker)
client = Stomp(config)
client.connect()
rd = "".join(random.sample("abcdefghijklmno", 2))
try:
i = 0
if args.n:
while i < int(args.n[0]):
send_msg(args, client, i, rd)
i += 1
else:
while True:
send_msg(args, client, i, rd)
i += 1
except KeyboardInterrupt:
client.disconnect()
raise SystemExit(1)
def run(self):
client = Stomp(self.config)
client.connect()
headers = {
# client-individual mode is necessary for concurrent processing
# (requires ActiveMQ >= 5.2)
StompSpec.ACK_HEADER:
StompSpec.ACK_CLIENT_INDIVIDUAL,
# the maximal number of messages the broker will let you work on at
# the same time
'activemq.prefetchSize':
'100',
}
client.subscribe(config['queue']['BotNet'], headers)
while True:
frame = client.receiveFrame()
data = json.loads(frame.body)
body = data.get('body')
if body and data.get('action') == 'update' and \
body.get('func_name') == 'updater.receive' and \
body.get('func_args'):
args = body.get('func_args')
if args.get('data'):
doc_pin = args['data']['objects'][0]['id']
Storage().remove(doc_pin)
client.ack(frame)
def configureClient(self, _login, _passcode):
CONFIG = StompConfig("tcp://datafeeds.networkrail.co.uk:61618", login=_login, passcode=_passcode)
client = Stomp(CONFIG)
client.connect()
client.subscribe("/topic/TD_KENT_MCC_SIG_AREA")
self._client = client
def get_stomp_conn():
mqcfg = settings.MESSAGE_QUEUE
config = StompConfig(mqcfg['url'],
login=mqcfg['username'],
passcode=mqcfg['password'])
stomp = Stomp(config)
stomp.connect(host=mqcfg['broker'])
return stomp
def send(data):
mqcfg = settings.MESSAGE_QUEUE
config = StompConfig(mqcfg['url'],
login=mqcfg['username'],
passcode=mqcfg['password'])
stomp = Stomp(config)
stomp.connect(host=mqcfg['broker'])
stomp.send(mqcfg['vote_queue'], json.dumps(data))
stomp.disconnect()
def recv_stomp ():
config = StompConfig (stomp_uri)
client = Stomp (config)
client.connect ()
client.subscribe (stomp_source, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
frame = client.receiveFrame ()
print "Received: {}".format (frame.info ())
client.ack (frame)
client.disconnect ()
def test(size):
config = StompConfig("tcp://%s:%d" % (ACTIVEMQ_HOST, ACTIVEMQ_PORT))
client = Stomp(config)
client.connect()
msg = "0" * size
for i in range(0, COUNT):
client.send(TOPIC, msg)
client.send(TOPIC, "quit")
client.disconnect()
def test_2_transaction(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.subscribe(
self.DESTINATION,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertFalse(client.canRead(self.TIMEOUT))
with client.transaction(4711) as transaction:
self.assertEquals(transaction, '4711')
client.send(self.DESTINATION, 'test message',
{StompSpec.TRANSACTION_HEADER: transaction})
self.assertFalse(client.canRead(0))
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEquals(frame.body, 'test message')
client.ack(frame)
with client.transaction(4713, receipt='4712') as transaction:
self.assertEquals(transaction, '4713')
self.assertEquals(
client.receiveFrame(),
StompFrame(StompSpec.RECEIPT,
{StompSpec.RECEIPT_ID_HEADER: '4712-begin'}))
client.send(self.DESTINATION, 'test message',
{StompSpec.TRANSACTION_HEADER: transaction})
client.send(self.DESTINATION, 'test message without transaction')
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEquals(frame.body, 'test message without transaction')
client.ack(frame)
self.assertFalse(client.canRead(0))
frames = [client.receiveFrame() for _ in xrange(2)]
frames = list(sorted(frames, key=lambda f: f.command))
frame = frames[0]
client.ack(frame)
self.assertEquals(frame.body, 'test message')
frame = frames[1]
self.assertEquals(
frame,
StompFrame(StompSpec.RECEIPT,
{StompSpec.RECEIPT_ID_HEADER: '4712-commit'}))
try:
with client.transaction(4714) as transaction:
self.assertEquals(transaction, '4714')
client.send(self.DESTINATION, 'test message',
{StompSpec.TRANSACTION_HEADER: transaction})
raise RuntimeError('poof')
except RuntimeError as e:
self.assertEquals(str(e), 'poof')
else:
raise
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-a', action='store_true', default=False, help='client ack')
parser.add_argument('-b', action='store_true', default=False, help='write body only')
parser.add_argument('-d', nargs=1, required=True, help='destination', metavar='dest')
parser.add_argument('-f', nargs=1, required=True, help='file with recorded msgs', metavar='file')
parser.add_argument('-n', nargs=1, default=False, help='number of msgs', metavar='int')
parser.add_argument('-r', action='store_true', default=False, help='reconnect')
parser.add_argument('-s', nargs=1, required=True, help='broker', metavar='broker')
parser.add_argument('-t', nargs=1, default=False, help='recv msg every sec', metavar='float')
args = parser.parse_args()
broker = 'tcp://%s:6163' % (args.s[0])
config = StompConfig(broker)
client = Stomp(config)
if not args.r:
client.connect()
if args.a:
client.subscribe(args.d[0], {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
else:
client.subscribe(args.d[0], {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
try:
consumed = 0
while True:
if args.r:
client.connect()
if args.a:
client.subscribe(args.d[0], {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
else:
client.subscribe(args.d[0], {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
if args.t:
time.sleep(float(args.t[0]))
else:
time.sleep(1)
fo = open(args.f[0], 'a+')
frame = client.receiveFrame()
consumed += 1
if args.b:
fo.write(frame.body+'\n')
else:
fo.write(frame.info()+'\n')
fo.close()
if args.a:
client.ack(frame)
if args.r:
client.disconnect()
if args.n:
if consumed == int(args.n[0]):
raise KeyboardInterrupt
except KeyboardInterrupt:
client.stop()
client.disconnect()
raise SystemExit(1)
except stompest.error.StompProtocolError:
pass
def _conn(cfg_uri, queue, _info):
from stompest.config import StompConfig
from stompest.sync import Stomp
_info('Init Stomp obj: [%s-%s]' % (cfg_uri, queue))
client = Stomp(StompConfig(cfg_uri))
_info('connecting... %s' % cfg_uri)
client.connect()
_info('connected %s' % cfg_uri)
return client
def run(self, messages=None):
client = Stomp(self.config)
client.connect()
j = 0
for message in messages:
client.send(queue, message,
receipt='message-asterisk-%d' %
j, headers={'persistent': 'true'})
j = j + 1
client.disconnect(receipt='bye')
def _conn(cfg_uri, queue, _info):
from stompest.config import StompConfig
from stompest.sync import Stomp
_info("Init Stomp obj: [%s-%s]" % (cfg_uri, queue))
client = Stomp(StompConfig(cfg_uri))
_info("connecting... %s" % cfg_uri)
client.connect()
_info("connected %s" % cfg_uri)
return client
def setUp(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
client.subscribe(self.DESTINATION, {StompSpec.ID_HEADER: 'bla', StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
while client.canRead(self.TIMEOUT):
frame = client.receiveFrame()
self.log.debug('Dequeued old %s' % frame.info())
client.disconnect()
def recv_stomp():
config = StompConfig(stomp_uri)
client = Stomp(config)
client.connect()
client.subscribe(stomp_source,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
frame = client.receiveFrame()
print "Received: {}".format(frame.info())
client.ack(frame)
client.disconnect()
def run(self):
if(processgid > 0): os.setgid(processgid)
if(processuid > 0): os.setuid(processuid)
config = StompConfig('tcp://%(HOST)s:%(PORT)s' % ActiveMQ)
topic = "/topic/%(FILESYNCTOPIC)s" % ActiveMQ
client = Stomp(config)
self.logger = logging.getLogger('nfssync')
self.logger.setLevel(logging.DEBUG)
handler = TimedRotatingFileHandler(LOGFILE, when='midnight', interval=1, backupCount=30)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
try:
client.connect()
client.subscribe(topic, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
while True:
frame = client.receiveFrame()
subdir = frame.body
srcpath = os.path.join(ActiveMQ['SRCDIR'], subdir)
dstpath = os.path.dirname(os.path.join(ActiveMQ['DSTDIR'], subdir))
self.logger.info("Syncing %s to %s" % (srcpath,dstpath))
if(not os.path.exists(srcpath)):
msg = "Source %s does not exist" % srcpath
self.logger.error(msg)
proxy_email(email_subject_error, msg)
client.ack(frame)
continue
elif(not os.path.isdir(srcpath)):
msg = "Source %s is not a directory" % srcpath
self.logger.error(msg)
proxy_email(email_subject_error, msg)
client.ack(frame)
continue
elif(not os.path.exists(dstpath)):
msg = "Destination %s does not exist" % dstpath
self.logger.warning(msg)
proxy_email(email_subject_warning, msg)
os.umask(0)
os.makedirs(dstpath, 0777)
cmd = "rsync -avzq --delete %s %s" % (srcpath,dstpath)
if(call(cmd, shell=True) > 0):
msg = "Sync %s failed" % cmd
self.logger.error(msg)
proxy_email(email_subject, msg)
client.ack(frame)
except Exception, e:
msg = "Exception in %s: %s" % (sys.argv[0], str(e))
self.logger.error(msg)
proxy_email(email_subject_error, msg)
exit(1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_file')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--logfile')
parser.add_argument('--purge', action='store_true')
parser.add_argument('--loadgen', action='store_true')
parser.add_argument('--sessionid')
parser.add_argument('--httpserver', action='store_true')
args = parser.parse_args()
if args.logfile:
fh = FileHandler(args.logfile)
formatter = logging.Formatter(FORMAT)
fh.setFormatter(formatter)
logger.addHandler(fh)
if args.debug:
logger.setLevel(logging.DEBUG)
config = yaml.load(open(args.config_file))
sessionmaker = config_to_db_session(config, Base)
stomp_endpoint = config['stomp_endpoint']
stomp_login = config['stomp_login']
stomp_password = config['stomp_password']
stomp_config = StompConfig(stomp_endpoint, login=stomp_login, passcode=stomp_password)
if args.purge:
purge_completed_jobs(sessionmaker())
return
elif args.loadgen:
if not args.sessionid:
raise ValueError('parameter sessionid is required for load generator')
jm = StompSync(stomp_config)
jm.connect()
generate_load(jm, args.sessionid)
jm.disconnect()
return
elif args.httpserver:
from jobmanager.www import app
app.config['DB'] = sessionmaker
app.config['params'] = config
app.run()
return
jm = JobManager(sessionmaker, config=stomp_config)
jm.run()
l = task.LoopingCall(jm.run_periodic_checks)
l.start(PERIODIC_CHECK_INTERVAL)
reactor.run()
def test_3_timeout(self):
timeout = 0.2
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61613)?startupMaxReconnectAttempts=1,randomize=false', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0))
client.connect(host=VIRTUALHOST, connectTimeout=timeout)
client.disconnect()
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61611)?startupMaxReconnectAttempts=1,backOffMultiplier=3', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0))
self.assertRaises(StompConnectionError, client.connect, host=VIRTUALHOST, connectTimeout=timeout)
client = Stomp(StompConfig(uri='failover:(tcp://localhost:61610,tcp://localhost:61613)?randomize=false', login=LOGIN, passcode=PASSCODE, version=StompSpec.VERSION_1_0)) # default is startupMaxReconnectAttempts = 0
self.assertRaises(StompConnectionError, client.connect, host=VIRTUALHOST, connectTimeout=timeout)
def test():
config = StompConfig("tcp://%s:%d" % (ACTIVEMQ_HOST, ACTIVEMQ_PORT))
client = Stomp(config)
client.connect()
client.subscribe(TOPIC,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
while True:
f = client.receiveFrame()
client.ack(f)
if f.body == "quit":
client.disconnect()
break
def test_1_integration(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 1')
client.send(self.DESTINATION, b'test message 2')
self.assertFalse(client.canRead(self.TIMEOUT))
client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertTrue(client.canRead(self.TIMEOUT))
client.ack(client.receiveFrame())
self.assertFalse(client.canRead(self.TIMEOUT))
class ConsumerAMQ(object):
'''
classdocs
'''
CONFIG = StompConfig('tcp://localhost:61613')
QUEUE = '/queue/revideomessagesFE'
FRAMEBODY = "MESSAGE"
def __init__(self, sel, config=None, queue=None):
'''
Constructor
'''
if config is not None:
self.CONFIG = config
if queue is not None:
self.QUEUE = queue
self.selector = sel
self.client = Stomp(self.CONFIG)
self.client.connect()
self.client.subscribe(
self.QUEUE, {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
"selector": self.selector
})
def receive(self):
frame = None
cnt = 5
while True:
frame = self.client.receiveFrame()
#print "frame: " + frame.info()
if frame.command == self.FRAMEBODY:
self.client.ack(frame)
break
frame = None
cnt = cnt - 1
if cnt <= 0:
break
if frame is None:
return frame
else:
return frame.body
def end(self):
self.client.disconnect()
def test_5_integration_stomp_1_1_heartbeat(self):
version = StompSpec.VERSION_1_1
if BROKER == 'apollo':
print "Broker %s doesn't properly support heart-beating. Skipping test." % BROKER
return
port = 61612 if (BROKER == 'activemq') else PORT # stomp+nio on 61613 does not work properly, so use stomp on 61612
client = Stomp(self.getConfig(StompSpec.VERSION_1_1, port))
self.assertEquals(client.lastReceived, None)
self.assertEquals(client.lastSent, None)
heartBeatPeriod = 100
try:
client.connect(host=VIRTUALHOST, heartBeats=(heartBeatPeriod, heartBeatPeriod), versions=[version])
except StompProtocolError as e:
print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)
return
self.assertTrue((time.time() - client.lastReceived) < 0.1)
if not (client.serverHeartBeat and client.clientHeartBeat):
print 'broker does not support heart-beating. disconnecting ...'
client.disconnect()
client.close()
return
serverHeartBeatInSeconds = client.serverHeartBeat / 1000.0
clientHeartBeatInSeconds = client.clientHeartBeat / 1000.0
start = time.time()
while (time.time() - start) < (2.5 * max(serverHeartBeatInSeconds, clientHeartBeatInSeconds)):
time.sleep(0.5 * min(serverHeartBeatInSeconds, clientHeartBeatInSeconds))
client.canRead(0)
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
if (time.time() - client.lastSent) > (0.5 * clientHeartBeatInSeconds):
client.beat()
self.assertTrue((time.time() - client.lastSent) < 0.1)
start = time.time()
try:
while not client.canRead(0.5 * clientHeartBeatInSeconds):
pass
except StompConnectionError:
self.assertTrue((time.time() - start) < (3.0 * clientHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastSent) > clientHeartBeatInSeconds)
else:
raise
client.close()
class ActiveMQSession(object):
"""
Create a scoped session for every request and
closes it when the request ends.
"""
def __init__(self, stompconf):
self.client = Stomp(stompconf)
def process_resource(self, req, resp, resource, params):
self.client.connect()
resource.activemq_conn = self.client
def process_response(self, req, resp, resource, req_succeeded):
if hasattr(resource, 'activemq_conn'):
self.client.disconnect()
self.client.close()
def _sender(queue_req, msg, session_id, msg_id):
stomp_config = config.mq_config.get('stomp')
client = Stomp(StompConfig(stomp_config))
client.connect()
#H = {StompSpec.CONTENT_LENGTH_HEADER: len(msg), "length": len(msg)}
H = {}
H['MsgID'] = msg_id
H['SessionID'] = session_id
H['Signature'] = sign(msg)
if isinstance(msg, dict): msg = json.dumps(msg)
print '===> [%s] session_ids: %s, body: %s' % (queue_req, session_id, msg)
client.send(queue_req, body=msg, headers=H, receipt="ok")
client.disconnect()
class AMQPublisher:
ip = Config.get("bus_ip")
port = Config.get("bus_port_stomp")
user = Config.get("bus_login")
pw = Config.get("bus_pass")
def __init__(self):
stompConfig = StompConfig("tcp://" + self.ip + ":" + self.port,
login=self.user,
passcode=self.pw)
self.client = Stomp(stompConfig)
self.client.connect()
def publish(self, topic, message):
self.client.send("/topic/" + topic, message)
def connect_to_amq(tb, queue=False, topic=False):
# Format the connection url
dl_ff_ip, dl_north_rest = tb.dl_northside_rest.split(":")
dl_activemq_stomp_port = 61613
url = "tcp://{}:{}".format(dl_ff_ip, dl_activemq_stomp_port)
# Create stomp config
config = StompConfig(url)
stomp = Stomp(config)
# Connect to activemq
stomp.connect()
if queue:
stomp.subscribe(queue, {StompSpec.ID_HEADER: u'testbench'})
elif topic:
stomp.subscribe('/topic/' + str(topic), {StompSpec.ID_HEADER: u'testbench'})
# return the stomp
return stomp
def test_5_integration_stomp_1_1_heartbeat(self):
version = StompSpec.VERSION_1_1
port = 61612 if (BROKER == 'activemq') else PORT # stomp+nio on 61613 does not work properly, so use stomp on 61612
client = Stomp(self.getConfig(StompSpec.VERSION_1_1, port))
self.assertEquals(client.lastReceived, None)
self.assertEquals(client.lastSent, None)
heartBeatPeriod = 100
try:
client.connect(host=VIRTUALHOST, heartBeats=(heartBeatPeriod, heartBeatPeriod), versions=[version])
except StompProtocolError as e:
print 'Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version)
return
self.assertTrue((time.time() - client.lastReceived) < 0.1)
if not (client.serverHeartBeat and client.clientHeartBeat):
print 'broker does not support heart-beating. disconnecting ...'
client.disconnect()
client.close()
return
serverHeartBeatInSeconds = client.serverHeartBeat / 1000.0
clientHeartBeatInSeconds = client.clientHeartBeat / 1000.0
start = time.time()
while (time.time() - start) < (2.5 * max(serverHeartBeatInSeconds, clientHeartBeatInSeconds)):
time.sleep(0.5 * min(serverHeartBeatInSeconds, clientHeartBeatInSeconds))
client.canRead(0)
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
if (time.time() - client.lastSent) > (0.5 * clientHeartBeatInSeconds):
client.beat()
self.assertTrue((time.time() - client.lastSent) < 0.1)
start = time.time()
try:
while not client.canRead(0.5 * clientHeartBeatInSeconds):
pass
if client.receiveFrame().command == StompSpec.ERROR:
raise StompProtocolError()
except (StompConnectionError, StompProtocolError):
self.assertTrue((time.time() - start) < (3.0 * clientHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastSent) > clientHeartBeatInSeconds)
else:
raise
client.close()
class Client():
_dados = Data()
startExperimentTS = ''
client = ''
queue = '/queue/tcc_stomp'
def establishConnection(self, address='localhost', port=61613):
dataTest = 'Hello World!'
print('endereco: ' + address)
print('porta: ' + str(port))
print('dados: ' + dataTest)
config = StompConfig('tcp://{}:{}'.format(address, port))
self.client = Stomp(config)
self.client.connect()
self.client.send(self.queue, dataTest.encode())
def startExperiment(self, reps=-1, timePerRep=1):
"""
Inicia o Experimento
>>Reps: quantidade de repetições que o experimento terá.
Cada repetição é igual a 1 pacote enviado.
Caso seja <= 0 serão enviados todos os dados disponíveis
>>timePerRep:
intervalo de tempo entre repetição, em segundos.
Padrão = 1seg
"""
print("\nIniciando o experimento às {0}".format(
util.getFormattedDatetimeWithMillisec()))
if (reps <= 0):
reps = self._dados.length()
print("Quantidade de pacotes que serão enviados: {}".format(reps))
duracao = timePerRep * reps
print("Tempo estimado de duração do experimento: {} ({})\n".format(
util.getFormattedDateTimeFromSeconds(duracao),
str(datetime.timedelta(seconds=duracao))))
for i in range(0, reps):
print("iniciando repetição {} às {}".format(
i + 1, str(util.getFormattedDatetimeWithMillisec())))
self.sendPackage(i)
time.sleep(timePerRep)
def sendPackage(self, index):
print(TAB_1 + "Enviando pacote ...")
self.client.send(self.queue, self._dados.getByIndex(index).encode())
def sendqueue(tokenId):
#try:
timestr = time.strftime("%Y-%m-%d")
QUEUE = jpsurvConfig.getAsString(QUEUE_NAME)
QUEUE_CONFIG = StompConfig(jpsurvConfig.getAsString(QUEUE_URL))
client = Stomp(QUEUE_CONFIG)
client.connect()
client.send(
QUEUE,
json.dumps({
"filepath": UPLOAD_DIR,
"token": tokenId,
"timestamp": timestr
}))
client.disconnect()
return
def change_order_status(id):
conn = get_connection()
cursor = conn.cursor()
cursor.execute("SELECT status FROM orders where order_id = " + str(id) +
"")
rows = cursor.fetchall()
status = rows[0][0]
next_status = None
if status == 'Ordered':
next_status = 'ReadyToShip'
queue = '/queue/readytoship'
elif status == 'ReadyToShip':
next_status = 'Shipped'
queue = '/queue/shipped'
elif status == 'Shipped':
next_status = 'Delivered'
if status == 'Delivered':
return True
if next_status == 'ReadyToShip' or next_status == 'Shipped':
amq_conf = None
#if isOpen('activemq.default', 61612):
amq_conf = StompConfig('tcp://activemq-service.default:61613')
#else:
# amq_conf = StompConfig('tcp://localhost:30012')
try:
client = Stomp(amq_conf)
client.connect()
client.send(queue, str(id).encode())
client.disconnect()
except:
print("something went wrong")
sql = "Update systeam_ecommerce.orders SET status = '" + next_status + "' where order_id = " + str(
id) + ""
rows = cursor.execute(sql)
sql = "Delete from systeam_ecommerce.next_order where status = '" + status + "'"
rows = cursor.execute(sql)
conn.commit()
conn.close()
return True
def test_2_transaction(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = Stomp(config)
client.connect(host=VIRTUALHOST)
client.subscribe(self.DESTINATION, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
self.assertFalse(client.canRead(self.TIMEOUT))
with client.transaction(4711) as transaction:
self.assertEqual(transaction, '4711')
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
self.assertFalse(client.canRead(0))
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEqual(frame.body, b'test message')
client.ack(frame)
with client.transaction(4713, receipt='4712') as transaction:
self.assertEqual(transaction, '4713')
self.assertEqual(client.receiveFrame(), StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-begin'}))
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
client.send(self.DESTINATION, b'test message without transaction')
self.assertTrue(client.canRead(self.TIMEOUT))
frame = client.receiveFrame()
self.assertEqual(frame.body, b'test message without transaction')
client.ack(frame)
self.assertFalse(client.canRead(0))
frames = [client.receiveFrame() for _ in range(2)]
frames = list(sorted(frames, key=lambda f: f.command))
frame = frames[0]
client.ack(frame)
self.assertEqual(frame.body, b'test message')
frame = frames[1]
self.assertEqual(frame, StompFrame(StompSpec.RECEIPT, {StompSpec.RECEIPT_ID_HEADER: '4712-commit'}))
try:
with client.transaction(4714) as transaction:
self.assertEqual(transaction, '4714')
client.send(self.DESTINATION, b'test message', {StompSpec.TRANSACTION_HEADER: transaction})
raise RuntimeError('poof')
except RuntimeError as e:
self.assertEqual(str(e), 'poof')
else:
raise
self.assertFalse(client.canRead(self.TIMEOUT))
client.disconnect()
def test_5_integration_stomp_1_1_heartbeat(self):
version = StompSpec.VERSION_1_1
client = Stomp(self.getConfig(StompSpec.VERSION_1_1))
self.assertEqual(client.lastReceived, None)
self.assertEqual(client.lastSent, None)
heartBeatPeriod = 100
try:
client.connect(host=VIRTUALHOST, heartBeats=(heartBeatPeriod, heartBeatPeriod), versions=[version])
except StompProtocolError as e:
print('Broker does not support STOMP protocol %s. Skipping this test case. [%s]' % (e, version))
return
self.assertTrue((time.time() - client.lastReceived) < 0.1)
if not (client.serverHeartBeat and client.clientHeartBeat):
print('broker does not support heart-beating. disconnecting ...')
client.disconnect()
client.close()
return
serverHeartBeatInSeconds = client.serverHeartBeat / 1000.0
clientHeartBeatInSeconds = client.clientHeartBeat / 1000.0
start = time.time()
while (time.time() - start) < (2.5 * max(serverHeartBeatInSeconds, clientHeartBeatInSeconds)):
time.sleep(0.5 * min(serverHeartBeatInSeconds, clientHeartBeatInSeconds))
client.canRead(0)
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
if (time.time() - client.lastSent) > (0.5 * clientHeartBeatInSeconds):
client.beat()
self.assertTrue((time.time() - client.lastSent) < 0.1)
start = time.time()
try:
while not client.canRead(0.5 * clientHeartBeatInSeconds):
pass
if client.receiveFrame().command == StompSpec.ERROR:
raise StompProtocolError()
except (StompConnectionError, StompProtocolError):
self.assertTrue((time.time() - start) < (3.0 * clientHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastReceived) < (2.0 * serverHeartBeatInSeconds))
self.assertTrue((time.time() - client.lastSent) > clientHeartBeatInSeconds)
else:
raise
client.close()
def run(self):
files = os.listdir(self.path)
for queue_file in files:
archive_file = '{0}/archive/{1}'.format(self.path, queue_file)
if queue_file.startswith('archive'):
pass
else:
with open("{0}/{1}".format(self.path, queue_file), 'r') as qf:
get_lines = list(qf)
for line in get_lines:
dts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
client = Stomp(config=self.config)
client.connect()
print "Sending message {0} to queue {1}".format(line, queue_file)
with open(archive_file, 'a') as af:
af.write("{0} Sent message: {1} to queue {2}\n".format(dts, line, queue_file))
client.send(queue_file, json.dumps(line))
client.disconnect()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', nargs=1, required=True, help='destination', metavar='dest')
parser.add_argument('-e', action='store_true', default=False, help='persistent msgs flag')
parser.add_argument('-a', nargs=1, required=False, help='fixed timestamp', metavar='fixed timestamp')
parser.add_argument('-n', nargs=1, default=False, help='number of msgs', metavar='int')
parser.add_argument('-o', default=6163, type=int, help='port', metavar='port')
parser.add_argument('-z', default=16, type=int, help='size of msg payload', metavar='msg prefix')
parser.add_argument('-p', nargs=1, default=False, help='msg prefix', metavar='msg prefix')
parser.add_argument('-s', nargs=1, required=True, help='broker', metavar='broker')
parser.add_argument('-i', action='store_true', required=False, default=False, help='paired service type')
parser.add_argument('-w', action='store_true', default=False, help='format message wrongly')
parser.add_argument('-t', nargs=1, default=False, help='send msg every sec', metavar='float')
parser.add_argument('-v', action='store_true', default=False, help='verbose')
args = parser.parse_args()
broker = 'tcp://%s:%i' % (args.s[0], args.o)
config = StompConfig(broker)
client = Stomp(config)
client.connect()
try:
i = 0
if args.n:
while i < int(args.n[0]):
msg = gen_msg(args)
if args.v:
print str(i)
print '%.128s' % msg
send_msg(args, client, msg)
i += 1
else:
while True:
msg = gen_msg(args)
if args.v:
print str(i)
print '%.128s' % msg
send_msg(args, client, msg)
i += 1
except KeyboardInterrupt:
client.disconnect()
raise SystemExit(1)
def send_message(messageBody, destination=None, queueName=None):
client = None
if destination != None:
client = Stomp(StompConfig(destination))
else:
client = Stomp(StompConfig("tcp://localhost:61613"))
QUEUE = None
if queueName != None:
QUEUE = queueName
else:
QUEUE = "pods2jbpm"
#client = Stomp(CONFIG)
client.connect()
body = messageBody
client.send(QUEUE, body)
client.disconnect()
def main(username, password):
hostname = 'ec2-184-72-17-222.us-west-1.compute.amazonaws.com'
port = '61613'
# The names of the channels we will subscribe to
channel1 = '/topic/moa.voevent'
channel2 = '/topic/test.test'
# Connect
client = Stomp(StompConfig('tcp://' + hostname + ':' + port))
client.connect(headers={'login': username, 'passcode': password})
# Subscribe to each channel. For a durable subscription, a subscription
# name is associated with each channel
for (name, channel) in [('topic1', channel1), ('topic2', channel2)]:
client.subscribe(channel, headers={'activemq.subscriptionName' : name})
while True:
try:
frame = client.receiveFrame()
# We got a frame - from which channel did it come from?
wherefrom = frame.headers['destination'] + ': '
moa_id, int_id, ra, dec, jd, concept= parse_voe(frame.body)
outs = wherefrom + ' '.join([moa_id, int_id, ra, dec, jd, concept])
print outs
except etree.XMLSyntaxError, e:
outs = wherefrom + frame.body + ' (doesn\'t look like XML)'
print str(e)
print outs
except KeyboardInterrupt:
print "Disconnecting on keyboard interrupt ..."
break
def send(self):
"""
Create a new stomp configuration client, connect and
then serializes message by message posting
them to your consumers in TOPIC standard
and disconnect.
"""
try:
configuration = StompConfig(uri=self._broker_uri)
client = Stomp(configuration)
client.connect(connectTimeout=self._broker_timeout)
for message in self._queue:
serialized_message = json.dumps(message, ensure_ascii=False)
client.send(self._queue_destination, serialized_message)
client.disconnect()
except Exception, e:
self.log.error(u"QueueManagerError - Error on sending objects from queue.")
self.log.debug(e)
def test_3_socket_failure_and_replay(self):
client = Stomp(self.getConfig(StompSpec.VERSION_1_0))
client.connect(host=VIRTUALHOST)
headers = {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}
token = client.subscribe(self.DESTINATION, headers)
client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 1')
client.ack(client.receiveFrame())
client.unsubscribe(token)
headers = {StompSpec.ID_HEADER: 'bla', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}
client.subscribe(self.DESTINATION, headers)
headers[StompSpec.DESTINATION_HEADER] = self.DESTINATION
client.sendFrame(StompFrame(StompSpec.DISCONNECT)) # DISCONNECT frame is out-of-band, as far as the session is concerned -> unexpected disconnect
self.assertRaises(StompConnectionError, client.receiveFrame)
client.connect(host=VIRTUALHOST)
client.send(self.DESTINATION, b'test message 2')
client.ack(client.receiveFrame())
client.unsubscribe((StompSpec.ID_HEADER, 'bla'))
client.disconnect()
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1)
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect(heartBeats=(0, 10000))
client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
client.send(QUEUE, 'test message 1')
client.send(QUEUE, 'test message 2')
while True:
frame = client.receiveFrame()
print 'Got %s' % frame.info()
client.ack(frame)
client.disconnect()
import ssl
from stompest.config import StompConfig
from stompest.protocol import StompSpec
from stompest.sync import Stomp
context = ssl.create_default_context()
# Disable cert validation for demo only
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
CONFIG = StompConfig('ssl://localhost:61612', sslContext=context)
QUEUE = '/queue/test'
if __name__ == '__main__':
client = Stomp(CONFIG)
client.connect()
client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
while True:
frame = client.receiveFrame()
print('Got %s' % frame.info())
client.ack(frame)
client.disconnect()
class PostProcessAdmin:
def __init__(self, data, conf):
logging.debug("json data: %s [%s]" % (str(data), type(data)))
if not type(data) == dict:
raise ValueError, "PostProcessAdmin expects a data dictionary"
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
# List of error messages to be handled as information
self.exceptions = self.conf.exceptions
stompConfig = StompConfig(self.conf.failover_uri, self.conf.amq_user, self.conf.amq_pwd)
self.client = Stomp(stompConfig)
self.data_file = None
self.facility = None
self.instrument = None
self.proposal = None
self.run_number = None
def _process_data(self, data):
"""
Retrieve run information from the data dictionary
provided with an incoming message.
@param data: data dictionary
"""
if data.has_key('data_file'):
self.data_file = str(data['data_file'])
if os.access(self.data_file, os.R_OK) == False:
raise ValueError("Data file does not exist or is not readable: %s" % self.data_file)
else:
raise ValueError("data_file is missing: %s" % self.data_file)
if data.has_key('facility'):
self.facility = str(data['facility']).upper()
else:
raise ValueError("Facility is missing")
if data.has_key('instrument'):
self.instrument = str(data['instrument']).upper()
else:
raise ValueError("Instrument is missing")
if data.has_key('ipts'):
self.proposal = str(data['ipts']).upper()
else:
raise ValueError("IPTS is missing")
if data.has_key('run_number'):
self.run_number = str(data['run_number'])
else:
raise ValueError("Run number is missing")
def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " + summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data))
return
# Run the reduction
out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
# Determine error condition
success, status_data = job_handling.determine_success_local(self.conf, out_err)
self.data.update(status_data)
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data))
else:
self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
def catalog_raw(self):
"""
Catalog a nexus file containing raw data
"""
self._process_data(self.data)
try:
from ingest_nexus import IngestNexus
self.send('/queue/' + self.conf.catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
ingestNexus = IngestNexus(self.data_file)
ingestNexus.execute()
ingestNexus.logout()
self.send('/queue/' + self.conf.catalog_complete, json.dumps(self.data))
except:
logging.error("catalog_raw: %s" % sys.exc_value)
self.data["error"] = "Catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.catalog_error, json.dumps(self.data))
def catalog_reduced(self):
"""
Catalog reduced data files for a given run
"""
self._process_data(self.data)
try:
from ingest_reduced import IngestReduced
self.send('/queue/' + self.conf.reduction_catalog_started, json.dumps(self.data))
if self.conf.comm_only is False:
# Send image to the web monitor
if len(self.conf.web_monitor_url.strip()) > 0:
monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd}
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
url_template = string.Template(self.conf.web_monitor_url)
url = url_template.substitute(instrument=self.instrument, run_number=self.run_number)
pattern = self.instrument + "_" + self.run_number + "*"
for dirpath, dirnames, filenames in os.walk(proposal_shared_dir):
listing = glob.glob(os.path.join(dirpath, pattern))
for filepath in listing:
f, e = os.path.splitext(filepath)
if e.startswith(os.extsep):
e = e[len(os.extsep):]
if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat") or filepath.endswith("plot_data.json"):
files = {'file': open(filepath, 'rb')}
# Post the file if it's small enough
if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size:
request = requests.post(url, data=monitor_user, files=files, verify=False)
logging.info("Submitted %s [status: %s]" % (filepath,
request.status_code))
ingestReduced = IngestReduced(self.facility, self.instrument, self.proposal, self.run_number)
ingestReduced.execute()
ingestReduced.logout()
self.send('/queue/' + self.conf.reduction_catalog_complete , json.dumps(self.data))
except:
logging.error("catalog_reduced: %s" % sys.exc_value)
self.data["error"] = "Reduction catalog: %s" % sys.exc_value
self.send('/queue/' + self.conf.reduction_catalog_error , json.dumps(self.data))
def create_reduction_script(self):
"""
Create a new reduction script from a template
"""
try:
import reduction_script_writer
writer = reduction_script_writer.ScriptWriter(self.data["instrument"])
writer.process_request(self.data,
configuration=self.conf,
send_function=self.send)
except:
logging.error("create_reduction_script: %s" % sys.exc_value)
def send(self, destination, data):
"""
Send an AMQ message
@param destination: AMQ queue to send to
@param data: payload of the message
"""
logging.info("%s: %s" % (destination, data))
self.client.connect()
self.client.send(destination, data)
self.client.disconnect()
