def __get_broker_conns(brokers, port, use_ssl, vhost, reconnect_attempts, ssl_key_file, ssl_cert_file, timeout, logger=logging.log):
logger(logging.DEBUG, 'resolving broker dns alias: %s' % brokers)
brokers_resolved = []
for broker in brokers:
addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
brokers_resolved.extend(ai[4][0] for ai in addrinfos)
logger(logging.DEBUG, 'broker resolved to %s', brokers_resolved)
conns = []
for broker in brokers_resolved:
if not use_ssl:
conns.append(Connection(host_and_ports=[(broker, port)],
use_ssl=False,
vhost=vhost,
timeout=timeout,
heartbeats=(0, 1000),
reconnect_attempts_max=reconnect_attempts))
else:
conns.append(Connection(host_and_ports=[(broker, port)],
use_ssl=True,
ssl_key_file=ssl_key_file,
ssl_cert_file=ssl_cert_file,
vhost=vhost,
timeout=timeout,
heartbeats=(0, 1000),
reconnect_attempts_max=reconnect_attempts))
return conns
def verify_destination_metrics_collection(self, destination_type):
from stomp import Connection
self.render_config_template(modules=[self.get_activemq_module_config(destination_type)])
proc = self.start_beat(home=self.beat_path)
destination_name = ''.join(random.choice(string.ascii_lowercase) for i in range(10))
conn = Connection([self.get_stomp_host_port()])
conn.start()
conn.connect(wait=True)
conn.send('/{}/{}'.format(destination_type, destination_name), 'first message')
conn.send('/{}/{}'.format(destination_type, destination_name), 'second message')
self.wait_until(lambda: self.destination_metrics_collected(destination_type, destination_name))
proc.check_kill_and_wait()
self.assert_no_logged_warnings()
output = self.read_output_json()
passed = False
for evt in output:
if self.all_messages_enqueued(evt, destination_type, destination_name):
assert 0 < evt['activemq'][destination_type]['messages']['size']['avg']
self.assert_fields_are_documented(evt)
passed = True
conn.disconnect()
assert passed
def getMessagesFromMQChannel(self, messageToPost):
class MyListener(ConnectionListener):
def __init__(self, conn):
self.conn = conn
def on_error(self, headers, message):
print('received an error "%s"' % message)
def on_message(self, headers, message):
print('received a message "%s"' % message)
for x in range(10):
print(x)
time.sleep(1)
print('processed message')
def on_disconnected(self):
print('disconnected')
connect_and_subscribe(self.conn)
def connect_and_subscribe(conn):
conn.connect('admin', 'admin', wait=True)
conn.subscribe(destination='/queue/restdemo_queue',
id=1,
ack='auto')
hosts = [('127.0.0.1', 61613)]
conn = Connection(host_and_ports=hosts)
conn.set_listener('', PrintingListener())
#conn.set_listener('', MyListener(conn))
conn.connect('admin', 'admin', wait=True)
conn.subscribe(destination='/queue/restdemo_queue', id=1, ack='auto')
conn.send(body=messageToPost, destination='/queue/restdemo_queue')
time.sleep(2)
conn.disconnect()
def write_to_active_mq(self, message):
rospy.loginfo(str(datetime.datetime.now()) + ': Posting to ActiveMQ ' + str(message))
conn = Connection(host_and_ports=[(AntennaConstants.ACTIVE_MQ_HOST, AntennaConstants.ACTIVE_MQ_PORT)])
conn.start()
conn.connect(AntennaConstants.ACTIVE_MQ_USER, AntennaConstants.ACTIVE_MQ_PASS, wait=True)
conn.send(body=json.dumps(message), destination=AntennaConstants.ACTIVE_MQ_TOPIC_OUT)
conn.disconnect()
def _make_connection(self):
if self._conn is None or not self._conn.is_connected():
_log.debug("Creating connection")
self._conn = Connection([(self.stomp_address, self.stomp_port)])
if self._override_thread_fc is not None:
self._conn.transport.override_threading(
self._override_thread_fc)
self._conn.connect(self.__user, self.__pass)
def test_subscribe_send_Topic(self):
class MyListener():
def __init__(self):
#hosts = [('127.0.0.1', 61613)]
#conn = Connection(host_and_ports=hosts)
#self.conn = conn
print("MyListener __init__")
def on_error(self, message):
print('received an error "%s"' % message)
def on_message(self, message):
print('received a message on topic "%s"' % message)
TestMQ.topic_messageReceived.append(message)
for x in range(2):
print(x)
time.sleep(1)
def on_disconnected(self):
print('disconnected')
#connect_and_subscribe(self.conn)
print("test_subscribe_send_Topic()")
hosts = [('127.0.0.1', 61613)]
conn1 = Connection(host_and_ports=hosts)
conn2 = Connection(host_and_ports=hosts)
topic_listener1 = MyListener
topic_listener2 = MyListener
conn1.set_listener('', topic_listener1)
conn2.set_listener('', topic_listener2)
conn1.connect('admin', 'admin', wait=True)
conn2.connect('admin', 'admin', wait=True)
conn1.subscribe(destination='/topic/restdemo_topic', id=1, ack='auto')
conn2.subscribe(destination='/topic/restdemo_topic', id=1, ack='auto')
conn1.send(body='This is a message sent to restdemo_topic from test case', destination='/topic/restdemo_topic')
time.sleep(4)
conn1.disconnect()
conn2.disconnect()
#for i in range( len(TestMQ.topic_messageReceived)):
#print("TestMQ.messageReceived = " + TestMQ.topic_messageReceived[i])
self.assertEqual(len(TestMQ.topic_messageReceived), 2)
def __init__(self, hostname, port, unique_id, request_timeout_millis):
hosts = [(hostname, port)]
connect_timeout = 10
self.conn = Connection(host_and_ports=hosts, timeout=connect_timeout)
self.conn.start()
self.conn.connect(wait=True)
self.unique_id = unique_id
self.request_timeout_millis = request_timeout_millis
self._timer = None
def __init__(self, host_and_ports, topic, username=None, password=None):
self.topic = topic
self.host_and_ports = host_and_ports
self.username = username
self.password = password
self.should_reconnect = True
self.conn = Connection(host_and_ports)
self.conn.set_listener('', self)
self.logger = logging.getLogger(EventListener.__name__)
self.subscription_id = 94
def __init__(self, hostname, port, request_queue_name, response_queue_name,
request_timeout_millis):
hosts = [(hostname, port)]
connect_timeout = 10
self.conn = Connection(host_and_ports=hosts, timeout=connect_timeout)
self.conn.connect(wait=True)
self.request_queue_name = request_queue_name
self.response_queue_name = response_queue_name
self.request_timeout_millis = request_timeout_millis
self._timer = None
def connect(self):
try:
self._connection = Connection([(self.host, self.port)], self.user,
self.password)
self._connection.start()
self._connection.connect(wait=True)
except ConnectFailedException, err:
raise OfflineBrokerError(
"Failed to connect to the broker at stomp://%s:%s/" %
(self.host, self.port))
def setUp(self):
self.topic = 'testEventListening'
self.host_and_ports = mq.host_and_ports
self.listener = EventListener(self.host_and_ports, self.topic)
reactor = Mock()
self.listener.start_listening(reactor)
self.sender_client = Connection(self.host_and_ports)
self.sender_client.start()
self.sender_client.connect(wait=True)
def _sendMessage(self, json_data):
# type: (str) -> void
try:
try:
from stomp import Connection
except ImportError as ex:
print "You need stomp to run the archiver, run 'scipion run pip stomp.py' first"
raise ex
self.connection.send(self.queue_name, json_data)
except AttributeError:
self.connection = Connection([(self.queueHost, self.queuePort)])
self.connection.connect(wait=True)
self._sendMessage(json_data)
def connect_to_mq(self):
rospy.loginfo(
str(datetime.datetime.now()) + ': Connecting to ActiveMQ')
self.connection = Connection(
host_and_ports=[(AntennaConstants.ACTIVE_MQ_HOST,
AntennaConstants.ACTIVE_MQ_PORT)])
self.connection.set_listener('my_listener',
ActiveMQConnectionListener(self))
self.connection.start()
self.connection.connect(AntennaConstants.ACTIVE_MQ_USER,
AntennaConstants.ACTIVE_MQ_PASS,
wait=True)
self.connection.subscribe(
destination=AntennaConstants.ACTIVE_MQ_TOPIC_IN, id=1, ack='auto')
def _make_connection(self):
if self._conn is None or not self._conn.is_connected():
_log.debug("Creating connection")
self._conn = Connection([(self.stomp_address, self.stomp_port)])
if self._override_thread_fc is not None:
self._conn.transport.override_threading(self._override_thread_fc)
try:
self._conn.connect(self.__user, self.__pass, wait=True)
except TypeError:
pass
except NotConnectedException:
pass
except AttributeError:
pass
def _make_connection(self):
if self._conn is None or not self._conn.is_connected():
_log.debug("Creating connection")
if not self.__token:
# get token
# get initial connection
dt=datetime.now()
replyDest = f"temp.token_resp.{self.__user__}-{dt}"
# create token request string
userAuthStr = f"{self.__user__}:{self.__pass__}"
base64Str = base64.b64encode(userAuthStr.encode())
# set up token callback
# send request to token topic
tokenTopic = "/topic/pnnl.goss.token.topic"
tmpConn = Connection([(self.stomp_address, self.stomp_port)])
if self._override_thread_fc is not None:
tmpConn.transport.override_threading(self._override_thread_fc)
tmpConn.connect(self.__user__, self.__pass__, wait=True)
class TokenResponseListener():
def __init__(self):
self.__token = None
def get_token(self):
return self.__token
def on_message(self, header, message):
_log.debug("Internal on message is: {} {}".format(header, message))
self.__token = str(message)
def on_error(self, headers, message):
_log.error("ERR: {}".format(headers))
_log.error("OUR ERROR: {}".format(message))
def on_disconnect(self, header, message):
_log.debug("Disconnected")
# receive token and set token variable
# set callback
listener = TokenResponseListener()
# self.subscribe(replyDest, listener)
tmpConn.subscribe('/queue/'+replyDest, 123)
tmpConn.set_listener('token_resp', listener)
tmpConn.send(body=base64Str, destination=tokenTopic,
headers={'reply-to': replyDest})
# while token is null or for x iterations
iter = 0
while not self.__token and iter < 10:
# wait
self.__token = listener.get_token()
sleep(1)
iter += 1
self._conn = Connection([(self.stomp_address, self.stomp_port)])
if self._override_thread_fc is not None:
self._conn.transport.override_threading(self._override_thread_fc)
try:
self._conn.connect(self.__token, "", wait=True)
except TypeError as e:
_log.error("TypeError: {e}".format(e=e))
except NotConnectedException as e:
_log.error("NotConnectedException: {e}".format(e=e))
except AttributeError as e:
_log.error("AttributeError: {e}".format(e=e))
def kronos_file(once=False,
thread=0,
brokers_resolved=None,
dataset_queue=None):
"""
Main loop to consume tracer reports.
"""
logging.info('tracer consumer starting')
hostname = gethostname()
pid = getpid()
thread = current_thread()
chunksize = config_get_int('tracer-kronos', 'chunksize')
prefetch_size = config_get_int('tracer-kronos', 'prefetch_size')
subscription_id = config_get('tracer-kronos', 'subscription_id')
use_ssl = True
try:
use_ssl = config_get_bool('tracer-kronos', 'use_ssl')
except:
pass
if not use_ssl:
username = config_get('tracer-kronos', 'username')
password = config_get('tracer-kronos', 'password')
excluded_usrdns = set(
config_get('tracer-kronos', 'excluded_usrdns').split(','))
conns = []
for broker in brokers_resolved:
if not use_ssl:
conns.append(
Connection(host_and_ports=[
(broker, config_get_int('tracer-kronos', 'port'))
],
use_ssl=False,
reconnect_attempts_max=config_get_int(
'tracer-kronos', 'reconnect_attempts')))
else:
conns.append(
Connection(host_and_ports=[
(broker, config_get_int('tracer-kronos', 'port'))
],
use_ssl=True,
ssl_key_file=config_get('tracer-kronos',
'ssl_key_file'),
ssl_cert_file=config_get('tracer-kronos',
'ssl_cert_file'),
ssl_version=PROTOCOL_TLSv1,
reconnect_attempts_max=config_get_int(
'tracer-kronos', 'reconnect_attempts')))
logging.info('(kronos_file) tracer consumer started')
sanity_check(executable='kronos-file', hostname=hostname)
while not graceful_stop.is_set():
live(executable='kronos-file',
hostname=hostname,
pid=pid,
thread=thread)
for conn in conns:
if not conn.is_connected():
logging.info('(kronos_file) connecting to %s' %
conn.transport._Transport__host_and_ports[0][0])
record_counter('daemons.tracer.kronos.reconnect.%s' %
conn.transport._Transport__host_and_ports[0]
[0].split('.')[0])
conn.set_listener(
'rucio-tracer-kronos',
AMQConsumer(
broker=conn.transport._Transport__host_and_ports[0],
conn=conn,
queue=config_get('tracer-kronos', 'queue'),
chunksize=chunksize,
subscription_id=subscription_id,
excluded_usrdns=excluded_usrdns,
dataset_queue=dataset_queue))
conn.start()
if not use_ssl:
conn.connect(username, password)
else:
conn.connect()
conn.subscribe(
destination=config_get('tracer-kronos', 'queue'),
ack='client-individual',
id=subscription_id,
headers={'activemq.prefetchSize': prefetch_size})
sleep(1)
logging.info('(kronos_file) graceful stop requested')
for conn in conns:
try:
conn.disconnect()
except:
pass
die(executable='kronos-file', hostname=hostname, pid=pid, thread=thread)
logging.info('(kronos_file) graceful stop done')
from stomp import Connection, PrintingListener
import time
c = Connection([('127.0.0.1', 61613)])
c.set_listener('', PrintingListener())
c.start()
c.connect('emmet', 'masterbuilder', wait=True)
c.subscribe('/queue/test01', 123)
# c.send('queue/test01', 'a test message')
time.sleep(60)
def __init__(self, host='localhost', port=61613, user='', passcode=''):
self.c = Connection([(host, port)], user, passcode)
self.c.set_listener('', self)
self.c.start()
def kronos_file(once=False,
thread=0,
brokers_resolved=None,
dataset_queue=None,
sleep_time=60):
"""
Main loop to consume tracer reports.
"""
logging.info('tracer consumer starting')
hostname = socket.gethostname()
pid = getpid()
thread = current_thread()
chunksize = config_get_int('tracer-kronos', 'chunksize')
prefetch_size = config_get_int('tracer-kronos', 'prefetch_size')
subscription_id = config_get('tracer-kronos', 'subscription_id')
try:
bad_files_patterns = []
pattern = get(section='kronos',
option='bad_files_patterns',
session=None)
pattern = str(pattern)
patterns = pattern.split(",")
for pat in patterns:
bad_files_patterns.append(re.compile(pat.strip()))
except ConfigNotFound:
bad_files_patterns = []
except Exception as error:
logging.error('(kronos_file) Failed to get bad_file_patterns' +
str(error))
bad_files_patterns = []
use_ssl = True
try:
use_ssl = config_get_bool('tracer-kronos', 'use_ssl')
except Exception:
pass
if not use_ssl:
username = config_get('tracer-kronos', 'username')
password = config_get('tracer-kronos', 'password')
excluded_usrdns = set(
config_get('tracer-kronos', 'excluded_usrdns').split(','))
vhost = config_get('tracer-kronos',
'broker_virtual_host',
raise_exception=False)
conns = []
for broker in brokers_resolved:
if not use_ssl:
conns.append(
Connection(host_and_ports=[
(broker, config_get_int('tracer-kronos', 'port'))
],
use_ssl=False,
vhost=vhost,
reconnect_attempts_max=config_get_int(
'tracer-kronos', 'reconnect_attempts')))
else:
conns.append(
Connection(host_and_ports=[
(broker, config_get_int('tracer-kronos', 'port'))
],
use_ssl=True,
ssl_key_file=config_get('tracer-kronos',
'ssl_key_file'),
ssl_cert_file=config_get('tracer-kronos',
'ssl_cert_file'),
vhost=vhost,
reconnect_attempts_max=config_get_int(
'tracer-kronos', 'reconnect_attempts')))
logging.info('(kronos_file) tracer consumer started')
sanity_check(executable='kronos-file', hostname=hostname)
while not graceful_stop.is_set():
start_time = time()
live(executable='kronos-file',
hostname=hostname,
pid=pid,
thread=thread)
for conn in conns:
if not conn.is_connected():
logging.info('(kronos_file) connecting to %s' %
conn.transport._Transport__host_and_ports[0][0])
record_counter('daemons.tracer.kronos.reconnect.%s' %
conn.transport._Transport__host_and_ports[0]
[0].split('.')[0])
conn.set_listener(
'rucio-tracer-kronos',
AMQConsumer(
broker=conn.transport._Transport__host_and_ports[0],
conn=conn,
queue=config_get('tracer-kronos', 'queue'),
chunksize=chunksize,
subscription_id=subscription_id,
excluded_usrdns=excluded_usrdns,
dataset_queue=dataset_queue,
bad_files_patterns=bad_files_patterns))
conn.start()
if not use_ssl:
conn.connect(username, password)
else:
conn.connect()
conn.subscribe(
destination=config_get('tracer-kronos', 'queue'),
ack='client-individual',
id=subscription_id,
headers={'activemq.prefetchSize': prefetch_size})
tottime = time() - start_time
if tottime < sleep_time:
logging.info('(kronos_file) Will sleep for %s seconds' %
(sleep_time - tottime))
sleep(sleep_time - tottime)
logging.info('(kronos_file) graceful stop requested')
for conn in conns:
try:
conn.disconnect()
except Exception:
pass
die(executable='kronos-file', hostname=hostname, pid=pid, thread=thread)
logging.info('(kronos_file) graceful stop done')
