def startBrokerAccess(self):
"""
New-style management access to the broker. Can be used in lieu of startQmf.
"""
if 'broker_conn' not in self.__dict__:
self.broker_conn = qpid.messaging.Connection(str(self.broker))
self.broker_conn.open()
self.broker_access = BrokerAgent(self.broker_conn)
def checkOverStockedQueues(host):
connection = Connection.establish(host)
broker = BrokerAgent(connection)
queues = broker.getAllQueues()
result = list()
for q in queues:
if (q.msgDepth != 0):
print(q.name + " " + str(q.msgDepth))
result.append([q.name, q.msgDepth])
return result
def test_relay(self):
brokerB = self.amqp_broker()
agentB = BrokerAgent(brokerB.connect())
agentB.create("queue", "q")
self.agent.create("domain", "BrokerB", {
"url": brokerB.host_port(),
"sasl_mechanisms": "NONE"
})
#send to q on broker B through brokerA
self.send_and_receive(send_config=Config(self.broker,
address="q@BrokerB"),
recv_config=Config(brokerB))
def _node_disambiguation_precreated(self, ambiguous_send):
agent = BrokerAgent(self.conn)
agent.addExchange("fanout", "ambiguous")
agent.addQueue("ambiguous")
try:
r1 = self.ssn.receiver("ambiguous; {node:{type:topic}}")
r2 = self.ssn.receiver("ambiguous; {node:{type:queue}}")
self._node_disambiguation_test(r1,
r2,
ambiguous_send=ambiguous_send)
finally:
agent.delExchange("ambiguous")
agent.delQueue("ambiguous", False, False)
def test_ambiguous_delete_2(self):
agent = BrokerAgent(self.conn)
agent.addExchange("fanout", "ambiguous")
agent.addQueue("ambiguous")
self.ssn.receiver(
"ambiguous; {delete:receiver, node:{type:queue}}").close()
exchange = agent.getExchange("ambiguous")
queue = agent.getQueue("ambiguous")
try:
assert (exchange)
assert (not queue)
finally:
if exchange: agent.delExchange("ambiguous")
if queue: agent.delQueue("ambiguous", False, False)
def test_ambiguous_create_2(self):
#create exchange first, then queue
r1 = self.ssn.receiver(
"ambiguous; {create:receiver, node:{type:topic}}")
r2 = self.ssn.receiver(
"ambiguous; {create:receiver, node:{type:queue}}")
agent = BrokerAgent(self.conn)
exchange = agent.getExchange("ambiguous")
queue = agent.getQueue("ambiguous")
try:
assert (exchange)
assert (queue)
self._node_disambiguation_test(r1, r2)
finally:
if exchange: agent.delExchange("ambiguous")
if queue: agent.delQueue("ambiguous", False, False)
def test_outgoing_link(self):
brokerB = self.amqp_broker()
agentB = BrokerAgent(brokerB.connect())
self.agent.create("queue", "q")
agentB.create("queue", "q")
self.agent.create("domain", "BrokerB", {
"url": brokerB.host_port(),
"sasl_mechanisms": "NONE"
})
self.agent.create("outgoing", "Link1", {
"domain": "BrokerB",
"source": "q",
"target": "q"
})
#send to brokerA, receive from brokerB
self.send_and_receive(recv_config=Config(brokerB))
def migrate(*args, **kwargs):
"""
Migrate qpid queues:
- Ensure pulp.task is no longer *exclusive*.
- Rename agent queues: consumer_id> => pulp.agent.<consumer_id>
"""
transport = pulp_conf.get('messaging', 'transport')
if transport != 'qpid':
# not using qpid
return
if not QPID_MESSAGING_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpid.messaging is not '
'installed. Pulp\'s Qpid client dependencies can be installed with the '
'\"pulp-server-qpid\" package group. See the installation docs for more '
'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.'
)
_logger.error(msg)
raise Exception(msg)
if not QPIDTOOLLIBS_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpidtoollibs is not '
'installed. Pulp\'s Qpid client dependencies can be installed with the '
'\"pulp-server-qpid\" package group. See the installation docs for more '
'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.'
)
_logger.error(msg)
raise Exception(msg)
url = urlparse(pulp_conf.get('messaging', 'url'))
connection = Connection(host=url.hostname,
port=url.port,
transport=url.scheme,
reconnect=False,
ssl_certfile=pulp_conf.get('messaging',
'clientcert'),
ssl_skip_hostname_check=True)
connection.attach()
broker = BrokerAgent(connection)
_migrate_reply_queue(broker)
_migrate_agent_queues(broker)
connection.detach()
def migrate(*args, **kwargs):
"""
Migrate qpid queues:
- Ensure pulp.task is no longer *exclusive*.
- Rename agent queues: consumer_id> => pulp.agent.<consumer_id>
"""
transport = pulp_conf.get('messaging', 'transport')
if transport != 'qpid':
# not using qpid
return
if not QPID_MESSAGING_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpid.messaging is not '
'installed. Please install qpid.messaging and rerun the migrations. See %s'
'for more information.')
msg = msg % QPID_MESSAGING_URL
_logger.error(msg)
raise Exception(msg)
if not QPIDTOOLLIBS_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpidtoollibs is not '
'installed. Please install qpidtoollibs and rerun the migrations. See %s for more '
'information.')
msg = msg % QPIDTOOLLIBS_URL
_logger.error(msg)
raise Exception(msg)
url = urlparse(pulp_conf.get('messaging', 'url'))
connection = Connection(host=url.hostname,
port=url.port,
transport=url.scheme,
reconnect=False,
ssl_certfile=pulp_conf.get('messaging',
'clientcert'),
ssl_skip_hostname_check=True)
connection.attach()
broker = BrokerAgent(connection)
_migrate_reply_queue(broker)
_migrate_agent_queues(broker)
connection.detach()
def __init__(self, address, **kwargs):
self._connection = Connection.establish(
address, client_properties={"qpid.ha-admin": 1}, **kwargs)
self._agent = BrokerAgent(self._connection)
except ImportError:
print 'Cannot run test without python MagicMock'
print 'Please install MagicMock: pip install mock'
exit(3)
connection = None
broker = None
try:
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
logger = logging.getLogger(__name__)
# setup broker connection
connection = Connection.establish('127.0.0.1')
broker = BrokerAgent(connection)
# add test service busname
busname = 'test-lofarbus-%s' % (uuid.uuid1())
broker.addExchange('topic', busname)
# the system under test is the service and the rpc, not the RADatabase
# so, patch (mock) the RADatabase class during these tests.
# when the service instantiates an RADatabase it will get the mocked class.
with patch('lofar.sas.resourceassignment.database.radb.RADatabase',
autospec=True) as MockRADatabase:
mock = MockRADatabase.return_value
# modify the return values of the various RADatabase methods with pre-cooked answers
mock.getTaskStatuses.return_value = [{
'id': 1,
'name': 'opened'
def setup_access(self):
if 'broker_agent' not in self.__dict__:
self.conn2 = qpid.messaging.Connection(self.broker)
self.conn2.open()
self.broker_agent = BrokerAgent(self.conn2)
return self.broker_agent
def test_08_integration_test_with_messagebus(self):
""" Full blown integration test listening for notifications on the bus,
and checking which dir is up for a visit next.
Needs a working local qpid broker. Test is skipped if qpid not available.
"""
try:
broker = None
connection = None
import uuid
from threading import Event
from qpid.messaging import Connection, ConnectError
from qpidtoollibs import BrokerAgent
from lofar.messaging.messagebus import ToBus
from lofar.messaging.messages import EventMessage
from lofar.lta.ingest.common.config import DEFAULT_INGEST_NOTIFICATION_PREFIX
# setup broker connection
connection = Connection.establish('127.0.0.1')
broker = BrokerAgent(connection)
# add test service bus
busname = 'test-LTASOIngestEventHandler-%s' % (uuid.uuid1())
broker.addExchange('topic', busname)
sync_event = Event()
class SyncedLTASOIngestEventHandler(LTASOIngestEventHandler):
"""This derived LTASOIngestEventHandler behaves exactly like the normal
object under test LTASOIngestEventHandler, but it also sets a sync_event
to sync between the listener thread and this main test thread"""
def _handleMessage(self, msg):
super(SyncedLTASOIngestEventHandler,
self)._handleMessage(msg)
sync_event.set()
with SyncedLTASOIngestEventHandler(self.dbcreds, busname=busname):
for site in self.db.sites():
for root_dir in self.db.rootDirectoriesForSite(site['id']):
self._markAllDirectoriesRecentlyVisited()
# create the subdir surl
sub_dir_name = '/foo'
sub_dir_path = root_dir['dir_name'] + sub_dir_name
surl = site['url'] + sub_dir_path
with ToBus(busname) as sender:
msg = EventMessage(
subject=DEFAULT_INGEST_NOTIFICATION_PREFIX +
"TaskFinished",
content={'srm_url': surl})
sender.send(msg)
# wait for the handler to have processed the message
self.assertTrue(sync_event.wait(2))
sync_event.clear()
# surl should have been scheduled for a visit, all other dir's were marked as visited already...
# so there should be a new dir for this surl, and it should be the least_recent_visited_dir
site_visit_stats = self.db.visitStats(
datetime.utcnow())[site['name']]
least_recent_visited_dir_id = site_visit_stats.get(
'least_recent_visited_dir_id')
self.assertIsNotNone(least_recent_visited_dir_id)
least_recent_visited_dir = self.db.directory(
least_recent_visited_dir_id)
self.assertEqual(sub_dir_path,
least_recent_visited_dir['dir_name'])
except ImportError as e:
logger.warning("skipping test due to: %s", e)
except ConnectError as e:
logger.warning("skipping test due to: %s", e)
finally:
# cleanup test bus and exit
if broker:
broker.delExchange(busname)
if connection:
connection.close()
def __init__(self, address, **kwargs):
self._connection = Connection.establish(
address, client_properties={"qpid.ha-admin": 1}, **kwargs)
self._agent = BrokerAgent(self._connection)
assert self._agent.getHaBroker(
), "HA module not loaded in broker at: %s" % (address)
def setUp(self):
BrokerTest.setUp(self)
os.putenv("QPID_LOAD_MODULE", BrokerTest.amqpc_lib)
self.broker = self.amqp_broker()
self.default_config = Config(self.broker)
self.agent = BrokerAgent(self.broker.connect())
