def __init__(self):
self._channel = None
self._uida_conf_vars = load_variables_from_uida_conf_files()
self._settings = get_settings(self._uida_conf_vars)
self._ida_api_url = self._uida_conf_vars['IDA_API_ROOT_URL']
self._hostname = socket.gethostname()
self._machine_name = self._hostname.split('.')[0]
self._process_pid = os.getpid()
self._sentinel_monitoring_file = '%s/%s-%s-%d' % (
self._uida_conf_vars['RABBIT_MONITORING_DIR'], self._hostname,
self.__class__.__name__, self._process_pid)
self.name = self._get_name() # name of the agent, displayed in logs
self.main_queue_name = None # the queue with original, new actions
self.failed_queue_name = None # the queue with republished, failed actions
self.rabbitmq_message = None # the currently being processed message from a queue
self._graceful_shutdown_started = False
self.gevent = None
# diagnostic variables for development and testing
self.last_completed_sub_action = {}
self.last_failed_action = {}
self.last_updated_action = {}
self._logger = get_logger(self.name, self._uida_conf_vars)
self.connect()
self._cleanup_old_sentinel_monitoring_files()
# on process close, try to remove sentinel monitoring files
signal.signal(signal.SIGTERM,
lambda signal, frame: self._signal_shutdown_started())
signal.signal(signal.SIGINT,
lambda signal, frame: self._signal_shutdown_started())
rabbitmq postprocessing agents require to operate.
Before executing the script, ensure:
- the rabbitmq management plugin is enabled
- rabbitmq admin user has been configured by using script utils/rabbitmq_create_users
- rabbitmq admin credentials are in place in config/config.sh
"""
from json import dumps as json_dumps
from sys import argv
import requests
from agents.utils.utils import get_settings, load_variables_from_uida_conf_files, executing_test_case
uida_conf_vars = load_variables_from_uida_conf_files()
settings = get_settings()
RABBITMQ_API_URL = 'http://%s:%d/api' % (uida_conf_vars['RABBIT_HOST'], uida_conf_vars['RABBIT_WEB_API_PORT'])
RABBITMQ_AUTH = (uida_conf_vars['RABBIT_ADMIN_USER'], uida_conf_vars['RABBIT_ADMIN_PASS'])
HEADERS = { 'Content-Type': 'application/json'}
VHOST_NAME = uida_conf_vars['RABBIT_VHOST']
EXCHANGES = [
{
'name': 'actions',
'type': 'fanout',
'arguments': {},
class BaseAgentTestCase(TestCase):
_settings = get_settings()
_uida_conf_vars = load_variables_from_uida_conf_files()
@classmethod
def setUpClass(cls):
# instead of setting a signal handler for ctrl+c,
# call teardown class to make sure everything is clean
try:
cls.tearDownClass()
except Exception:
pass
def setUp(self):
test_utils.init_rabbitmq()
self._init_files()
self._prepare_api_responses()
def tearDown(self):
test_utils.teardown_rabbitmq()
self._teardown_files()
ok = self.currentResult.wasSuccessful()
# errors = self.currentResult.errors
# failures = self.currentResult.failures
#print("Test Successful" if ok else "%d Errors and %d Failures found" % (len(errors), len(failures)))
print('\033[92m\033[1m' + "\t ....OK" + '\033[0m' if ok else '\033[91m\033[1m' + "\t ....FAIL" + '\033[0m')
def run(self, result=None):
self.currentResult = result # remember result for use in tearDown
TestCase.run(self, result) # call superclass run method
@classmethod
def tearDownClass(cls):
test_utils.teardown_rabbitmq()
cls._teardown_files(cls)
def _publish_test_messages(self, index=None, exchange='actions'):
"""
Publish one message from index 'index' from test data, or publish all messages if
no index is passed.
"""
message = None
ga = GenericAgent()
if index is not None:
message = ida_test_data['actions'][index]
ga.publish_message(message, exchange=exchange)
else:
for action in ida_test_data['actions']:
ga.publish_message(action, exchange=exchange)
# must give some time for messages to get processed in rabbitmq,
# so that the agents and the asserts can detect the new messages
sleep(0.5)
return message
def _prepare_response(self, method, service, url, status=None, body=None):
"""
Prepare responses that will be returned by the mock apis that are called
by the agents during tests, i.e. the response generated here would be what
the ida-api and metax-api would be expected to return.
"""
if service == 'ida':
root_url = self._uida_conf_vars['IDA_API_ROOT_URL']
elif service == 'metax':
root_url = self._uida_conf_vars['METAX_API_ROOT_URL']
else:
raise Exception('oops! check your settings')
responses.add(
method,
'%s%s' % (root_url, url),
status=status,
content_type='application/json',
body=body
)
def _prepare_api_responses(self):
"""
Generate mock responses made to...
- ida api using test data. The testdata file is basically a mocked database.
- metax api. all requests are by default assumed successful. tests which test
against a failing metax action will override the mock response
"""
# GET action
for action in ida_test_data['actions']:
self._prepare_response(
responses.GET,
'ida',
'/actions/%s' % action['pid'],
status=200,
body=json_dumps(action)
)
# POST action
for action in ida_test_data['actions']:
self._prepare_response(
responses.POST,
'ida',
'/actions/%s' % action['pid'],
status=204
)
# GET nodes associated with action
for action in ida_test_data['actions']:
self._prepare_response(
responses.GET,
'ida',
'/files/action/%s' % action['pid'],
status=200,
body=json_dumps(
[ node for node in ida_test_data['nodes'] if node['action'] == action['pid'] ]
)
)
# POST nodes
for node in ida_test_data['nodes']:
self._prepare_response(
responses.POST,
'ida',
'/files/%s' % node['pid'],
status=204
)
# POST to metax /files
self._prepare_response(
responses.POST,
'metax',
'/files?atomic=true',
status=200,
body=json_dumps({
'success': [
{ 'object': 'a successfully created object' }
],
'failed': []
})
)
# DELETE to metax /files
for node in ida_test_data['nodes']:
self._prepare_response(
responses.DELETE,
'metax',
'/files',
status=204
)
def _init_files(self):
"""
Create files to the file system from test data
"""
for f in (f for f in ida_test_data['nodes'] if f['type'] == 'file'):
test_utils.create_test_file(self._uida_conf_vars, f)
def _teardown_files(self):
test_utils.delete_test_directory(self._uida_conf_vars['STORAGE_OC_DATA_ROOT'])
test_utils.delete_test_directory(self._uida_conf_vars['DATA_REPLICATION_ROOT'])
test_utils.delete_test_directory(self._uida_conf_vars['RABBIT_MONITORING_DIR'])
def assert_messages_ended_in_failed_queue(self, message_qty):
"""
At the end of a test it is useful to check how many messages being processed
eneded up in the failed queue.
"""
self.assertEqual(self.agent.messages_in_queue(self.agent.failed_queue_name), message_qty)
