def test_rmgr_base_populate(t, i):
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
sid = 'test.0012'
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
with pytest.raises(ree.EnTKError):
rm._populate()
rm._validate_resource_desc()
rm._populate()
res_dict = {
'resource': 'local.localhost',
'walltime': 40,
'cpus': 100,
'gpus': 25,
'project': 'new',
'queue': 'high',
'access_schema': 'gsissh'
}
rmgr_id = ru.generate_id('test.%(item_counter)04d', ru.ID_CUSTOM)
rmgr = BaseRmgr(res_dict, sid=rmgr_id, rts=None, rts_config={})
rmgr._validate_resource_desc()
rmgr._populate()
assert rmgr._sid == rmgr_id
assert rmgr._resource == 'local.localhost'
assert rmgr._walltime == 40
assert rmgr._cpus == 100
assert rmgr._gpus == 25
assert rmgr._project == 'new'
assert rmgr._access_schema == 'gsissh'
assert rmgr._queue == 'high'
assert rmgr._validated is True
def test_rmgr_base_assignment_exception(s, l, i, b, se):
data = [s, l, i, b, se]
for d in data:
with pytest.raises(TypeError):
rmgr = BaseRmgr(d, 'test.0000', None, {})
def test_tmgr_base_initialization(s, l, i):
try:
home = os.environ.get('HOME', '/home')
folder = glob.glob('%s/.radical/utils/test.*' % home)
for f in folder:
shutil.rmtree(f)
except:
pass
sid = 'test.0001'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
assert tmgr._uid == 'task_manager.0000'
assert tmgr._pending_queue == ['pending-1']
assert tmgr._completed_queue == ['completed-1']
assert tmgr._hostname == hostname
assert tmgr._port == port
assert tmgr._rts is None
assert tmgr._log
assert tmgr._prof
assert tmgr._hb_request_q == '%s-hb-request' % sid
assert tmgr._hb_response_q == '%s-hb-response' % sid
assert not tmgr._tmgr_process
assert not tmgr._hb_thread
def test_tmgr_base_assignment_exceptions(s, l, i, b, se, di):
sid = 'test.0000'
rmgr = BaseRmgr({}, sid, None, {})
data_type = [s, l, i, b, se, di]
for d in data_type:
if not isinstance(d, str):
with pytest.raises(TypeError):
tmgr = BaseTmgr(sid=s,
pending_queue=s,
completed_queue=s,
rmgr=rmgr,
mq_hostname=s,
port=d,
rts=None)
if not isinstance(d, int):
with pytest.raises(TypeError):
tmgr = BaseTmgr(sid=s,
pending_queue=s,
completed_queue=s,
rmgr=rmgr,
mq_hostname=s,
port=d,
rts=None)
def test_tmgr_base_heartbeat():
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0003'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._hb_terminate = mt.Event()
tmgr._hb_thread = mt.Thread(target=tmgr._heartbeat, name='heartbeat')
tmgr._hb_thread.start()
proc = mp.Process(target=func_for_heartbeat_test,
args=(hostname, port, username, password,
tmgr._hb_request_q, tmgr._hb_response_q))
proc.start()
proc.join()
tmgr._hb_thread.join()
def func(obj, obj_type, new_state, queue1):
hostname = os.environ.get('RMQ_HOSTNAME', 'localhost')
port = int(os.environ.get('RMQ_PORT', 5672))
username = os.environ.get('RMQ_USERNAME')
password = os.environ.get('RMQ_PASSWORD')
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0013'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
mq_connection = pika.BlockingConnection(rmq_conn_params)
mq_channel = mq_connection.channel()
tmgr._advance(obj, obj_type, new_state, mq_channel, queue1)
mq_connection.close()
def test_tmgr_base_assignment_exceptions(s, l, i, b, se, di):
sid = 'test.0002'
rmgr = BaseRmgr({}, sid, None, {})
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
data_type = [s, l, i, b, se, di]
for d in data_type:
if not isinstance(d, str):
with pytest.raises(ree.TypeError):
BaseTmgr(sid=s,
pending_queue=s,
completed_queue=s,
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
if not isinstance(d, int):
with pytest.raises(ree.TypeError):
BaseTmgr(sid=s,
pending_queue=s,
completed_queue=s,
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
def test_tmgr_base_methods():
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0009'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
with pytest.raises(NotImplementedError):
tmgr._tmgr(uid=None,
rmgr=None,
rmq_conn_params=None,
pending_queue=None,
completed_queue=None)
with pytest.raises(NotImplementedError):
tmgr.start_manager()
def test_rmgr_base_initialization(d):
try:
import glob
import shutil
import os
home = os.environ.get('HOME', '/home')
test_fold = glob.glob('%s/.radical/utils/test.*' % home)
for f in test_fold:
shutil.rmtree(f)
except:
pass
rmgr_id = ru.generate_id('test.%(item_counter)04d', ru.ID_CUSTOM)
rmgr = BaseRmgr(d, rmgr_id, None, {})
assert rmgr._resource_desc == d
assert rmgr._sid == rmgr_id
assert rmgr._rts == None
assert rmgr._rts_config == {}
assert rmgr.resource == None
assert rmgr.walltime == None
assert rmgr.cpus == 1
assert rmgr.gpus == 0
assert rmgr.project == None
assert rmgr.access_schema == None
assert rmgr.queue == None
assert rmgr._validated == False
assert rmgr._uid == 'resource_manager.0000'
assert rmgr._logger
assert rmgr._prof
# Shared data list
assert isinstance(rmgr.shared_data, list)
def test_tmgr_base_terminate_heartbeat():
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0005'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._hb_terminate = mt.Event()
assert tmgr.start_heartbeat()
assert not tmgr._hb_terminate.is_set()
assert tmgr._hb_thread.is_alive()
tmgr.terminate_heartbeat()
time.sleep(5)
assert not tmgr._hb_thread
assert tmgr._hb_terminate.is_set()
def test_tmgr_base_heartbeat():
sid = 'test.0000'
rmgr = BaseRmgr({}, sid, None, {})
os.environ['ENTK_HB_INTERVAL'] = '30'
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
tmgr._hb_terminate = threading.Event()
tmgr._hb_thread = threading.Thread(target=tmgr._heartbeat,
name='heartbeat')
tmgr._hb_thread.start()
proc = Process(target=func_for_heartbeat_test,
args=(hostname, port, tmgr._hb_request_q,
tmgr._hb_response_q))
proc.start()
proc.join()
tmgr._hb_thread.join()
def test_amgr_synchronizer():
amgr = Amgr(hostname=host, port=port, username=username, password=password)
amgr._setup_mqs()
p = Pipeline()
s = Stage()
# Create and add 10 tasks to the stage
for cnt in range(10):
t = Task()
t.executable = 'some-executable-%s' % cnt
s.add_tasks(t)
p.add_stages(s)
p._validate()
amgr.workflow = [p]
sid = 'test.0016'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=amgr._rmq_conn_params,
rts=None)
amgr._rmgr = rmgr
rmgr._task_manager = tmgr
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
# Start the synchronizer method in a thread
amgr._terminate_sync = mt.Event()
sync_thread = mt.Thread(target=amgr._synchronizer,
name='synchronizer-thread')
sync_thread.start()
# Start the synchronizer method in a thread
proc = mp.Process(target=func_for_synchronizer_test, name='temp-proc',
args=(amgr._sid, p, tmgr))
proc.start()
proc.join()
# Wait for AppManager to finish the message exchange
# no need to set *)terminate_sync* but a timeout instead
# amgr._terminate_sync.set()
sync_thread.join(15)
for t in p.stages[0].tasks:
assert t.state == states.COMPLETED
def test_tmgr_base_check_manager():
sid = 'test.0000'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
tmgr._tmgr_process = Process(target=tmgr._tmgr, name='heartbeat')
tmgr._tmgr_terminate = Event()
tmgr._tmgr_process.start()
assert tmgr.check_manager()
tmgr.terminate_manager()
def test_tmgr_base_check_heartbeat():
rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port)
sid = 'test.0007'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._hb_thread = mt.Thread(target=tmgr._heartbeat, name='heartbeat')
tmgr._hb_terminate = mt.Event()
tmgr._hb_thread.start()
assert tmgr.check_heartbeat()
tmgr.terminate_heartbeat()
def test_tmgr_base_check_manager():
rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port)
sid = 'test.0008'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._tmgr_process = mp.Process(target=tmgr._tmgr, name='heartbeat')
tmgr._tmgr_terminate = mp.Event()
tmgr._tmgr_process.start()
assert tmgr.check_manager()
tmgr.terminate_manager()
def test_tmgr_base_check_heartbeat():
sid = 'test.0007'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
tmgr._hb_thread = mt.Thread(target=tmgr._heartbeat, name='heartbeat')
tmgr._hb_terminate = mt.Event()
tmgr._hb_thread.start()
assert tmgr.check_heartbeat()
tmgr.terminate_heartbeat()
def test_tmgr_base_check_heartbeat():
sid = 'test.0000'
rmgr = BaseRmgr({}, sid, None, {})
os.environ['ENTK_HB_INTERVAL'] = '30'
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
tmgr._hb_thread = threading.Thread(target=tmgr._heartbeat, name='heartbeat')
tmgr._hb_terminate = threading.Event()
tmgr._hb_thread.start()
assert tmgr.check_heartbeat()
tmgr.terminate_heartbeat()
def syncer(obj, obj_type, queue1):
hostname = os.environ.get('RMQ_HOSTNAME', 'localhost')
port = int(os.environ.get('RMQ_PORT', 5672))
rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port)
mq_connection = pika.BlockingConnection(rmq_conn_params)
mq_channel = mq_connection.channel()
sid = 'test.0015'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._sync_with_master(obj, obj_type, mq_channel, queue1)
mq_connection.close()
def test_tmgr_base_start_heartbeat():
sid = 'test.0004'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
assert tmgr.start_heartbeat()
assert not tmgr._hb_terminate.is_set()
assert tmgr._hb_thread.is_alive()
tmgr._hb_terminate.set()
time.sleep(15)
tmgr._hb_thread.join()
assert not tmgr._hb_thread.is_alive()
def test_tmgr_base_start_heartbeat():
rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port)
sid = 'test.0004'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
assert tmgr.start_heartbeat()
assert not tmgr._hb_terminate.is_set()
assert tmgr._hb_thread.is_alive()
tmgr._hb_terminate.set()
time.sleep(15)
tmgr._hb_thread.join()
assert not tmgr._hb_thread.is_alive()
def func(obj, obj_type, new_state, queue1):
hostname = os.environ.get('RMQ_HOSTNAME', 'localhost')
port = int(os.environ.get('RMQ_PORT', 5672))
sid = 'test.0013'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(
host=hostname, port=port))
mq_channel = mq_connection.channel()
tmgr._advance(obj, obj_type, new_state, mq_channel, queue1)
mq_connection.close()
def test_tmgr_base_methods():
sid = 'test.0009'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
with pytest.raises(NotImplementedError):
tmgr._tmgr(uid=None,
rmgr=None,
mq_hostname=None,
port=None,
pending_queue=None,
completed_queue=None)
with pytest.raises(NotImplementedError):
tmgr.start_manager()
def test_tmgr_base_terminate_manager():
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0006'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
tmgr._tmgr_process = mp.Process(target=tmgr._tmgr, name='heartbeat')
tmgr._tmgr_terminate = mp.Event()
tmgr.terminate_manager()
assert not tmgr._tmgr_process
assert tmgr._tmgr_terminate.is_set()
def test_tmgr_base_terminate_heartbeat():
sid = 'test.0000'
rmgr = BaseRmgr({}, sid, None, {})
os.environ['ENTK_HB_INTERVAL'] = '30'
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
mq_hostname=hostname,
port=port,
rts=None)
tmgr._hb_terminate = threading.Event()
assert tmgr.start_heartbeat()
assert not tmgr._hb_terminate.is_set()
assert tmgr._hb_thread.is_alive()
tmgr.terminate_heartbeat()
sleep(30)
assert not tmgr._hb_thread
assert tmgr._hb_terminate.is_set()
def test_rmgr_base_initialization(d):
try:
home = os.environ.get('HOME', '/home')
folder = glob.glob('%s/.radical/utils/test.*' % home)
for f in folder:
shutil.rmtree(f)
except:
pass
rmgr_id = ru.generate_id('test.%(item_counter)04d', ru.ID_CUSTOM)
rmgr = BaseRmgr(d, rmgr_id, None, {})
assert rmgr._resource_desc == d
assert rmgr._sid == rmgr_id
assert rmgr._rts is None
assert rmgr._rts_config == {}
assert rmgr.resource is None
assert rmgr.walltime is None
assert rmgr.cpus == 1
assert rmgr.gpus == 0
assert rmgr.project is None
assert rmgr.access_schema is None
assert rmgr.queue is None
assert rmgr._validated is False
# rmgr id is incremental, and it is valid as long as it is in the range
prefix, uid = rmgr._uid.split(".")
assert prefix == 'resource_manager'
assert int(uid) >= 0
assert int(uid) <= 9999
assert rmgr._logger
assert rmgr._prof
assert isinstance(rmgr.shared_data, list)
def test_tmgr_base_initialization():
try:
home = os.environ.get('HOME', '/home')
folder = glob.glob('%s/.radical/utils/test.*' % home)
for f in folder:
shutil.rmtree(f)
except:
pass
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
sid = 'test.0001'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=rmq_conn_params,
rts=None)
assert tmgr._uid == 'task_manager.0000'
assert tmgr._pending_queue == ['pending-1']
assert tmgr._completed_queue == ['completed-1']
assert tmgr._rmq_conn_params == rmq_conn_params
assert tmgr._rts is None
assert tmgr._log
assert tmgr._prof
assert tmgr._hb_request_q == '%s-hb-request' % sid
assert tmgr._hb_response_q == '%s-hb-response' % sid
assert not tmgr._tmgr_process
assert not tmgr._hb_thread
def test_rmgr_base_terminate_resource_request():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr._terminate_resource_request()
def test_rmgr_base_get_resource_allocation_state():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_resource_allocation_state()
def test_rmgr_base_validate_resource_desc(t, i):
rmgr = BaseRmgr({}, sid='test.0000', rts=None, rts_config={})
with pytest.raises(MissingError):
rmgr._validate_resource_desc()
sid = 'test.0000'
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 20,
}
with pytest.raises(TypeError):
res_dict = {
'resource': i,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
assert rm._validate_resource_desc()
def test_rmgr_base_completed_states():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_completed_states()