def test_amgr_synchronizer():
amgr = Amgr(hostname=host, port=port, username=username, password=password)
amgr._setup_mqs()
p = Pipeline()
s = Stage()
# Create and add 10 tasks to the stage
for cnt in range(10):
t = Task()
t.executable = 'some-executable-%s' % cnt
s.add_tasks(t)
p.add_stages(s)
p._validate()
amgr.workflow = [p]
sid = 'test.0016'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=amgr._rmq_conn_params,
rts=None)
amgr._rmgr = rmgr
rmgr._task_manager = tmgr
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
# Start the synchronizer method in a thread
amgr._terminate_sync = mt.Event()
sync_thread = mt.Thread(target=amgr._synchronizer,
name='synchronizer-thread')
sync_thread.start()
# Start the synchronizer method in a thread
proc = mp.Process(target=func_for_synchronizer_test, name='temp-proc',
args=(amgr._sid, p, tmgr))
proc.start()
proc.join()
# Wait for AppManager to finish the message exchange
# no need to set *)terminate_sync* but a timeout instead
# amgr._terminate_sync.set()
sync_thread.join(15)
for t in p.stages[0].tasks:
assert t.state == states.COMPLETED
def test_rmgr_base_validate_resource_desc(t, i):
rmgr = BaseRmgr({}, sid='test.0000', rts=None, rts_config={})
with pytest.raises(MissingError):
rmgr._validate_resource_desc()
sid = 'test.0000'
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 20,
}
with pytest.raises(TypeError):
res_dict = {
'resource': i,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
assert rm._validate_resource_desc()
def test_rmgr_base_completed_states():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_completed_states()
def test_rmgr_base_get_resource_allocation_state():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_resource_allocation_state()
def test_rmgr_base_terminate_resource_request():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr._terminate_resource_request()
def test_rmgr_base_populate(t, i):
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
with pytest.raises(EnTKError):
rm._populate()
rm._validate_resource_desc()
rm._populate()
res_dict = {
'resource': 'local.localhost',
'walltime': 40,
'cpus': 100,
'gpus': 25,
'project': 'new',
'queue': 'high',
'access_schema': 'gsissh'
}
rmgr_id = ru.generate_id('test.%(item_counter)04d', ru.ID_CUSTOM)
rmgr = BaseRmgr(res_dict, sid=rmgr_id, rts=None, rts_config={})
rmgr._validate_resource_desc()
rmgr._populate()
assert rmgr._sid == rmgr_id
assert rmgr._resource == 'local.localhost'
assert rmgr._walltime == 40
assert rmgr._cpus == 100
assert rmgr._gpus == 25
assert rmgr._project == 'new'
assert rmgr._access_schema == 'gsissh'
assert rmgr._queue == 'high'
assert rmgr._validated == True
def test_rmgr_base_submit_resource_request():
rmgr = BaseRmgr({}, 'test.0023', None, {})
with pytest.raises(NotImplementedError):
rmgr._submit_resource_request()
def test_validate(self, mocked_init, mocked_Logger, mocked_Profiler,
res_descr):
rmgr = Rmgr({'resource':'localhost'}, 'test_rmgr', 'rp', 'test_config')
rmgr._resource_desc = res_descr
rmgr._rts_config = {'rts': 'some_rts'}
rmgr._logger = mocked_Logger
rmgr._prof = mocked_Profiler
rmgr._uid = 'rmgr.0000'
self.assertTrue(rmgr._validate_resource_desc())
self.assertTrue(rmgr._validated)
rmgr._resource_desc = {'resource' : 'localhost',
'walltime' : 30,
'cpus' : 1,
'gpus' : 1,
'project' : 'some_project',
'access_schema' : 'access',
'queue' : 'queue'}
rmgr._rts_config = None
with self.assertRaises(TypeError):
rmgr._validate_resource_desc()
def test_populate(self, mocked_init, mocked_Logger, mocked_Profiler,
res_descr):
rmgr = Rmgr({'resource':'localhost'}, 'test_rmgr', 'rp', 'test_config')
rmgr._validated = False
rmgr._resource_desc = res_descr
rmgr._logger = mocked_Logger
rmgr._prof = mocked_Profiler
rmgr._uid = 'rmgr.0000'
with self.assertRaises(EnTKError):
rmgr._populate()
rmgr._validated = True
rmgr._resource_desc = res_descr
rmgr._logger = mocked_Logger
rmgr._prof = mocked_Profiler
rmgr._uid = 'rmgr.0000'
rmgr._populate()
self.assertEqual(rmgr._resource, res_descr['resource'])
self.assertEqual(rmgr._walltime, res_descr['walltime'])
self.assertEqual(rmgr._cpus, res_descr['cpus'])
self.assertEqual(rmgr._gpus, res_descr['gpus'])
self.assertEqual(rmgr._project, res_descr['project'])
self.assertEqual(rmgr._access_schema, res_descr['access_schema'])
self.assertEqual(rmgr._queue, res_descr['queue'])
def test_rmgr_base_validate_resource_desc(t, i):
rmgr = BaseRmgr({}, sid='test.0000', rts=None, rts_config={})
with pytest.raises(MissingError):
rmgr._validate_resource_desc()
sid = 'test.0000'
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 20,
}
with pytest.raises(TypeError):
res_dict = {
'resource': i,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': t,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': t,
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
with pytest.raises(TypeError):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': i
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
rm._validate_resource_desc()
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
assert rm._validate_resource_desc()
def test_rmgr_base_completed_states():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_completed_states()
def test_rmgr_base_get_resource_allocation_state():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr.get_resource_allocation_state()
def test_rmgr_base_terminate_resource_request():
rmgr = BaseRmgr({}, 'test.0000', None, {})
with pytest.raises(NotImplementedError):
rmgr._terminate_resource_request()
def test_rmgr_base_populate(t, i):
if isinstance(t, str):
res_dict = {
'resource': t,
'walltime': i,
'cpus': i,
'gpus': i,
'project': t,
'access_schema': t,
'queue': t
}
rm = BaseRmgr(res_dict, sid=sid, rts=None, rts_config={})
with pytest.raises(EnTKError):
rm._populate()
rm._validate_resource_desc()
rm._populate()
res_dict = {
'resource': 'local.localhost',
'walltime': 40,
'cpus': 100,
'gpus': 25,
'project': 'new',
'queue': 'high',
'access_schema': 'gsissh'
}
rmgr_id = ru.generate_id('test.%(item_counter)04d', ru.ID_CUSTOM)
rmgr = BaseRmgr(res_dict, sid=rmgr_id, rts=None, rts_config={})
rmgr._validate_resource_desc()
rmgr._populate()
assert rmgr._sid == rmgr_id
assert rmgr._resource == 'local.localhost'
assert rmgr._walltime == 40
assert rmgr._cpus == 100
assert rmgr._gpus == 25
assert rmgr._project == 'new'
assert rmgr._access_schema == 'gsissh'
assert rmgr._queue == 'high'
assert rmgr._validated == True
