def test_runtime_normal_uid(rp_task_manager, pilot_description):
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.task_manager')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.db.database')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.session')
session: rp.Session = rp_task_manager.session
state = Runtime(session=session)
state.task_manager(rp_task_manager.uid)
# We only expect one pilot
pilot: rp.Pilot = rp_task_manager.get_pilots()[0]
# We get a dictionary...
# assert isinstance(pilot, rp.Pilot)
# But it looks like it has the pilot id in it.
pilot_uid = typing.cast(dict, pilot)['uid']
# It is an error to set a Pilot before the PilotManager has been set.
with pytest.raises(APIError):
state.pilot(pilot_uid)
pmgr_uid = typing.cast(dict, pilot)['pmgr']
state.pilot_manager(pmgr_uid)
state.pilot(pilot_uid)
def _check_pilot(runtime: Runtime, pilot_description: rp.PilotDescription,
venv):
pilot_manager = runtime.pilot_manager()
pilot = runtime.pilot()
_check_pilot_manager(runtime=runtime)
if runtime.pilot_manager() is not pilot_manager \
or pilot is None \
or pilot.state in rp.FINAL:
if runtime.pilot_manager() is not pilot_manager:
logger.info('PilotManager refreshed. Now refreshing Pilot.')
assert pilot is None or isinstance(
pilot, rp.Pilot) and pilot.state in rp.FINAL
if pilot is None:
logger.info(f'Creating a Pilot for {runtime.session.uid}')
if isinstance(pilot, rp.Pilot):
if pilot.state in rp.FINAL:
logger.info(f'Old Pilot is {pilot.state}')
else:
logger.warning(
f'Canceling old pilot {pilot.uid}, which should have '
'already been canceled.')
pilot.cancel()
pilot = _new_pilot(session=runtime.session,
pilot_manager=runtime.pilot_manager(),
pilot_description=pilot_description,
venv=venv)
logger.info(f'New Pilot is {pilot.uid}')
runtime.pilot(pilot)
else:
assert pilot is runtime.pilot()
def test_runtime_normal_instance(rp_task_manager, pilot_description):
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.task_manager')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.db.database')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.session')
session: rp.Session = rp_task_manager.session
state = Runtime(session=session)
state.task_manager(rp_task_manager)
# We only expect one pilot
pilot: rp.Pilot = rp_task_manager.get_pilots()[0]
# We get a dictionary...
# assert isinstance(pilot, rp.Pilot)
# But it looks like it has the pilot id in it.
pilot_uid = typing.cast(dict, pilot)['uid']
pmgr_uid = typing.cast(dict, pilot)['pmgr']
pmgr: rp.PilotManager = session.get_pilot_managers(pmgr_uids=pmgr_uid)
assert isinstance(pmgr, rp.PilotManager)
state.pilot_manager(pmgr)
pilot = pmgr.get_pilots(uids=pilot_uid)
assert isinstance(pilot, rp.Pilot)
state.pilot(pilot)
def rp_pilot(rp_runtime: Runtime, rp_pilot_manager: rp.PilotManager,
pilot_description: rp.PilotDescription, rp_venv):
_check_pilot(runtime=rp_runtime,
pilot_description=pilot_description,
venv=rp_venv)
pilot = rp_runtime.pilot()
assert pilot is not None
assert pilot.state not in rp.FINAL
yield pilot
def _check_pilot_manager(runtime: Runtime):
# Caller should destroy and recreate Pilot if this call has to replace PilotManager.
session = runtime.session
original_pilot_manager: rp.PilotManager = runtime.pilot_manager()
if session.closed:
logger.info(f'{session.uid} is closed. Creating new Session.')
session: rp.Session = _new_session()
runtime.reset(session)
if original_pilot_manager is None or original_pilot_manager is not runtime.pilot_manager(
):
# Is there a way to check whether the PilotManager is healthy?
logger.info(f'Creating a new PilotManager for {runtime.session.uid}')
if isinstance(original_pilot_manager, rp.PilotManager):
logger.info('Closing old PilotManager')
original_pilot_manager.close()
pilot_manager = _new_pilotmanager(runtime.session)
logger.info(f'New PilotManager is {pilot_manager.uid}')
runtime.pilot_manager(pilot_manager)
def _check_task_manager(runtime: Runtime,
pilot_description: rp.PilotDescription, venv):
task_manager = runtime.task_manager()
original_pilot = runtime.pilot()
_check_pilot(runtime=runtime,
pilot_description=pilot_description,
venv=venv)
pilot = runtime.pilot()
if task_manager is None or pilot is not original_pilot:
if pilot is not original_pilot and original_pilot is not None:
logger.info(
'Pilot has changed. Creating and binding a new TaskManager.')
if task_manager is not None:
assert isinstance(task_manager, rp.TaskManager)
logger.info('Closing old TaskManager.')
task_manager.close()
logger.info(f'Creating new TaskManager for {runtime.session.uid}')
task_manager = _new_taskmanager(session=runtime.session, pilot=pilot)
if runtime.task_manager() is not task_manager:
runtime.task_manager(task_manager)
def rp_task_manager(rp_runtime: Runtime, pilot_description, rp_venv):
_check_task_manager(runtime=rp_runtime,
pilot_description=pilot_description,
venv=rp_venv)
task_manager: rp.TaskManager = rp_runtime.task_manager()
yield task_manager
def rp_pilot_manager(rp_runtime: Runtime):
_check_pilot_manager(runtime=rp_runtime)
yield rp_runtime.pilot_manager()
def test_runtime_bad_uid(pilot_description):
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.task_manager')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.db.database')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.session')
session = rp.Session()
with session:
state = Runtime(session=session)
with pytest.raises(ValueError):
state.task_manager('spam')
tmgr = rp.TaskManager(session=session)
state.task_manager(tmgr)
with pytest.raises(ValueError):
state.pilot_manager('spam')
pmgr = rp.PilotManager(session=session)
state.pilot_manager(pmgr)
with pytest.raises(ValueError):
state.pilot_manager('spam')
tmgr.close()
pmgr.close()
assert session.closed
def test_runtime_mismatch(pilot_description):
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.task_manager')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.db.database')
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='radical.pilot.session')
session = rp.Session()
with session:
original_pmgr = rp.PilotManager(session=session)
pilot = original_pmgr.submit_pilots(rp.PilotDescription(pilot_description))
original_tmgr = rp.TaskManager(session=session)
original_tmgr.add_pilots(pilot)
assert session.closed
# This assertion may not be true:
# assert pilot.state in rp.FINAL
# Note that Pilot and other components may still be shutting down, but the
# intention is that, from this point, pmgr, pilot, and tmgr are now "stale".
session = rp.Session()
with session:
state = Runtime(session=session)
with pytest.raises(APIError):
state.task_manager(original_tmgr)
original_tmgr.close()
tmgr = rp.TaskManager(session=session)
state.task_manager(tmgr)
with pytest.raises(APIError):
state.pilot_manager(original_pmgr)
original_pmgr.close()
pmgr = rp.PilotManager(session=session)
state.pilot_manager(pmgr)
# The UID will not resolve in the stored PilotManager.
with pytest.raises(ValueError):
state.pilot(pilot.uid)
# The Pilot is detectably invalid.
with pytest.raises(APIError):
state.pilot(pilot)
# Even here, the old Pilot may still be in 'PMGR_ACTIVE_PENDING'
if pilot.state not in rp.FINAL:
pilot.cancel()
tmgr.close()
pmgr.close()
assert session.closed