class Client(object):
"""
The Client object is instantiated by the AdaptiveMD master in order to invoke components
on the runtime system (RTS) side. These components then interact with the MongoDB to extract
task, resource and configuration descriptions. Using these descriptions, RTS specific
components are created and executed.
:arguments:
:dburl: MongoDB URL to be used for RADICAL Pilot
:project: Store name to be used on MongoDB. This is the store where all the configurations, resources and task
descriptions are present.
"""
def __init__(self, dburl, project):
self._uid = ru.generate_id('client.rp')
self._logger = ru.get_logger('client.rp')
self._dburl = dburl
self._project = project
#self._cb_buffer = CB_Buffer()
self._cb_manager = Manager()
self._cb_buffer = self._cb_manager.list()
# Process related data
self._proc = None
self._cb_proc = None
self._terminate = None
self._rmgr = None
self._tmgr = None
# ------------------------------------------------------------------------------------------------------------------
# Private methods
# ------------------------------------------------------------------------------------------------------------------
def _cb_watcher(self, cb_buffer):
self._db = Database(self._dburl, self._project)
# Asynchronous task updates can lead to running
# state after CU cancellation, any cus in this
# list will recieve no further state updates
self._cancelled_cus = list()
try:
while not self._terminate.is_set():
updates = self._cb_check(cb_buffer)
if not updates:
sleep(5)
except Exception as ex:
print(traceback.format_exc())
#pass
finally:
if cb_buffer:
self._logger.warning(
'Client process ended before callbacks cleared:\n%s' %
pformat(cb_buffer))
# TODO utilize the 'xxx_many' methods
# in pymongo (requires downstream
# changes and lumping 'updates')
def _cb_check(self, cb_buffer):
updates = False
while self._cb_buffer:
updates = True
cb_dct = self._cb_buffer.pop()
for uid, operations in cb_dct.items():
if uid not in self._cancelled_cus:
for col, updates in operations.items():
while updates:
update = updates.pop()
if col == 'tasks':
self._db.update_task_description_status(
uid, update)
if 'cancelled' in update:
self._cancelled_cus.append(uid)
elif col == 'files':
if update == 'create':
self._db.file_created(uid)
elif update == 'remove':
self._db.file_removed(uid)
return updates
def _get_resource_desc_for_pilot(self, processed_configs,
processed_resource_reqs):
selected_resources = list()
for resource_reqs in processed_resource_reqs:
resource_name = resource_reqs['resource']
matching_configs = get_matching_configurations(
configurations=processed_configs, resource_name=resource_name)
for matched_configs in matching_configs:
selected_resource = dict()
selected_resource['resource'] = str(
matched_configs['resource'])
selected_resource['runtime'] = resource_reqs['total_time']
selected_resource['cores'] = resource_reqs['total_cpus']
selected_resource['gpus'] = resource_reqs['total_gpus']
selected_resource['project'] = str(matched_configs['project'])
selected_resource['queue'] = str(matched_configs['queue'])
selected_resource['shared_path'] = str(
matched_configs.get('shared_path', '$HOME'))
selected_resources.append(selected_resource)
# The length of matching_configs is the number of pilots we will launch. Currently, simply return the first
# one.
return selected_resources[0]
def _runme(self, cb_buffer):
"""
We run the RP methods in a separate process because although the master process starts these methods,
we don't want to block the master process.
NOTE: DO NOT MOVE RP components outside this process. We need to keep them all inside one process since
they processes will create new copies. Not to be run in separate threads since RP components are not
thread-safe.
"""
try:
self._db = Database(self._dburl, self._project)
raw_resource_reqs = self._db.get_resource_requirements()
processed_resource_reqs = process_resource_requirements(
raw_resource_reqs)
raw_configurations = self._db.get_configurations()
processed_configurations = process_configurations(
raw_configurations)
resource_desc_for_pilot = self._get_resource_desc_for_pilot(
processed_configurations, processed_resource_reqs)
if len(resource_desc_for_pilot) > 0:
self._rmgr = ResourceManager(
resource_desc=resource_desc_for_pilot, db=self._db)
self._rmgr.submit_resource_request()
self._tmgr = TaskManager(
session=self._rmgr.session,
db_obj=self._db,
cb_buffer=cb_buffer,
) #scheduler='continuous')#'hombre')#scheduler)
while not self._terminate.is_set():
task_descs = self._db.get_task_descriptions()
self._tmgr.cancel_stalled_tasks()
if task_descs:
cuds = create_cud_from_task_def(
task_descs, self._db,
resource_desc_for_pilot['shared_path'],
self._project)
self._tmgr.run_cuds(cuds)
else:
sleep(3)
else:
raise Error(
msg=
"No matching resource found in configuration file. Please check your configuration file and the resource object."
)
except Exception as ex:
self._logger.error('Client process failed, error: %s' % ex)
print(traceback.format_exc())
raise Error(msg=ex)
finally:
if self._tmgr:
#print("CANCELLING TMGR checker")
self._tmgr.stop_checker()
if self._rmgr:
# TODO what to do when _rmgs.pilot is None (ie pilot fails but we get here)?
self._rmgr.pilot.cancel()
self._rmgr.session.close(download=True, cleanup=False)
# ------------------------------------------------------------------------------------------------------------------
# Public methods
# ------------------------------------------------------------------------------------------------------------------
@property
def running(self):
'''
Check if there is a running RP Client Process
'''
if not self._proc:
return False
else:
return self._proc.is_alive()
def start(self):
"""
Method exposed to the master process to start all the RP components and processes. This is a non-blocking command.
"""
try:
self._proc = Process(target=self._runme, args=(self._cb_buffer, ))
self._terminate = Event()
self._proc.start()
self._cb_proc = Process(target=self._cb_watcher,
args=(self._cb_buffer, ))
self._cb_proc.start()
except Exception as ex:
self._logger.error("Error starting RP process, error: %s" % ex)
self.end()
raise Error(msg=ex)
def stop(self):
"""
Method exposed to the master process to end all the RP components and processes. This is a blocking command.
"""
try:
if self._proc.is_alive():
if not self._terminate.is_set():
self._terminate.set()
self._cb_proc.join()
self._proc.join()
except Exception as ex:
self._logger.error("Error stopping RP process, error: %s" % ex)
raise Error(msg=ex)