class GCThreadPool(object):
def __init__(self):
self._lock = GCLock()
self._notify = GCEvent()
self._token = 0
self._token_time = {}
self._token_desc = {}
self._log = logging.getLogger('thread_pool')
self._ex_collector = ExceptionCollector(self._log)
def wait_and_drop(self, timeout = None):
while True:
self._lock.acquire()
try:
t_current = time.time()
# discard stale threads
for token in list(self._token_time):
if timeout and (t_current - self._token_time.get(token, 0) > timeout):
self._token_time.pop(token, None)
self._token_desc.pop(token, None)
if not self._token_time: # no active threads
return True
# drop all threads if timeout is reached
if (timeout is not None) and (timeout <= 0):
self._token_time = {}
self._token_desc = {}
return False
finally:
self._lock.release()
# wait for thread to finish and adapt timeout for next round
self._notify.wait(timeout)
if timeout is not None:
timeout -= time.time() - t_current
def start_thread(self, desc, fun, *args, **kwargs):
self._lock.acquire()
try:
self._token += 1
self._token_time[self._token] = time.time()
self._token_desc[self._token] = desc
finally:
self._lock.release()
start_thread(desc, self._run_thread, self._token, fun, args, kwargs)
def _run_thread(self, token, fun, args, kwargs):
try:
fun(*args, **kwargs)
except Exception:
self._lock.acquire()
try:
self._ex_collector.collect(logging.ERROR, 'Exception in thread %r', self._token_desc[token], exc_info = get_current_exception())
finally:
self._lock.release()
self._lock.acquire()
try:
self._token_time.pop(token, None)
self._token_desc.pop(token, None)
finally:
self._lock.release()
self._notify.set()
def execute(self,
wms_id_list): # yields list of (wms_id, job_status, job_info)
exc = ExceptionCollector()
for wms_id in wms_id_list:
try:
job_info = filter_dict(
dict(self._status_fun(wms_id)),
value_filter=lambda v: v not in ['', '0'])
job_info[CheckInfo.RAW_STATUS] = job_info.pop('status',
'').lower()
if 'destination' in job_info:
try:
dest_info = job_info['destination'].split('/', 1)
job_info[CheckInfo.SITE] = dest_info[0].strip()
job_info[CheckInfo.QUEUE] = dest_info[1].strip()
except Exception:
clear_current_exception()
yield (wms_id,
self._status_map.get(job_info[CheckInfo.RAW_STATUS],
Job.UNKNOWN), job_info)
except Exception:
exc.collect()
if abort():
break
exc.raise_any(
BackendError('Encountered errors while checking job status'))
def __new__(cls, config, name):
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' %
repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView',
set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
wms = config.get('wms', '')
if wms:
return _create_backend(wms)
exc = ExceptionCollector()
for cmd, wms in [('sacct', 'SLURM'), ('sgepasswd', 'OGE'),
('pbs-config', 'PBS'), ('qsub', 'OGE'),
('condor_q', 'Condor'), ('bsub', 'LSF'),
('job_slurm', 'JMS')]:
try:
resolve_install_path(cmd)
except Exception:
exc.collect()
continue
return _create_backend(wms)
# at this point all backends have failed!
exc.raise_any(BackendError('No valid local backend found!'))
def __new__(cls, config, name):
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' % repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView', set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
wms = config.get('wms', '')
if wms:
return _create_backend(wms)
exc = ExceptionCollector()
(wms_search_dict, wms_search_order) = config.get_dict('wms search list',
default={'sacct': 'SLURM', 'sgepasswd': 'OGE', 'pbs-config': 'PBS', 'qsub': 'OGE',
'condor_q': 'Condor', 'bsub': 'LSF', 'job_slurm': 'JMS'},
default_order=['sacct', 'sgepasswd', 'pbs-config', 'qsub', 'condor_q', 'bsub', 'job_slurm'])
for cmd in wms_search_order:
try:
resolve_install_path(cmd)
except Exception:
exc.collect()
continue
return _create_backend(wms_search_dict[cmd])
# at this point all backends have failed!
exc.raise_any(BackendError('No valid local backend found!'))
def patlist2pathlist(value, mustExist):
ec = ExceptionCollector()
for pattern in value:
try:
for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError):
yield fn
except Exception:
ec.collect()
ec.raise_any(ConfigError('Error resolving paths'))
def patlist2pathlist(value, mustExist):
ec = ExceptionCollector()
for pattern in value:
try:
for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError):
yield fn
except Exception:
ec.collect()
ec.raise_any(ConfigError('Error resolving paths'))
def _patlist2pathlist(value, must_exist):
exc = ExceptionCollector()
search_path_list = self._config_view.config_vault.get('path:search', [])
for pattern in value:
try:
for fn in resolve_paths(pattern, search_path_list, must_exist, ConfigError):
yield fn
except Exception:
exc.collect()
exc.raise_any(ConfigError('Error resolving paths'))
def get_dataset_name_list(self):
if self._cache_dataset is None:
self._cache_dataset = set()
exc = ExceptionCollector()
for provider in self._provider_list:
try:
self._cache_dataset.update(provider.get_dataset_name_list())
except Exception:
exc.collect()
exc.raise_any(DatasetError('Could not retrieve all datasets!'))
return list(self._cache_dataset)
def getBlocksInternal(self):
ec = ExceptionCollector()
for provider in self.subprovider:
try:
for block in provider.getBlocks():
yield block
except Exception:
ec.collect()
if utils.abort():
raise DatasetError('Could not retrieve all datasets!')
ec.raise_any(DatasetError('Could not retrieve all datasets!'))
def _patlist2pathlist(value, must_exist):
exc = ExceptionCollector()
search_path_list = self._config_view.config_vault.get(
'path:search', [])
for pattern in value:
try:
for fn in resolve_paths(pattern, search_path_list,
must_exist, ConfigError):
yield fn
except Exception:
exc.collect()
exc.raise_any(ConfigError('Error resolving paths'))
def getDatasets(self):
if self._cache_dataset is None:
self._cache_dataset = []
ec = ExceptionCollector()
for provider in self._providerList:
try:
self._cache_dataset.extend(provider.getDatasets())
except Exception:
ec.collect()
if utils.abort():
raise DatasetError('Could not retrieve all datasets!')
ec.raise_any(DatasetError('Could not retrieve all datasets!'))
return self._cache_dataset
def getDatasets(self):
if self._cache_dataset is None:
self._cache_dataset = []
ec = ExceptionCollector()
for provider in self._providerList:
try:
self._cache_dataset.extend(provider.getDatasets())
except Exception:
ec.collect()
if utils.abort():
raise DatasetError('Could not retrieve all datasets!')
ec.raise_any(DatasetError('Could not retrieve all datasets!'))
return self._cache_dataset
def __new__(cls, config, name):
ec = ExceptionCollector()
for cmd, wms in [('sgepasswd', 'OGE'), ('pbs-config', 'PBS'), ('qsub', 'OGE'), ('bsub', 'LSF'), ('job_slurm', 'SLURM')]:
try:
utils.resolveInstallPath(cmd)
except Exception:
ec.collect()
continue
try:
wmsCls = WMS.getClass(wms)
except Exception:
raise BackendError('Unable to load backend class %s' % repr(wms))
config_wms = config.changeView(viewClass = 'TaggedConfigView', setClasses = [wmsCls])
return WMS.createInstance(wms, config_wms, name)
ec.raise_any(BackendError('No valid local backend found!')) # at this point all backends have failed!
def execute(self, wmsIDs): # yields list of (wmsID, job_status, job_info)
ec = ExceptionCollector()
for wmsID in wmsIDs:
try:
job_info = utils.filterDict(dict(self._status_fun(wmsID)), vF = lambda v: v not in ['', '0'])
job_info[CheckInfo.RAW_STATUS] = job_info.pop('status', '').lower()
if 'destination' in job_info:
try:
dest_info = job_info['destination'].split('/', 1)
job_info[CheckInfo.SITE] = dest_info[0].strip()
job_info[CheckInfo.QUEUE] = dest_info[1].strip()
except Exception:
clear_current_exception()
yield (wmsID, self._status_map.get(job_info[CheckInfo.RAW_STATUS], Job.UNKNOWN), job_info)
except Exception:
ec.collect()
if utils.abort():
break
ec.raise_any(BackendError('Encountered errors while checking job status'))
def __new__(cls, config, name):
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' %
repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView',
set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
wms = config.get('wms', '')
if wms:
return _create_backend(wms)
exc = ExceptionCollector()
(wms_search_dict,
wms_search_order) = config.get_dict('wms search list',
default={
'sacct': 'SLURM',
'sgepasswd': 'OGE',
'pbs-config': 'PBS',
'qsub': 'OGE',
'condor_q': 'Condor',
'bsub': 'LSF',
'job_slurm': 'JMS'
},
default_order=[
'sacct', 'sgepasswd',
'pbs-config', 'qsub',
'condor_q', 'bsub',
'job_slurm'
])
for cmd in wms_search_order:
try:
resolve_install_path(cmd)
except Exception:
exc.collect()
continue
return _create_backend(wms_search_dict[cmd])
# at this point all backends have failed!
exc.raise_any(BackendError('No valid local backend found!'))
def __new__(cls, config, name):
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' % repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView', set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
wms = config.get('wms', '')
if wms:
return _create_backend(wms)
exc = ExceptionCollector()
for cmd, wms in [('sacct', 'SLURM'), ('sgepasswd', 'OGE'), ('pbs-config', 'PBS'),
('qsub', 'OGE'), ('condor_q', 'Condor'), ('bsub', 'LSF'), ('job_slurm', 'JMS')]:
try:
resolve_install_path(cmd)
except Exception:
exc.collect()
continue
return _create_backend(wms)
# at this point all backends have failed!
exc.raise_any(BackendError('No valid local backend found!'))
class ANSIGUI(GUI):
alias_list = ['ansi']
def __new__(cls, config, workflow):
if is_dumb_terminal():
return GUI.create_instance('BasicConsoleGUI', config, workflow)
return GUI.__new__(cls)
def __init__(self, config, workflow):
GUI.__init__(self, config, workflow)
install_console_reset()
self._console_lock = GCLock(threading.RLock()) # terminal output lock
self._exc = ExceptionCollector()
(self._redraw_thread, self._redraw_shutdown) = (None, False)
(self._redraw_event,
self._immediate_redraw_event) = (GCEvent(rlock=True),
GCEvent(rlock=True))
self._redraw_interval = config.get_float('gui redraw interval',
0.1,
on_change=None)
self._redraw_delay = config.get_float('gui redraw delay',
0.05,
on_change=None)
element = config.get_composited_plugin('gui element',
'report activity log',
'MultiGUIElement',
cls=GUIElement,
on_change=None,
bind_kwargs={'inherit': True},
pargs=(workflow,
self._redraw_event,
sys.stdout))
self._element = FrameGUIElement(config, 'gui', workflow,
self._redraw_event, sys.stdout,
self._immediate_redraw_event, element)
def end_interface(
self
): # lots of try ... except .. finally - for clean console state restore
def _end_interface():
try:
self._finish_drawing()
finally:
GCStreamHandler.set_global_lock()
Console.reset_console()
rethrow(GUIException('GUI shutdown exception'), _end_interface)
self._exc.raise_any(GUIException('GUI drawing exception'))
def start_interface(self):
GCStreamHandler.set_global_lock(self._console_lock)
with_lock(self._console_lock, self._element.draw_startup)
self._redraw_shutdown = False # start redraw thread
self._redraw_thread = start_daemon('GUI draw thread', self._redraw)
def _finish_drawing(self):
def _final_draw():
try:
self._element.make_dirty()
finally:
self._redraw_shutdown = True # stop redraw thread
self._redraw_event.set()
try:
try:
with_lock(self._console_lock, _final_draw) # last redraw
finally:
if self._redraw_thread:
self._redraw_thread.join(5 + self._redraw_interval)
finally:
with_lock(self._console_lock,
self._element.draw_finish) # draw finish
def _redraw(self):
try:
while not self._redraw_shutdown:
self._redraw_event.wait(timeout=self._redraw_interval)
self._immediate_redraw_event.wait(timeout=self._redraw_delay)
with_lock(self._console_lock, self._element.redraw)
self._immediate_redraw_event.clear()
self._redraw_event.clear()
except Exception:
self._exc.collect()
abort(True)
class GCThreadPool(object):
# Class to manage a collection of threads
def __init__(self, limit=None):
self._lock = GCLock()
self._notify = GCEvent()
(self._limit, self._queue) = (limit, [])
(self._token, self._token_time, self._token_desc) = (0, {}, {})
self._log = logging.getLogger('thread_pool')
self._exc = ExceptionCollector(self._log)
def start_daemon(self, desc, fun, *args, **kwargs):
self._queue.append((desc, fun, args, kwargs))
with_lock(self._lock, self._queue_update)
def wait_and_drop(self, timeout=None):
while True:
result = with_lock(self._lock, self._monitor_token, timeout)
if result is not None:
return result
t_current = time.time()
self._notify.wait(timeout) # wait for thread to finish and adapt timeout for next round
if timeout is not None:
timeout -= time.time() - t_current
with_lock(self._lock, self._queue_update)
def _collect_exc(self, token, exc_info):
self._exc.collect(logging.ERROR, 'Exception in thread %r',
self._token_desc[token], exc_info=exc_info)
def _monitor_token(self, timeout):
t_current = time.time()
# discard stale threads
for token in list(self._token_time):
if timeout and (t_current - self._token_time.get(token, 0) > timeout):
self._unregister_token(token)
if not self._token_time: # no active threads
return True
# drop all threads if timeout is reached
if (timeout is not None) and (timeout <= 0):
self._token_time = {}
self._token_desc = {}
return False
def _queue_update(self):
while self._queue and ((self._limit is None) or (len(self._token_time) < self._limit)):
(desc, fun, args, kwargs) = self._queue.pop(0)
_start_thread(desc=desc, daemon=True, fun=self._run_thread,
args=(self._register_token(desc), fun, args, kwargs), kwargs={})
self._notify.clear()
def _register_token(self, desc):
self._token += 1
self._token_time[self._token] = time.time()
self._token_desc[self._token] = desc
return self._token
def _run_thread(self, token, fun, args, kwargs):
trace_fun = get_trace_fun()
if trace_fun:
sys.settrace(trace_fun)
try:
fun(*args, **kwargs)
except Exception:
with_lock(self._lock, self._collect_exc, token, sys.exc_info())
with_lock(self._lock, self._unregister_token, token)
with_lock(self._lock, self._notify.set)
def _unregister_token(self, token):
self._token_time.pop(token, None)
self._token_desc.pop(token, None)
class GCThreadPool(object):
# Class to manage a collection of threads
def __init__(self, limit=None):
self._lock = GCLock()
self._notify = GCEvent()
(self._limit, self._queue) = (limit, [])
(self._token, self._token_time, self._token_desc) = (0, {}, {})
self._log = logging.getLogger('thread_pool')
self._exc = ExceptionCollector(self._log)
def start_daemon(self, desc, fun, *args, **kwargs):
self._queue.append((desc, fun, args, kwargs))
with_lock(self._lock, self._queue_update)
def wait_and_drop(self, timeout=None):
while True:
result = with_lock(self._lock, self._monitor_token, timeout)
if result is not None:
return result
t_current = time.time()
self._notify.wait(timeout) # wait for thread to finish and adapt timeout for next round
if timeout is not None:
timeout -= time.time() - t_current
with_lock(self._lock, self._queue_update)
def _collect_exc(self, token, exc_info):
self._exc.collect(logging.ERROR, 'Exception in thread %r',
self._token_desc[token], exc_info=exc_info)
def _monitor_token(self, timeout):
t_current = time.time()
# discard stale threads
for token in list(self._token_time):
if timeout and (t_current - self._token_time.get(token, 0) > timeout):
self._unregister_token(token)
if not self._token_time: # no active threads
return True
# drop all threads if timeout is reached
if (timeout is not None) and (timeout <= 0):
self._token_time = {}
self._token_desc = {}
return False
def _queue_update(self):
while self._queue and ((self._limit is None) or (len(self._token_time) < self._limit)):
(desc, fun, args, kwargs) = self._queue.pop(0)
_start_thread(desc=desc, daemon=True, fun=self._run_thread,
args=(self._register_token(desc), fun, args, kwargs), kwargs={})
self._notify.clear()
def _register_token(self, desc):
self._token += 1
self._token_time[self._token] = time.time()
self._token_desc[self._token] = desc
return self._token
def _run_thread(self, token, fun, args, kwargs):
trace_fun = get_trace_fun()
if trace_fun:
sys.settrace(trace_fun)
try:
fun(*args, **kwargs)
except Exception:
with_lock(self._lock, self._collect_exc, token, get_current_exception())
with_lock(self._lock, self._unregister_token, token)
with_lock(self._lock, self._notify.set)
def _unregister_token(self, token):
self._token_time.pop(token, None)
self._token_desc.pop(token, None)