def __init__(self, cmd, *args, **kwargs):
self._time_started = None
self._time_finished = None
self._event_shutdown = GCEvent()
self._event_finished = GCEvent()
self._buffer_stdin = GCQueue()
self._buffer_stdout = GCQueue()
self._buffer_stderr = GCQueue()
self._env = kwargs.pop('environment', None) or dict(os.environ)
# Stream setup
self._do_log = kwargs.pop('logging', True) or None
self.stdout = ProcessReadStream(self._buffer_stdout, self._event_shutdown, self._event_finished, log = self._do_log)
self.stderr = ProcessReadStream(self._buffer_stderr, self._event_shutdown, self._event_finished, log = self._do_log)
self.stdin = ProcessWriteStream(self._buffer_stdin, log = self._do_log)
self.clear_logs() # reset log to proper start value
self._args = []
for arg in args:
self._args.append(str(arg))
if not cmd:
raise ProcessError('Invalid executable!')
if not os.path.isabs(cmd): # Resolve executable path
for path in os.environ.get('PATH', '').split(os.pathsep):
candidate = os.path.join(path, cmd)
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
cmd = candidate
break
if not os.path.exists(cmd):
raise ProcessError('%r does not exist' % cmd)
if not os.access(cmd, os.X_OK):
raise ProcessError('%r is not executable' % cmd)
self._log = logging.getLogger('process.%s' % os.path.basename(cmd).lower())
self._log.debug('External programm called: %s %s', cmd, self._args)
self._cmd = cmd
self.start()
def __init__(self, config, workflow):
GUI.__init__(self, config, workflow)
install_console_reset()
self._console_lock = GCLock(threading.RLock()) # terminal output lock
self._exc = ExceptionCollector()
(self._redraw_thread, self._redraw_shutdown) = (None, False)
(self._redraw_event,
self._immediate_redraw_event) = (GCEvent(rlock=True),
GCEvent(rlock=True))
self._redraw_interval = config.get_float('gui redraw interval',
0.1,
on_change=None)
self._redraw_delay = config.get_float('gui redraw delay',
0.05,
on_change=None)
element = config.get_composited_plugin('gui element',
'report activity log',
'MultiGUIElement',
cls=GUIElement,
on_change=None,
bind_kwargs={'inherit': True},
pargs=(workflow,
self._redraw_event,
sys.stdout))
self._element = FrameGUIElement(config, 'gui', workflow,
self._redraw_event, sys.stdout,
self._immediate_redraw_event, element)
def __init__(self, cmd, *args, **kwargs):
self._event_shutdown = GCEvent()
self._event_finished = GCEvent()
self._buffer_stdin = GCQueue()
self._buffer_stdout = GCQueue()
self._buffer_stderr = GCQueue()
# Stream setup
do_log = kwargs.get('logging', True) or None
self.stdout = ProcessReadStream(self._buffer_stdout, self._event_shutdown, self._event_finished, log = do_log)
self.stderr = ProcessReadStream(self._buffer_stderr, self._event_shutdown, self._event_finished, log = do_log)
self.stdin = ProcessWriteStream(self._buffer_stdin, log = do_log)
self.clear_logs() # reset log to proper start value
self._args = []
for arg in args:
self._args.append(str(arg))
if not cmd:
raise RuntimeError('Invalid executable!')
if not os.path.isabs(cmd): # Resolve executable path
for path in os.environ.get('PATH', '').split(os.pathsep):
if os.path.exists(os.path.join(path, cmd)):
cmd = os.path.join(path, cmd)
break
if not os.path.exists(cmd):
raise ProcessError('%r does not exist' % cmd)
if not os.access(cmd, os.X_OK):
raise ProcessError('%r is not executable' % cmd)
self._log = logging.getLogger('process.%s' % os.path.basename(cmd).lower())
self._log.debug('External programm called: %s %s', cmd, self._args)
self._cmd = cmd
self.start()
def download_single_file(opts, jobnum, fi_idx, fi, status_mon):
(source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi)
show_file_info(jobnum, fi_idx, fi)
# Copy files to local folder
if not accepted_se(opts, fi):
return status_mon.register_file_result(jobnum, fi_idx, 'skipping file on blacklisted SE',
FileDownloadStatus.FILE_SE_BLACKLIST)
activity_check = Activity('Checking file existance')
try:
if opts.skip_existing and (se_exists(target_se_path).status(timeout=10, terminate=True) == 0):
return status_mon.register_file_result(jobnum, fi_idx, 'skipping already existing file',
FileDownloadStatus.FILE_EXISTS)
finally:
activity_check.finish()
if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0:
activity = Activity('Creating target directory')
try:
mkdir_proc = se_mkdir(os.path.dirname(target_se_path))
if mkdir_proc.status(timeout=10, terminate=True) != 0:
return status_mon.register_file_result(jobnum, fi_idx, 'unable to create target dir',
FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc)
finally:
activity.finish()
if 'file://' in target_se_path:
local_se_path = target_se_path
copy_timeout_event = GCEvent()
copy_ended_event = GCEvent()
monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor,
jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event)
cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path)
while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1):
pass
copy_ended_event.set()
monitor_thread.join()
if copy_timeout_event.is_set():
cp_proc.terminate(timeout=1)
return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout',
FileDownloadStatus.FILE_TIMEOUT)
elif cp_proc.status(timeout=0, terminate=True) != 0:
return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error',
FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc)
return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
class Process(object):
def __init__(self, cmd, *args, **kwargs):
self._time_started = None
self._time_finished = None
self._event_shutdown = GCEvent()
self._event_finished = GCEvent()
self._buffer_stdin = GCQueue()
self._buffer_stdout = GCQueue()
self._buffer_stderr = GCQueue()
self._env_dict = kwargs.pop('env_dict', None) or dict(os.environ)
# Stream setup
self._do_log = kwargs.pop('logging', True) or None
self.stdout = ProcessReadStream(self._buffer_stdout, self._do_log,
self._event_shutdown, self._event_finished)
self.stderr = ProcessReadStream(self._buffer_stderr, self._do_log,
self._event_shutdown, self._event_finished)
self.stdin = ProcessWriteStream(self._buffer_stdin, self._do_log)
self.clear_logs() # reset log to proper start value
self._args = []
for arg in args:
self._args.append(str(arg))
if not cmd:
raise ProcessError('Invalid executable!')
if not os.path.isabs(cmd): # Resolve executable path
for path in os.environ.get('PATH', '').split(os.pathsep):
candidate = os.path.join(path, cmd)
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
cmd = candidate
break
if not os.path.exists(cmd):
raise ProcessError('%r does not exist' % cmd)
if not os.access(cmd, os.X_OK):
raise ProcessError('%r is not executable' % cmd)
self._log = logging.getLogger('process.%s' % os.path.basename(cmd).lower())
self._log.debug('External program called: %s %s', cmd, self._args)
self._cmd = cmd
self.start()
def __repr__(self):
msg = 'cmd = %s, args = %s, status = %s' % (self._cmd, repr(self._args), self.status(0))
if self._do_log:
msg += ', stdin log = %r, stdout log = %r, stderr log = %r' % (
self.stdin.read_log(), self.stdout.read_log(), self.stderr.read_log())
return '%s(%s)' % (self.__class__.__name__, msg)
def clear_logs(self):
self.stdout.clear_log()
self.stderr.clear_log()
self.stdin.clear_log()
def finish(self, timeout):
status = self.status_raise(timeout)
return (status, self.stdout.read(timeout=0), self.stderr.read(timeout=0))
def get_call(self):
return str.join(' ', imap(repr, [self._cmd] + self._args))
def get_output(self, timeout, raise_errors=True):
status = self.status(timeout)
result = self.stdout.read(timeout=0)
if status is None:
self.terminate(timeout=1)
if raise_errors and (status is None):
raise ProcessTimeout('Process is still running after waiting for %s seconds' % timeout)
elif raise_errors and (status != 0):
raise ProcessError('Command %s %s returned with exit code %s' % (
self._cmd, repr(self._args), status))
return result
def get_runtime(self):
if self._time_started is not None:
if self._time_finished is not None:
return self._time_finished - self._time_started
return time.time() - self._time_started
def kill(self, sig=signal.SIGTERM):
raise AbstractError
def restart(self, timeout):
if self.status(timeout=0) is None:
self.terminate(timeout)
result = self.status(timeout=0)
self.start()
return result
def start(self):
self.clear_logs()
self._event_shutdown.clear()
self._event_finished.clear()
self.stdout.reset_buffer()
self.stderr.reset_buffer()
self.stdin.reset_buffer()
return self._start()
def status(self, timeout, terminate=False):
raise AbstractError
def status_raise(self, timeout):
status = self.status(timeout)
if status is None:
self.terminate(timeout=1) # hard timeout
raise ProcessTimeout('Process is still running after waiting for %s seconds' % timeout)
return status
def terminate(self, timeout):
raise AbstractError
def _start(self):
raise AbstractError
class Process(object):
def __init__(self, cmd, *args, **kwargs):
self._time_started = None
self._time_finished = None
self._event_shutdown = GCEvent()
self._event_finished = GCEvent()
self._buffer_stdin = GCQueue()
self._buffer_stdout = GCQueue()
self._buffer_stderr = GCQueue()
self._env = kwargs.pop('environment', None) or dict(os.environ)
# Stream setup
self._do_log = kwargs.pop('logging', True) or None
self.stdout = ProcessReadStream(self._buffer_stdout,
self._event_shutdown,
self._event_finished,
log=self._do_log)
self.stderr = ProcessReadStream(self._buffer_stderr,
self._event_shutdown,
self._event_finished,
log=self._do_log)
self.stdin = ProcessWriteStream(self._buffer_stdin, log=self._do_log)
self.clear_logs() # reset log to proper start value
self._args = []
for arg in args:
self._args.append(str(arg))
if not cmd:
raise ProcessError('Invalid executable!')
if not os.path.isabs(cmd): # Resolve executable path
for path in os.environ.get('PATH', '').split(os.pathsep):
candidate = os.path.join(path, cmd)
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
cmd = candidate
break
if not os.path.exists(cmd):
raise ProcessError('%r does not exist' % cmd)
if not os.access(cmd, os.X_OK):
raise ProcessError('%r is not executable' % cmd)
self._log = logging.getLogger('process.%s' %
os.path.basename(cmd).lower())
self._log.debug('External programm called: %s %s', cmd, self._args)
self._cmd = cmd
self.start()
def get_runtime(self):
if self._time_started is not None:
if self._time_finished is not None:
return self._time_finished - self._time_started
return time.time() - self._time_started
def __repr__(self):
msg = 'cmd = %s, args = %s, status = %s' % (self._cmd, repr(
self._args), self.status(0))
if self._do_log:
msg += ', stdin log = %r, stdout log = %r, stderr log = %r' % (
self.stdin.read_log(), self.stdout.read_log(),
self.stderr.read_log())
return '%s(%s)' % (self.__class__.__name__, msg)
def clear_logs(self):
self.stdout.clear_log()
self.stderr.clear_log()
self.stdin.clear_log()
def get_call(self):
return str.join(' ', imap(repr, [self._cmd] + self._args))
def start(self):
self.clear_logs()
self._event_shutdown.clear()
self._event_finished.clear()
self.stdout.reset_buffer()
self.stderr.reset_buffer()
self.stdin.reset_buffer()
return self._start()
def _start(self):
raise AbstractError
def terminate(self, timeout):
raise AbstractError
def kill(self, sig=signal.SIGTERM):
raise AbstractError
def restart(self, timeout):
if self.status(timeout=0) is None:
self.terminate(timeout)
result = self.status(0)
self.start()
return result
def status(self, timeout, terminate=False):
raise AbstractError
def status_raise(self, timeout):
status = self.status(timeout)
if status is None:
self.terminate(timeout=1)
raise ProcessTimeout(
'Process is still running after waiting for %s seconds' %
timeout) # hard timeout
return status
def get_output(self, timeout, raise_errors=True):
status = self.status(timeout)
result = self.stdout.read(timeout=0)
if status is None:
self.terminate(timeout=1)
if raise_errors and (status is None):
raise ProcessTimeout(
'Process is still running after waiting for %s seconds' %
timeout)
elif raise_errors and (status != 0):
raise ProcessError('Command %s %s returned with exit code %s' %
(self._cmd, repr(self._args), status))
return result
def finish(self, timeout):
status = self.status_raise(timeout)
return (status, self.stdout.read(timeout=0),
self.stderr.read(timeout=0))
def download_single_file(opts, jobnum, fi_idx, fi, status_mon):
(source_se_path, target_se_path,
local_se_path) = get_fi_path_tuple(opts, fi)
show_file_info(jobnum, fi_idx, fi)
# Copy files to local folder
if not accepted_se(opts, fi):
return status_mon.register_file_result(
jobnum, fi_idx, 'skipping file on blacklisted SE',
FileDownloadStatus.FILE_SE_BLACKLIST)
activity_check = Activity('Checking file existance')
try:
if opts.skip_existing and (se_exists(target_se_path).status(
timeout=10, terminate=True) == 0):
return status_mon.register_file_result(
jobnum, fi_idx, 'skipping already existing file',
FileDownloadStatus.FILE_EXISTS)
finally:
activity_check.finish()
if se_exists(os.path.dirname(target_se_path)).status(timeout=10,
terminate=True) != 0:
activity = Activity('Creating target directory')
try:
mkdir_proc = se_mkdir(os.path.dirname(target_se_path))
if mkdir_proc.status(timeout=10, terminate=True) != 0:
return status_mon.register_file_result(
jobnum,
fi_idx,
'unable to create target dir',
FileDownloadStatus.FILE_MKDIR_FAILED,
proc=mkdir_proc)
finally:
activity.finish()
if 'file://' in target_se_path:
local_se_path = target_se_path
copy_timeout_event = GCEvent()
copy_ended_event = GCEvent()
monitor_thread = start_daemon('Download monitor %s' % jobnum,
download_monitor, jobnum, fi_idx, fi,
local_se_path, copy_ended_event,
copy_timeout_event)
cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path)
while (cp_proc.status(timeout=0) is
None) and not copy_timeout_event.wait(timeout=0.1):
pass
copy_ended_event.set()
monitor_thread.join()
if copy_timeout_event.is_set():
cp_proc.terminate(timeout=1)
return status_mon.register_file_result(jobnum, fi_idx,
'Transfer timeout',
FileDownloadStatus.FILE_TIMEOUT)
elif cp_proc.status(timeout=0, terminate=True) != 0:
return status_mon.register_file_result(jobnum,
fi_idx,
'Transfer error',
FileDownloadStatus.FILE_TIMEOUT,
proc=cp_proc)
return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
def get_event(self, name):
vault_key = 'event:%s' % name
if vault_key not in self._config_view.config_vault:
self._config_view.config_vault[vault_key] = GCEvent()
return self._config_view.config_vault[vault_key]