def convert_to_mobi(assembled_ebooks):
converted_ebooks = {}
kindle_gen_exists = Popen('kindlegen',
stdout=PIPE,
stderr=STDOUT,
stdin=PIPE,
shell=True)
kindle_gen_exists.communicate()
if kindle_gen_exists.returncode == 0:
for volume, ebook_file in assembled_ebooks.items():
info('Converting file "{}" to MOBI...'.format(ebook_file))
kindle_gen_convert = Popen('kindlegen "{}" -c2'.format(ebook_file),
stdout=PIPE,
stderr=STDOUT,
stdin=PIPE,
shell=True)
kindle_gen_convert.communicate()
if kindle_gen_convert.returncode not in [
0, 1
]: # acceptable exit codes
error('Error to convert file "{}" to MOBI'.format(ebook_file))
continue
os.remove(ebook_file)
converted_ebooks[volume] = os.path.join(
ebook_file,
str(ebook_file).replace('.epub', '.mobi'))
else:
warning('KindleGen not found!')
return converted_ebooks
def addFile(self, sourcefile):
if self.type in ['RAR', 'RAR5']:
raise NotImplementedError
process = Popen('7z a -y "' + self.filepath + '" "' + sourcefile + '"',
stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True)
process.communicate()
if process.returncode != 0:
raise OSError('Failed to add the file.')
def run_command(cmd, data, location, chw):
cwd = os.getcwd()
if location is not None and chw is True:
cwd = location
elif location is not None and chw is False:
cmd = '{0} {1}'.format(cmd, location)
r = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE, cwd=cwd)
if data is None:
output = r.communicate()[0].decode('utf-8')
else:
output = r.communicate(input=data)[0]
return output
def __init__(self, filepath):
self.filepath = filepath
self.type = None
if not os.path.isfile(self.filepath):
raise OSError('File not found.')
process = Popen('7z l -y -p1 "' + self.filepath + '"', stderr=STDOUT, stdout=PIPE, stdin=PIPE, shell=True)
for line in process.stdout:
if b'Type =' in line:
self.type = line.rstrip().decode().split(' = ')[1].upper()
break
process.communicate()
if process.returncode != 0:
raise OSError('Archive is corrupted or encrypted.')
elif self.type not in ['7Z', 'RAR', 'RAR5', 'ZIP']:
raise OSError('Unsupported archive format.')
def run_command(cmd, data, location, chw):
cwd = os.getcwd()
if location is not None and chw is True:
cwd = location
elif location is not None and chw is False:
cmd = '{0} {1}'.format(cmd, location)
r = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE, cwd=cwd)
if data is None:
output = r.communicate()[0].decode('utf-8')
else:
output = r.communicate(input=data)[0]
return output
def extract(self, targetdir):
if not os.path.isdir(targetdir):
raise OSError('Target directory don\'t exist.')
process = Popen('7z x -y -xr!__MACOSX -xr!.DS_Store -xr!thumbs.db -xr!Thumbs.db -o"' + targetdir + '" "' +
self.filepath + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True)
process.communicate()
if process.returncode != 0:
raise OSError('Failed to extract archive.')
tdir = os.listdir(targetdir)
if 'ComicInfo.xml' in tdir:
tdir.remove('ComicInfo.xml')
if len(tdir) == 1 and os.path.isdir(os.path.join(targetdir, tdir[0])):
for f in os.listdir(os.path.join(targetdir, tdir[0])):
move(os.path.join(targetdir, tdir[0], f), targetdir)
os.rmdir(os.path.join(targetdir, tdir[0]))
return targetdir
def sshcmd_shell(self, dst, cmd):
output = []
errors = []
ex = [
'ssh', '-l root', '-q', '-o UserKnownHostsFile=/dev/null',
'-o StrictHostKeyChecking=no', '-o ConnectTimeout=10', dst
]
ex += cmd
self.logger.debug("Execution list: %r", ex)
execline = ' '.join(ex)
self.logger.debug("Exec line: %r", execline)
sp = Popen(execline,
env=os.environ,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = sp.communicate()
output = [l.strip() for l in out.decode('utf-8').split('\n') if len(l)]
self.logger.debug("SSH CMD return code: %r", sp.returncode)
return sp.returncode, output, errors
def cmdshell(self, cmd, timeout=None, inpt=None, log_cmd=True):
if timeout is None:
timeout = conf['SHELL_COMMAND_TIMEOUT']
output = []
errors = []
execline = ' '.join(cmd)
if log_cmd:
display_cmd(self.logger, execline)
p = Popen(execline,
env=os.environ,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
try:
if inpt:
p.stdin = subprocess.PIPE
out, err = p.communicate(input=bytes(inpt, 'utf-8'),
timeout=timeout)
else:
out, err = p.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
p.kill()
out, err = p.communicate()
if out:
output = [
l.strip() for l in out.decode('utf-8').split('\n') if len(l)
]
if log_cmd:
display_cmd_output(self.logger, output)
if err:
errors = [l.strip() for l in err.decode().split('\n')]
self.logger.debug("Errors: %r", errors)
self.logger.debug("CMD return code: %r", p.returncode)
return p.returncode, output, errors
def run_subprocess(
command: Sequence[str],
shell: bool = False,
doexec: bool = True,
monitor_log: logging.Logger = None,
monitor_interval: int = 5,
tile_id: str = None,
) -> bool:
"""Runs a subprocess with `psutil.Popen` and monitors its status.
If subprocess returns non-zero exit code, STDERR is sent to the log.
:param command: The command to execute.
:param shell: Passed to `psutil.Popen`. Defaults to False.
:param doexec: Do execute the subprocess or just print out the concatenated
command. Used for testing.
:param monitor_log: A resource logger, which is returned by
:func:`~.recorder.configure_resource_logging`.
:param monitor_interval: How often query the resource usage of the process?
In seconds.
:param tile_id: Used for monitoring only.
:return: True/False on success/failure
"""
if doexec:
cmd = " ".join(command)
if shell:
command = cmd
log.debug(f"Tile {tile_id} command: {command}")
start = time()
popen = Popen(command, shell=shell, stderr=PIPE, stdout=PIPE)
if monitor_log is not None:
while True:
sleep(monitor_interval)
monitor_log.info(
f"{tile_id}\t{popen.pid}\t{popen.cpu_times().user}"
f"\t{popen.memory_info().rss}")
if (not popen.is_running() or popen.status() == STATUS_ZOMBIE
or popen.status() == STATUS_SLEEPING):
break
stdout, stderr = popen.communicate()
err = stderr.decode(getpreferredencoding(do_setlocale=True))
out = stdout.decode(getpreferredencoding(do_setlocale=True))
finish = time()
log.info(f"Tile {tile_id} finished in {(finish-start)/60} minutes")
if popen.returncode != 0:
log.error(
f"Tile {tile_id} process returned with {popen.returncode}")
else:
log.debug(
f"Tile {tile_id} process returned with {popen.returncode}")
log.debug(f"Tile {tile_id} stdout: \n{out}")
log.debug(f"Tile {tile_id} stderr: \n{err}")
return True if popen.returncode == 0 else False
else:
log.debug(f"Tile {tile_id} not executing {command}")
return True
def popen_wrapper(args, universal_newlines=True):
p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
close_fds=os.name != 'nt', universal_newlines=universal_newlines)
output, errors = p.communicate()
return (
output,
errors,
p.returncode
)
def extractMetadata(self):
process = Popen('7z x -y -so "' + self.filepath + '" ComicInfo.xml',
stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True)
xml = process.communicate()
if process.returncode != 0:
raise OSError('Failed to extract archive.')
try:
return parseString(xml[0])
except ExpatError:
return None
def run_command(cmd, data, location, chw, env=None):
cmd_env = None
if env:
cmd_env = os.environ.copy()
cmd_env.update(env)
cwd = os.getcwd()
if location is not None and chw is True:
cwd = location
elif location is not None and chw is False:
cmd = "{0} {1}".format(cmd, location)
r = Popen(shlex.split(cmd),
stdout=PIPE,
stdin=PIPE,
stderr=PIPE,
cwd=cwd,
env=cmd_env)
if data is None:
output = r.communicate()[0].decode("utf-8")
else:
output = r.communicate(input=data)[0]
return output
def make(base_dir, timeout=5):
"""compile source code.
return (ok?, msg)"""
cmd = ['clang++', '-std=c++11', 'main.cpp']
p = Popen(cmd, stderr=PIPE, cwd=base_dir)
try:
p.wait(timeout)
except TimeoutExpired:
return False, 'compilation take too much time.'
else:
if p.returncode == 0:
return True, ''
else:
return False, p.communicate()[1]
def make(base_dir, timeout=5):
"""compile source code.
return (ok?, msg)"""
cmd = ['clang++', '-std=c++11', 'main.cpp']
p = Popen(cmd, stderr=PIPE, cwd=base_dir)
try:
p.wait(timeout)
except TimeoutExpired:
return False, 'compilation take too much time.'
else:
if p.returncode == 0:
return True, ''
else:
return False, p.communicate()[1]
def execute(cmd, stderr_to_stdout=False, stdin=None):
"""Execute a command in the shell and return a tuple (rc, stdout, stderr)"""
if stderr_to_stdout:
stderr = STDOUT
else:
stderr = PIPE
if stdin is None:
_stdin = None
else:
_stdin = PIPE
p = Popen(cmd, close_fds=True, stdin=_stdin, stdout=PIPE, stderr=stderr, preexec_fn=os.setsid)
stdout, stderr = p.communicate(input=stdin)
return p.returncode, stdout, stderr
def _execute(cmd, stdin=None):
"""Run command and return output"""
logger.warn('Running command (panel): %s',
cmd) # Warning level because we want to see this in logs
proc = Popen(cmd,
bufsize=0,
close_fds=True,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE)
stdout, stderr = proc.communicate(input=stdin)
return {
'returncode': proc.returncode,
'stdout': stdout,
'stderr': stderr,
}
def scpcmd(self, src, dst, src_remote=False, log_cmd=True, timeout=None):
if timeout is None:
timeout = conf['SSH_CONNECT_TIMEOUT']
output = []
errors = []
ex = [
'scp', '-q', '-r', '-o UserKnownHostsFile=/dev/null',
'-o StrictHostKeyChecking=no',
'-o ConnectTimeout=%s' % timeout, src, dst
]
if src_remote:
ex.insert(1, '-3')
if log_cmd:
display_cmd(self.logger, ' '.join(ex))
sp = Popen(ex,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
out, err = sp.communicate()
if out:
output = [
l.strip() for l in out.decode('utf-8').split('\n') if len(l)
]
if log_cmd:
display_cmd_output(self.logger, output)
if err:
errors = [l.strip() for l in err.decode('utf-8').split('\n')]
self.logger.info("Errors: %r", errors)
self.logger.debug("SCP CMD return code: %r", sp.returncode)
return sp.returncode, output, errors
def _execute(cmd, stdin=None, stderr_to_stdout=False):
"""Run command and return output"""
logger.log(IMPORTANT, 'Running command (panel): %s', cmd)
if stderr_to_stdout:
stderr = STDOUT
else:
stderr = PIPE
proc = Popen(cmd,
bufsize=0,
close_fds=True,
stdout=PIPE,
stderr=stderr,
stdin=PIPE)
stdout, stderr = proc.communicate(input=stdin)
return {
'returncode': proc.returncode,
'stdout': stdout,
'stderr': stderr,
}
class McStasImage(ImageChannelMixin, PassiveChannel):
"""Image channel based on McStas simulation."""
_mythread = None
_process = None
parameters = {
'size':
Param(
'Detector size in pixels (x, y)',
settable=False,
type=tupleof(intrange(1, 8192), intrange(1, 8192)),
default=(1, 1),
),
'mcstasprog':
Param('Name of the McStas simulation executable',
type=str,
settable=False),
'mcstasdir':
Param('Directory where McStas stores results',
type=str,
default='singlecount',
settable=False),
'mcstasfile':
Param('Name of the McStas data file', type=str, settable=False),
'mcsiminfo':
Param('Name for the McStas Siminfo file',
settable=False,
type=str,
default='mccode.sim'),
'ci':
Param('Constant ci applied to simulated intensity I',
settable=False,
type=floatrange(0.),
default=1e3)
}
def doInit(self, mode):
self.arraydesc = ArrayDesc(self.name, self.size, '<u4')
self._workdir = os.getcwd()
def doReadArray(self, quality):
self.log.debug('quality: %s', quality)
if quality == LIVE:
self._send_signal(SIGUSR2)
elif quality == FINAL:
if self._mythread and self._mythread.is_alive():
self._mythread.join(1.)
if self._mythread.is_alive():
self.log.exception("Couldn't join readout thread.")
else:
self._mythread = None
self._readpsd(quality == LIVE)
return self._buf
def _prepare_params(self):
"""Return a list of key=value strings.
Each entry defines a parameter setting for the mcstas simulation call.
examples:
param=10
"""
raise NotImplementedError('Please implement _prepare_params method')
def doPrepare(self):
self._mcstas_params = ' '.join(self._prepare_params())
self.log.debug('McStas parameters: %s', self._mcstas_params)
self._buf = np.zeros(self.size[::-1])
self.readresult = [0]
def valueInfo(self):
return (Value(self.name + '.sum',
unit='cts',
type='counter',
errors='sqrt',
fmtstr='%d'), )
def doStart(self):
self._mythread = createThread('detector %s' % self, self._run)
def doStatus(self, maxage=0):
if self._mythread and self._mythread.is_alive():
return status.BUSY, 'busy'
return status.OK, 'idle'
def doFinish(self):
self.log.debug('finish')
self._send_signal(SIGTERM)
def _send_signal(self, sig):
if self._process and self._process.is_running():
self._process.send_signal(sig)
# wait for mcstas releasing fds
datafile = path.join(self._workdir, self.mcstasdir,
self.mcstasfile)
siminfo = path.join(self._workdir, self.mcstasdir, self.mcsiminfo)
try:
while self._process and self._process.is_running():
fnames = [f.path for f in self._process.open_files()]
if siminfo not in fnames and datafile not in fnames:
break
session.delay(.01)
except (AccessDenied, NoSuchProcess):
self.log.debug(
'McStas process already terminated in _send_signal(%r)',
sig)
self.log.debug('McStas process has written file on signal (%r)',
sig)
def _run(self):
"""Run McStas simulation executable.
The current settings of the instrument parameters will be transferred
to it.
"""
try:
shutil.rmtree(self.mcstasdir)
except (IOError, OSError):
self.log.info('could not remove old data')
command = '%s -n 1e8 -d %s %s' % (self.mcstasprog, self.mcstasdir,
self._mcstas_params)
self.log.debug('run %s', command)
try:
self._process = Popen(command.split(),
stdout=PIPE,
stderr=PIPE,
cwd=self._workdir)
out, err = self._process.communicate()
if out:
self.log.debug('McStas output:')
for line in out.splitlines():
self.log.debug('[McStas] %s', line)
if err:
self.log.warning('McStas found some problems:')
for line in err.splitlines():
self.log.warning('[McStas] %s', line)
except OSError as e:
self.log.error('Execution failed: %s', e)
self._process.wait()
self._process = None
def _readpsd(self, ignore_error=False):
try:
with open(
path.join(self._workdir, self.mcstasdir, self.mcstasfile),
'r') as f:
lines = f.readlines()[-3 * (self.size[0] + 1):]
if lines[0].startswith('# Data') and self.mcstasfile in lines[0]:
self._buf = (
np.loadtxt(lines[1:self.size[0] + 1], dtype=np.float32) *
self.ci).astype(np.uint32)
self.readresult = [self._buf.sum()]
elif not ignore_error:
raise IOError('Did not find start line: %s' % lines[0])
except IOError:
if not ignore_error:
self.log.exception('Could not read result file')
def run_subprocess(command,
shell=False,
doexec=True,
monitor=False,
tile_id=None):
"""Subprocess runner
If subrocess returns non-zero exit code, STDERR is sent to the logger.
Parameters
----------
command : list of str
Command to pass to subprocess.run(). Eg ['wget', '-q', '-r', dl_url]
shell : bool
Passed to subprocess.run()
doexec : bool
Execute the subprocess or just print out the concatenated command
Returns
-------
nothing
nothing
"""
if doexec:
cmd = " ".join(command)
if shell:
command = cmd
logger.debug(command)
popen = Popen(command, shell=shell, stderr=PIPE, stdout=PIPE)
pid = popen.pid
if monitor:
proc = Process(pid)
with proc.oneshot():
try:
logger_perf.debug(
"%s;%s;%s" %
(tile_id, virtual_memory().used, swap_memory().used))
except NoSuchProcess or ZombieProcess:
logger.debug("%s is Zombie or NoSuchProcess" % tile_id)
except AccessDenied as e:
logger_perf.exception(e)
# if monitor:
# running = True
# proc = Process(pid)
# with proc.oneshot():
# while running:
# try:
# logger_perf.debug("%s - %s - %s - %s - %s" % (
# tile_id, proc.cpu_percent(), proc.cpu_times(), proc.memory_full_info(), swap_memory()))
# except NoSuchProcess or ZombieProcess:
# logger.debug("%s is Zombie or NoSuchProcess" % tile_id)
# break
# except AccessDenied as e:
# logger_perf.exception(e)
# break
# running = proc.is_running()
# logger.debug("%s is running: %s" % (tile_id, running))
# sleep(1)
stdout, stderr = popen.communicate()
err = stderr.decode(locale.getpreferredencoding(do_setlocale=True))
popen.wait()
if popen.returncode != 0:
logger.debug("Process returned with non-zero exit code: %s",
popen.returncode)
logger.error(err)
return False
else:
return True
else:
logger.debug("Not executing %s", command)
return True
class Run(object):
"""Class to handle processes.
:ivar cmds: The ``cmds`` argument passed to the __init__ method
(a command line passed in a list, or a list of command lines passed as
a list of list).
:ivar status: The exit status. As the exit status is only meaningful after
the process has exited, its initial value is None. When a problem
running the command is detected and a process does not get
created, its value gets set to the special value 127.
:ivar out: process standard output (if instanciated with output = PIPE)
:ivar err: same as out but for standard error
:ivar pid: PID. Set to -1 if the command failed to run.
"""
def __init__(self,
cmds,
cwd=None,
output=PIPE,
error=STDOUT,
input=None,
bg=False,
timeout=None,
env=None,
set_sigpipe=True,
parse_shebang=False,
ignore_environ=True):
"""Spawn a process.
:param cmds: two possibilities:
1) a command line: a tool name and its arguments, passed
in a list. e.g. ['ls', '-a', '.']
2) a list of command lines (as defined in (1)): the
different commands will be piped. This means that
[['ps', '-a'], ['grep', 'vxsim']] will be equivalent to
the system command line 'ps -a | grep vxsim'.
:type cmds: list[str] | list[list[str]]
:param cwd: directory in which the process should be executed (string
or None). If None then current directory is used
:type cwd: str | None
:param output: can be PIPE (default), a filename string, a fd on an
already opened file, a python file object or None (for stdout).
:type output: int | str | file | None
:param error: same as output or STDOUT, which indicates that the
stderr data from the applications should be captured into the same
file handle as for stdout.
:type error: int | str | file | None
:param input: same as output
:type input: int | str | file | None
:param bg: if True then run in background
:type bg: bool
:param timeout: limit execution time (in seconds), None means
unlimited
:type timeout: int | None
:param env: dictionary for environment variables (e.g. os.environ)
:type env: dict
:param set_sigpipe: reset SIGPIPE handler to default value
:type set_sigpipe: bool
:param parse_shebang: take the #! interpreter line into account
:type parse_shebang: bool
:param ignore_environ: Applies only when env parameter is not None.
When set to True (the default), the only environment variables
passed to the program are the ones provided by the env parameter.
Otherwise, the environment passed to the program consists of the
environment variables currently defined (os.environ) augmented by
the ones provided in env.
:type ignore_environ: bool
:raise OSError: when trying to execute a non-existent file.
If you specify a filename for output or stderr then file content is
reseted (equiv. to > in shell). If you prepend the filename with '+'
then the file will be opened in append mode (equiv. to >> in shell)
If you prepend the input with '|', then the content of input string
will be used for process stdin.
"""
def add_interpreter_command(cmd_line):
"""Add the interpreter defined in the #! line to cmd_line.
If the #! line cannot be parsed, just return the cmd_line
unchanged
On windows, /usr/bin/env will be ignored to avoid a dependency on
cygwin and /bin/bash & /bin/sh are replaced by $SHELL if defined.
:param cmd_line: command line
:type cmd_line: list[str]
"""
if not parse_shebang:
# nothing to do
return cmd_line
prog = which(cmd_line[0], default=None)
if prog is None:
# Not found. Do not modify the command line
return cmd_line
with open(prog) as f:
try:
header = f.read()[0:2]
except UnicodeDecodeError: # py3-only
# unknown header - cannot decode the first two bytes
return cmd_line
if header != "#!":
# Unknown header
return cmd_line
# Header found, get the interpreter command in the first line
f.seek(0)
line = f.readline()
interpreter_cmds = [
l.strip() for l in line[line.find('!') + 1:].split()
]
# Pass the program path to the interpreter
if len(cmd_line) > 1:
cmd_line = [prog] + list(cmd_line[1:])
else:
cmd_line = [prog]
if sys.platform == 'win32': # unix: no cover
if interpreter_cmds[0] == '/usr/bin/env':
return interpreter_cmds[1:] + cmd_line
elif interpreter_cmds[0] in ('/bin/bash', '/bin/sh') and \
'SHELL' in os.environ:
return [os.environ['SHELL']] + cmd_line
return interpreter_cmds + cmd_line
# First resolve output, error and input
self.input_file = File(input, 'r')
self.output_file = File(output, 'w')
self.error_file = File(error, 'w')
self.status = None
self.out = ''
self.err = ''
self.cmds = []
if env is not None and not ignore_environ:
# ignore_environ is False, so get a copy of the current
# environment and update it with the env dictionnary.
tmp = os.environ.copy()
tmp.update(env)
env = tmp
rlimit_args = []
if timeout is not None:
rlimit = get_rlimit()
if os.path.exists(rlimit):
rlimit_args = [rlimit, '%d' % timeout]
else:
logger.warning('cannot find rlimit at %s', rlimit)
rlimit_args = []
try:
if isinstance(cmds[0], basestring):
self.cmds = rlimit_args + list(add_interpreter_command(cmds))
else:
self.cmds = [add_interpreter_command(c) for c in cmds]
self.cmds[0] = rlimit_args + list(self.cmds[0])
cmdlogger.debug('Run: cd %s; %s',
cwd if cwd is not None else os.getcwd(),
self.command_line_image())
if isinstance(cmds[0], basestring):
popen_args = {
'stdin': self.input_file.fd,
'stdout': self.output_file.fd,
'stderr': self.error_file.fd,
'cwd': cwd,
'env': env,
'universal_newlines': True
}
if sys.platform != 'win32' and \
set_sigpipe: # windows: no cover
# preexec_fn is no supported on windows
popen_args['preexec_fn'] = subprocess_setup
if WIN_NEW_PG and sys.platform == 'win32':
popen_args['creationflags'] = \
subprocess.CREATE_NEW_PROCESS_GROUP
self.internal = Popen(self.cmds, **popen_args)
else:
runs = []
for index, cmd in enumerate(self.cmds):
if index == 0:
stdin = self.input_file.fd
else:
stdin = runs[index - 1].stdout
# When connecting two processes using a Pipe don't use
# universal_newlines mode. Indeed commands transmitting
# binary data between them will crash
# (e.g. gzip -dc foo.txt | tar -xf -)
if index == len(self.cmds) - 1:
stdout = self.output_file.fd
txt_mode = True
else:
stdout = subprocess.PIPE
txt_mode = False
popen_args = {
'stdin': stdin,
'stdout': stdout,
'stderr': self.error_file.fd,
'cwd': cwd,
'env': env,
'universal_newlines': txt_mode
}
if sys.platform != 'win32' and \
set_sigpipe: # windows: no cover
# preexec_fn is no supported on windows
popen_args['preexec_fn'] = subprocess_setup
if WIN_NEW_PG and sys.platform == 'win32':
popen_args['creationflags'] = \
subprocess.CREATE_NEW_PROCESS_GROUP
try:
runs.append(Popen(cmd, **popen_args))
except OSError:
logger.error('error when spawning %s', cmd)
# We have an error (e.g. file not found), try to kill
# all processes already started.
for p in runs:
p.terminate()
raise
self.internal = runs[-1]
except Exception as e: # defensive code
self.__error(e, self.cmds)
raise
self.pid = self.internal.pid
if not bg:
self.wait()
def command_line_image(self):
"""Get shell command line image of the spawned command(s).
:rtype: str
This just a convenient wrapper around the function of the same
name.
"""
return command_line_image(self.cmds)
def close_files(self):
"""Close all file descriptors."""
self.output_file.close()
self.error_file.close()
self.input_file.close()
def __error(self, error, cmds):
"""Set pid to -1 and status to 127 before closing files."""
self.close_files()
logger.error(error)
def not_found(path):
"""Raise OSError.
:param path: path of the executable
:type path: str
"""
logger.error("%s not found", path)
e3.log.debug('PATH=%s', os.environ['PATH'])
raise OSError(errno.ENOENT,
'No such file or directory, %s not found' % path)
# Try to send an helpful message if one of the executable has not
# been found.
if isinstance(cmds[0], basestring):
if which(cmds[0], default=None) is None:
not_found(cmds[0])
else:
for cmd in cmds:
if which(cmd[0], default=None) is None:
not_found(cmd[0])
def wait(self):
"""Wait until process ends and return its status.
:return: exit code of the process
:rtype: int
"""
if self.status is not None:
# Wait has already been called
return self.status
# If there is no pipe in the loop then just do a wait. Otherwise
# in order to avoid blocked processes due to full pipes, use
# communicate.
if self.output_file.fd != subprocess.PIPE and \
self.error_file.fd != subprocess.PIPE and \
self.input_file.fd != subprocess.PIPE:
self.status = self.internal.wait()
else:
tmp_input = None
if self.input_file.fd == subprocess.PIPE:
tmp_input = self.input_file.get_command()
(self.out, self.err) = self.internal.communicate(tmp_input)
self.status = self.internal.returncode
self.close_files()
return self.status
def poll(self):
"""Check the process status and set self.status if available.
This method checks whether the underlying process has exited
or not. If it hasn't, then it just returns None immediately.
Otherwise, it stores the process' exit code in self.status
and then returns it.
:return: None if the process is still alive; otherwise, returns
the process exit status.
:rtype: int | None
"""
if self.status is not None:
# Process is already terminated and wait been called
return self.status
result = self.internal.poll()
if result is not None:
# Process is finished, call wait to finalize it (closing handles,
# ...)
return self.wait()
else:
return None
def kill(self, recursive=True, timeout=3):
"""Kill the process.
:param recursive: if True, try to kill the complete process tree
:type recursive: bool
:param timeout: wait timeout (in seconds) after sending the kill
signal (when recursive=True)
:type timeout: int
"""
if recursive:
kill_process_tree(self.internal, timeout=timeout)
else:
self.internal.kill()
def interrupt(self):
"""Send SIGINT to the process, kill on Windows."""
if sys.platform == 'win32':
self.kill() # Ctrl-C event is unreliable on Windows
else:
self.internal.send_signal(signal.SIGINT)
def is_running(self):
"""Check whether the process is running.
:rtype: bool
"""
if psutil is None: # defensive code
# psutil not imported, use our is_running function
return is_running(self.pid)
else:
return self.internal.is_running()
def children(self):
"""Return list of child processes (using psutil).
:rtype: list[psutil.Process]
"""
if psutil is None: # defensive code
raise NotImplementedError('Run.children() require psutil')
return self.internal.children()
class McStasImage(ImageChannelMixin, PassiveChannel):
"""Image channel based on McStas simulation.
This channel should be used together with `McStasTimerChannel`
which provides the preselection [s] for calculating the number of
simulated neutron counts:
Ncounts = preselection [s] * ratio [cts/s]
Note: Please configure **ratio** to match the average simulated neutron
counts per second on your system.
"""
_mythread = None
_process = None
_started = None
parameters = {
'size':
Param(
'Detector size in pixels (x, y)',
settable=False,
type=tupleof(intrange(1, 8192), intrange(1, 8192)),
default=(1, 1),
),
'mcstasprog':
Param('Name of the McStas simulation executable',
type=str,
settable=False),
'mcstasdir':
Param('Directory where McStas stores results',
type=str,
default='%(session.experiment.dataroot)s'
'/singlecount',
settable=False),
'mcstasfile':
Param('Name of the McStas data file', type=str, settable=False),
'mcsiminfo':
Param('Name for the McStas Siminfo file',
settable=False,
type=str,
default='mccode.sim'),
'ratio':
Param(
'Simulated neutrons per second for this machine. Please'
' tune this parameter according to your hardware for '
' realistic count times',
settable=False,
type=floatrange(1e3),
default=1e6),
'ci':
Param('Constant ci multiplied with simulated intensity I',
settable=False,
type=floatrange(1.)),
# preselection time, usually set by McStasTimer
'preselection':
Param('Preset value for this channel',
type=float,
settable=True,
default=1.),
}
def doInit(self, mode):
self.arraydesc = ArrayDesc(self.name, self.size, '<u4')
self._workdir = os.getcwd()
self._start_time = None
def doReadArray(self, quality):
self.log.debug('quality: %s', quality)
if quality == LIVE:
self._send_signal(SIGUSR2)
elif quality == FINAL:
if self._mythread and self._mythread.is_alive():
self._mythread.join(1.)
if self._mythread.is_alive():
self.log.exception("Couldn't join readout thread.")
else:
self._mythread = None
self._readpsd(quality)
return self._buf
def _prepare_params(self):
"""Return a list of key=value strings.
Each entry defines a parameter setting for the mcstas simulation call.
examples:
param=10
"""
raise NotImplementedError('Please implement _prepare_params method')
def doPrepare(self):
self._mcstas_params = ' '.join(self._prepare_params())
self.log.debug('McStas parameters: %s', self._mcstas_params)
self._buf = np.zeros(self.size[::-1])
self.readresult = [0]
self._start_time = None
self._mcstasdirpath = session.experiment.data.expandNameTemplates(
self.mcstasdir)[0]
def valueInfo(self):
return (Value(self.name + '.sum',
unit='cts',
type='counter',
errors='sqrt',
fmtstr='%d'), )
def doStart(self):
self._started = True
self._mythread = createThread('detector %s' % self, self._run)
def doStatus(self, maxage=0):
if self._started or (self._mythread and self._mythread.is_alive()):
return status.BUSY, 'busy'
return status.OK, 'idle'
def doFinish(self):
self.log.debug('finish')
self._started = None
self._send_signal(SIGTERM)
def _send_signal(self, sig):
if self._process and self._process.is_running():
self._process.send_signal(sig)
# wait for mcstas releasing fds
datafile = path.join(self._mcstasdirpath, self.mcstasfile)
siminfo = path.join(self._mcstasdirpath, self.mcsiminfo)
try:
while self._process and self._process.is_running():
fnames = [f.path for f in self._process.open_files()]
if siminfo not in fnames and datafile not in fnames:
break
session.delay(.01)
except (AccessDenied, NoSuchProcess):
self.log.debug(
'McStas process already terminated in _send_signal(%r)',
sig)
self.log.debug('McStas process has written file on signal (%r)',
sig)
def _run(self):
"""Run McStas simulation executable.
The current settings of the instrument parameters will be transferred
to it.
"""
try:
shutil.rmtree(self._mcstasdirpath)
except OSError:
self.log.warning('could not remove old data')
command = '%s -n %d -d %s %s' % (
self.mcstasprog,
self.ratio * self.preselection,
self._mcstasdirpath,
self._mcstas_params,
)
self.log.debug('run %s', command)
try:
self._start_time = time.time()
self._process = Popen(command.split(),
stdout=PIPE,
stderr=PIPE,
cwd=self._workdir)
out, err = self._process.communicate()
if out:
self.log.debug('McStas output:')
for line in out.splitlines():
self.log.debug('[McStas] %s',
line.decode('utf-8', 'ignore'))
if err:
self.log.warning('McStas found some problems:')
for line in err.splitlines():
self.log.warning('[McStas] %s',
line.decode('utf-8', 'ignore'))
except OSError as e:
self.log.error('Execution failed: %s', e)
if self._process:
self._process.wait()
self._process = None
self._started = None
def _readpsd(self, quality):
try:
with open(path.join(self._mcstasdirpath, self.mcstasfile),
'r') as f:
lines = f.readlines()[-3 * (self.size[0] + 1):]
if lines[0].startswith('# Data') and self.mcstasfile in lines[0]:
if quality == FINAL:
seconds = self.preselection
else:
seconds = min(time.time() - self._start_time,
self.preselection)
self._buf = (
np.loadtxt(lines[1:self.size[0] + 1], dtype=np.float32) *
self.ci * seconds).astype(np.uint32)
self.readresult = [self._buf.sum()]
elif quality != LIVE:
raise OSError('Did not find start line: %s' % lines[0])
except OSError:
if quality != LIVE:
self.log.exception('Could not read result file')
itertools.product([-1, 1], repeat=int(log2(num_processes))))):
p_launch_times.append(time.time())
p = Popen([
'python.exe', 'vortex_signs_classification_and_statistics.py',
str(path),
str(config)
],
cwd=scriptdir,
stdout=PIPE,
stderr=PIPE) #, creationflags=CREATE_NEW_CONSOLE)
p_list.append(p)
print "process %d PID=%d" % (i, p.pid)
result = []
for i, p in enumerate(p_list):
stdout, stderr = p.communicate()
print 'subprocess %d output (note all stdout is printed first, followed by all stderr)' % i
print 'launched at: ', p_launch_times[i]
print stdout.replace('\r', '')
sys.stderr.write(stderr.replace('\r', ''))
sys.stderr.flush()
print '----'
try:
result.append(ast.literal_eval(stdout.split('\r\n')[-2]))
except:
sys.stderr.write('Failed to process result from subprocess %d' % i)
sys.stderr.flush()
end_time = time.time()
def _execute(self, cmd, stdin, meta=None, callback=None):
"""
The "real" execute function.
Just like executing a command in the shell on the compute node.
Do not use directly. Call the execute() wrapper instead.
"""
request = self.request
p = Popen(cmd,
shell=True,
bufsize=0,
close_fds=True,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
preexec_fn=os.setsid)
exec_time = datetime.utcnow()
try:
stdout, stderr = p.communicate(input=stdin)
except (Terminated, KeyboardInterrupt, SystemExit) as exc:
# This is mainly used for fetching SIGTERM
# The SIGTERM signal will be caught here as Terminated exception and the SIGKILL will never be caught here.
sig = _exc_signal(exc)
logger.error(
'Task %s received %r exception -> sending signal %d to %d',
request.id, exc, sig, p.pid)
try:
os.killpg(p.pid, sig) # Send signal to process group
except OSError:
pass
try:
p.send_signal(sig) # Send signal to process and wait
p.wait()
except (OSError, NoSuchProcess):
pass
raise exc
finish_time = datetime.utcnow()
if meta is None:
meta = {}
elif meta:
if 'replace_text' in meta:
for i in meta['replace_text']:
stdout = stdout.replace(i[0], i[1])
stderr = stderr.replace(i[0], i[1])
del meta['replace_text']
if 'replace_stdout' in meta:
for i in meta['replace_stdout']:
stdout = stdout.replace(i[0], i[1])
del meta['replace_stdout']
if 'replace_stderr' in meta:
for i in meta['replace_stderr']:
stderr = stderr.replace(i[0], i[1])
del meta['replace_stderr']
if 'compress_stdout' in meta:
stdout = compress(stdout)
del meta['compress_stdout']
if 'compress_stderr' in meta:
stderr = compress(stderr)
del meta['compress_stderr']
if 'encode_stdout' in meta:
stdout = b64encode(stdout)
del meta['encode_stdout']
if 'encode_stderr' in meta:
stderr = b64encode(stderr)
del meta['encode_stderr']
meta['exec_time'] = exec_time.isoformat()
meta['finish_time'] = finish_time.isoformat()
if 'output' in meta:
result = meta.pop('output', {})
result['meta'] = meta
_stdout = result.pop('stdout', None)
if _stdout:
result[_stdout] = stdout.strip()
_stderr = result.pop('stderr', None)
if _stderr:
result[_stderr] = stderr.strip()
_returncode = result.pop('returncode', None)
if _returncode:
result[_returncode] = p.returncode
else:
result = {
'returncode': p.returncode,
'stdout': stdout,
'stderr': stderr,
'meta': meta,
}
# Implicit logging if no callback is specified
# Use callback=False to disable automatic logging
if callback is None:
callback = [LOGTASK, meta, None]
if callback:
nolog = meta.get('nolog', False)
cb_name = callback[0]
cb_kwargs = {}
cb_expire = None
if len(callback) > 1:
cb_kwargs = callback[1]
if len(callback) > 2:
cb_expire = callback[2]
t = send_task_forever(request.id,
cb_name,
nolog=nolog,
args=(result, request.id),
kwargs=cb_kwargs,
queue=Q_MGMT,
expires=cb_expire,
task_id=task_id_from_task_id(request.id))
result['meta']['cb_name'] = cb_name
result['meta']['callback'] = t.id
# Do not run emergency callback in after_return
self.all_done = True
return result
class Run(object):
"""Class to handle processes.
:ivar cmds: The ``cmds`` argument passed to the __init__ method
(a command line passed in a list, or a list of command lines passed as
a list of list).
:ivar status: The exit status. As the exit status is only meaningful after
the process has exited, its initial value is None. When a problem
running the command is detected and a process does not get
created, its value gets set to the special value 127.
:ivar out: process standard output (if instanciated with output = PIPE)
:ivar err: same as out but for standard error
:ivar pid: PID. Set to -1 if the command failed to run.
"""
def __init__(self, cmds, cwd=None, output=PIPE,
error=STDOUT, input=None, bg=False, timeout=None,
env=None, set_sigpipe=True, parse_shebang=False,
ignore_environ=True, python_executable=sys.executable):
"""Spawn a process.
:param cmds: two possibilities:
1) a command line: a tool name and its arguments, passed
in a list. e.g. ['ls', '-a', '.']
2) a list of command lines (as defined in (1)): the
different commands will be piped. This means that
[['ps', '-a'], ['grep', 'vxsim']] will be equivalent to
the system command line 'ps -a | grep vxsim'.
:type cmds: list[str] | list[list[str]]
:param cwd: directory in which the process should be executed (string
or None). If None then current directory is used
:type cwd: str | None
:param output: can be PIPE (default), a filename string, a fd on an
already opened file, a python file object or None (for stdout).
:type output: int | str | file | None
:param error: same as output or STDOUT, which indicates that the
stderr data from the applications should be captured into the same
file handle as for stdout.
:type error: int | str | file | None
:param input: same as output
:type input: int | str | file | None
:param bg: if True then run in background
:type bg: bool
:param timeout: limit execution time (in seconds), None means
unlimited
:type timeout: int | None
:param env: dictionary for environment variables (e.g. os.environ)
:type env: dict
:param set_sigpipe: reset SIGPIPE handler to default value
:type set_sigpipe: bool
:param parse_shebang: take the #! interpreter line into account
:type parse_shebang: bool
:param ignore_environ: Applies only when env parameter is not None.
When set to True (the default), the only environment variables
passed to the program are the ones provided by the env parameter.
Otherwise, the environment passed to the program consists of the
environment variables currently defined (os.environ) augmented by
the ones provided in env.
:type ignore_environ: bool
:param python_executable: name or path to the python executable
:type python_executable: str
:raise OSError: when trying to execute a non-existent file.
If you specify a filename for output or stderr then file content is
reseted (equiv. to > in shell). If you prepend the filename with '+'
then the file will be opened in append mode (equiv. to >> in shell)
If you prepend the input with '|', then the content of input string
will be used for process stdin.
"""
def add_interpreter_command(cmd_line):
"""Add the interpreter defined in the #! line to cmd_line.
If the #! line cannot be parsed, just return the cmd_line
unchanged
If the interpreter command line contains /usr/bin/env python it
will be replaced by the value of python_executable
On windows, /usr/bin/env will be ignored to avoid a dependency on
cygwin
:param cmd_line: command line
:type cmd_line: list[str]
"""
if not parse_shebang:
# nothing to do
return cmd_line
prog = which(cmd_line[0], default=None)
if prog is None:
# Not found. Do not modify the command line
return cmd_line
with open(prog) as f:
header = f.read()[0:2]
if header != "#!":
# Unknown header
return cmd_line
# Header found, get the interpreter command in the first line
f.seek(0)
line = f.readline()
interpreter_cmds = [l.strip() for l in
line[line.find('!') + 1:].split()]
# Pass the program path to the interpreter
if len(cmd_line) > 1:
cmd_line = [prog] + list(cmd_line[1:])
else:
cmd_line = [prog]
# If the interpreter is '/usr/bin/env python', use
# python_executable instead to keep the same python executable
if interpreter_cmds[0:2] == ['/usr/bin/env', 'python']:
if len(interpreter_cmds) > 2:
return [python_executable] + \
interpreter_cmds[2:] + cmd_line
else:
return [python_executable] + cmd_line
elif sys.platform == 'win32': # unix: no cover
if interpreter_cmds[0] == '/usr/bin/env':
return interpreter_cmds[1:] + cmd_line
elif interpreter_cmds[0] in ('/bin/bash', '/bin/sh') and \
'SHELL' in os.environ:
return [os.environ['SHELL']] + cmd_line
return interpreter_cmds + cmd_line
# First resolve output, error and input
self.input_file = File(input, 'r')
self.output_file = File(output, 'w')
self.error_file = File(error, 'w')
self.status = None
self.out = ''
self.err = ''
self.cmds = []
if env is not None and not ignore_environ:
# ignore_environ is False, so get a copy of the current
# environment and update it with the env dictionnary.
tmp = os.environ.copy()
tmp.update(env)
env = tmp
rlimit_args = []
if timeout is not None:
rlimit = get_rlimit()
if os.path.exists(rlimit):
rlimit_args = [rlimit, '%d' % timeout]
else:
logger.warning('cannot find rlimit at %s', rlimit)
rlimit_args = []
try:
if isinstance(cmds[0], basestring):
self.cmds = rlimit_args + list(add_interpreter_command(cmds))
else:
self.cmds = [add_interpreter_command(c) for c in cmds]
self.cmds[0] = rlimit_args + list(self.cmds[0])
cmdlogger.debug('Run: cd %s; %s' % (
cwd if cwd is not None else os.getcwd(),
self.command_line_image()))
if isinstance(cmds[0], basestring):
popen_args = {
'stdin': self.input_file.fd,
'stdout': self.output_file.fd,
'stderr': self.error_file.fd,
'cwd': cwd,
'env': env,
'universal_newlines': True}
if sys.platform != 'win32' and set_sigpipe:
# preexec_fn is no supported on windows
popen_args['preexec_fn'] = subprocess_setup
self.internal = Popen(self.cmds, **popen_args)
else:
runs = []
for index, cmd in enumerate(self.cmds):
if index == 0:
stdin = self.input_file.fd
else:
stdin = runs[index - 1].stdout
# When connecting two processes using a Pipe don't use
# universal_newlines mode. Indeed commands transmitting
# binary data between them will crash
# (e.g. gzip -dc foo.txt | tar -xf -)
if index == len(self.cmds) - 1:
stdout = self.output_file.fd
txt_mode = True
else:
stdout = subprocess.PIPE
txt_mode = False
popen_args = {
'stdin': stdin,
'stdout': stdout,
'stderr': self.error_file.fd,
'cwd': cwd,
'env': env,
'universal_newlines': txt_mode}
if sys.platform != 'win32' and set_sigpipe:
# preexec_fn is no supported on windows
popen_args['preexec_fn'] = subprocess_setup
try:
runs.append(Popen(cmd, **popen_args))
except OSError as e:
logger.error('error when spawning %s', cmd)
# We have an error (e.g. file not found), try to kill
# all processes already started.
for p in runs:
p.terminate()
raise
self.internal = runs[-1]
except Exception as e:
self.__error(e, self.cmds)
raise
self.pid = self.internal.pid
if not bg:
self.wait()
def command_line_image(self):
"""Get shell command line image of the spawned command(s).
:rtype: str
This just a convenient wrapper around the function of the same
name.
"""
return command_line_image(self.cmds)
def close_files(self):
"""Close all file descriptors."""
self.output_file.close()
self.error_file.close()
self.input_file.close()
def __error(self, error, cmds):
"""Set pid to -1 and status to 127 before closing files."""
self.close_files()
logger.error(error)
def not_found(path):
"""Raise OSError.
:param path: path of the executable
:type path: str
"""
logger.error("%s not found", path)
e3.log.debug('PATH=%s', os.environ['PATH'])
raise OSError(errno.ENOENT,
'No such file or directory, %s not found' % path)
# Try to send an helpful message if one of the executable has not
# been found.
if isinstance(cmds[0], basestring):
if which(cmds[0], default=None) is None:
not_found(cmds[0])
else:
for cmd in cmds:
if which(cmd[0], default=None) is None:
not_found(cmd[0])
def wait(self):
"""Wait until process ends and return its status.
:return: exit code of the process
:rtype: int
"""
if self.status is not None:
# Wait has already been called
return self.status
# If there is no pipe in the loop then just do a wait. Otherwise
# in order to avoid blocked processes due to full pipes, use
# communicate.
if self.output_file.fd != subprocess.PIPE and \
self.error_file.fd != subprocess.PIPE and \
self.input_file.fd != subprocess.PIPE:
self.status = self.internal.wait()
else:
tmp_input = None
if self.input_file.fd == subprocess.PIPE:
tmp_input = self.input_file.get_command()
(self.out, self.err) = self.internal.communicate(tmp_input)
self.status = self.internal.returncode
self.close_files()
return self.status
def poll(self):
"""Check the process status and set self.status if available.
This method checks whether the underlying process has exited
or not. If it hasn't, then it just returns None immediately.
Otherwise, it stores the process' exit code in self.status
and then returns it.
:return: None if the process is still alive; otherwise, returns
the process exit status.
:rtype: int | None
"""
if self.status is not None:
# Process is already terminated and wait been called
return self.status
result = self.internal.poll()
if result is not None:
# Process is finished, call wait to finalize it (closing handles,
# ...)
return self.wait()
else:
return None
def kill(self):
"""Kill the process."""
self.internal.kill()
def interrupt(self):
"""Send SIGINT CTRL_C_EVENT to the process."""
# On windows CTRL_C_EVENT is available and SIGINT is not;
# and the other way around on other platforms.
interrupt_signal = getattr(signal, 'CTRL_C_EVENT', signal.SIGINT)
self.internal.send_signal(interrupt_signal)
def is_running(self):
"""Check whether the process is running.
:rtype: bool
"""
if psutil is None:
# psutil not imported, use our is_running function
return is_running(self.pid)
else:
return self.internal.is_running()
def children(self):
"""Return list of child processes (using psutil).
:rtype: list[psutil.Process]
"""
if psutil is None:
raise NotImplementedError('Run.children() require psutil')
return self.internal.children()
class Run:
"""Class to handle processes.
:ivar cmds: The ``cmds`` argument passed to the __init__ method
(a command line passed in a list, or a list of command lines passed as
a list of list).
:ivar status: The exit status. As the exit status is only meaningful after
the process has exited, its initial value is None. When a problem
running the command is detected and a process does not get
created, its value gets set to the special value 127.
:ivar raw_out: process standard output as bytes (if instanciated with
output = PIPE). Use self.out to get a decoded string.
:ivar raw_err: same as raw_out but for standard error.
:ivar pid: PID. Set to -1 if the command failed to run.
"""
def __init__(
self,
cmds: AnyCmdLine,
cwd: Optional[str] = None,
output: STDOUT_VALUE | DEVNULL_VALUE | PIPE_VALUE | str | IO
| None = PIPE,
error: STDOUT_VALUE | DEVNULL_VALUE | PIPE_VALUE | str | IO
| None = STDOUT,
input: DEVNULL_VALUE | PIPE_VALUE | str | IO
| None = None, # noqa: A002
bg: bool = False,
timeout: Optional[int] = None,
env: Optional[dict] = None,
set_sigpipe: bool = True,
parse_shebang: bool = False,
ignore_environ: bool = True,
) -> None:
"""Spawn a process.
:param cmds: two possibilities:
1) a command line: a tool name and its arguments, passed
in a list. e.g. ['ls', '-a', '.']
2) a list of command lines (as defined in (1)): the
different commands will be piped. This means that
[['ps', '-a'], ['grep', 'vxsim']] will be equivalent to
the system command line 'ps -a | grep vxsim'.
:param cwd: directory in which the process should be executed (string
or None). If None then current directory is used
:param output: can be PIPE (default), a filename string, a fd on an
already opened file, a python file object or None (for stdout).
:param error: same as output or STDOUT, which indicates that the
stderr data from the applications should be captured into the same
file handle as for stdout.
:param input: same as output
:param bg: if True then run in background
:param timeout: limit execution time (in seconds), None means
unlimited
:param env: dictionary for environment variables (e.g. os.environ)
:param set_sigpipe: reset SIGPIPE handler to default value
:param parse_shebang: take the #! interpreter line into account
:param ignore_environ: Applies only when env parameter is not None.
When set to True (the default), the only environment variables
passed to the program are the ones provided by the env parameter.
Otherwise, the environment passed to the program consists of the
environment variables currently defined (os.environ) augmented by
the ones provided in env.
:raise OSError: when trying to execute a non-existent file.
If you specify a filename for output or stderr then file content is
reseted (equiv. to > in shell). If you prepend the filename with '+'
then the file will be opened in append mode (equiv. to >> in shell)
If you prepend the input with '|', then the content of input string
will be used for process stdin.
"""
def add_interpreter_command(cmd_line: CmdLine) -> CmdLine:
"""Add the interpreter defined in the #! line to cmd_line.
If the #! line cannot be parsed, just return the cmd_line
unchanged
On windows, /usr/bin/env will be ignored to avoid a dependency on
cygwin and /bin/bash & /bin/sh are replaced by $SHELL if defined.
:param cmd_line: command line
"""
if not parse_shebang:
# nothing to do
return cmd_line
prog = which(cmd_line[0], default=None)
if prog is None:
# Not found. Do not modify the command line
return cmd_line
with open(prog) as f:
try:
header = f.read()[0:2]
except UnicodeDecodeError:
# unknown header - cannot decode the first two bytes
return cmd_line
if header != "#!":
# Unknown header
return cmd_line
# Header found, get the interpreter command in the first line
f.seek(0)
line = f.readline()
interpreter_cmds = [
word.strip() for word in line[line.find("!") + 1:].split()
]
# Pass the program path to the interpreter
if len(cmd_line) > 1:
cmd_line = [prog] + list(cmd_line[1:])
else:
cmd_line = [prog]
if sys.platform == "win32": # unix: no cover
if interpreter_cmds[0] == "/usr/bin/env":
# On windows be sure that PATH is taken into account by
# using which. In some cases involving python
# interpreter, the python interpreter used to run this
# module has been used rather than the first one on the
# path.
interpreter_cmds[1] = which(
interpreter_cmds[1], default=interpreter_cmds[1])
return interpreter_cmds[1:] + cmd_line
elif (interpreter_cmds[0] in ("/bin/bash", "/bin/sh")
and "SHELL" in os.environ):
return [os.environ["SHELL"]] + cmd_line
return interpreter_cmds + cmd_line
# First resolve output, error and input
self.input_file = File(input, "r")
self.output_file = File(output, "w")
self.error_file = File(error, "w")
self.status: Optional[int] = None
self.raw_out = b""
self.raw_err = b""
self.cmds = []
if env is not None:
if ignore_environ:
if sys.platform == "win32":
# On Windows not all environment variables can be
# discarded. At least SYSTEMDRIVE, SYSTEMROOT should be
# set. In order to be portable propagate their value in
# case the user does not pass them in env when
# ignore_environ is set to True.
tmp = {}
for var in ("SYSTEMDRIVE", "SYSTEMROOT"):
if var not in env and var in os.environ:
tmp[var] = os.environ[var]
tmp.update(env)
env = tmp
else:
# ignore_environ is False, so get a copy of the current
# environment and update it with the env dictionary.
tmp = os.environ.copy()
tmp.update(env)
env = tmp
rlimit_args = []
if timeout is not None:
rlimit = get_rlimit()
if os.path.exists(rlimit):
rlimit_args = [rlimit, "%d" % timeout]
else:
logger.warning("cannot find rlimit at %s", rlimit)
rlimit_args = []
try:
self.cmds = [
add_interpreter_command(c) for c in to_cmd_lines(cmds)
]
self.cmds[0] = rlimit_args + list(self.cmds[0])
cmdlogger.debug(
"Run: cd %s; %s",
cwd if cwd is not None else os.getcwd(),
self.command_line_image(),
)
if len(self.cmds) == 1:
popen_args = {
"stdin": self.input_file.fd,
"stdout": self.output_file.fd,
"stderr": self.error_file.fd,
"cwd": cwd,
"env": env,
"universal_newlines": False,
}
if sys.platform != "win32" and set_sigpipe: # windows: no cover
# preexec_fn is no supported on windows
popen_args["preexec_fn"] = subprocess_setup # type: ignore
if sys.platform == "win32":
popen_args[
"creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
self.internal = Popen(self.cmds[0], **popen_args)
else:
runs: list[subprocess.Popen] = []
for index, cmd in enumerate(self.cmds):
if index == 0:
stdin: int | IO[Any] = self.input_file.fd
else:
previous_stdout = runs[index - 1].stdout
assert previous_stdout is not None
stdin = previous_stdout
# When connecting two processes using a Pipe don't use
# universal_newlines mode. Indeed commands transmitting
# binary data between them will crash
# (e.g. gzip -dc foo.txt | tar -xf -)
if index == len(self.cmds) - 1:
stdout = self.output_file.fd
else:
stdout = subprocess.PIPE
popen_args = {
"stdin": stdin,
"stdout": stdout,
"stderr": self.error_file.fd,
"cwd": cwd,
"env": env,
"universal_newlines": False,
}
if sys.platform != "win32" and set_sigpipe: # windows: no cover
# preexec_fn is no supported on windows
popen_args[
"preexec_fn"] = subprocess_setup # type: ignore
if sys.platform == "win32":
popen_args[
"creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
try:
runs.append(Popen(cmd, **popen_args))
except OSError:
logger.error("error when spawning %s", cmd)
# We have an error (e.g. file not found), try to kill
# all processes already started.
for p in runs:
p.terminate()
raise
self.internal = runs[-1]
except Exception as e: # defensive code
self.__error(e, self.cmds)
raise
self.pid = self.internal.pid
if not bg:
self.wait()
@property
def out(self) -> str:
"""Process output as string.
Attempt is done to decode as utf-8 the output. If the output is not in
utf-8 a string representation will be returned
(see e3.text.bytes_as_str).
"""
return bytes_as_str(self.raw_out)
@property
def err(self) -> str:
"""Process error as string.
Attempt is done to decode as utf-8 the output. If the output is not in
utf-8 a string representation will be returned
(see e3.text.bytes_as_str).
"""
return bytes_as_str(self.raw_err)
def command_line_image(self) -> str:
"""Get shell command line image of the spawned command(s).
This just a convenient wrapper around the function of the same
name.
"""
return command_line_image(self.cmds)
def close_files(self) -> None:
"""Close all file descriptors."""
self.output_file.close()
self.error_file.close()
self.input_file.close()
def __error(self, error: Exception, cmds: list[CmdLine]) -> None:
"""Set pid to -1 and status to 127 before closing files."""
self.close_files()
logger.error(error)
def not_found(path: str) -> NoReturn:
"""Raise OSError.
:param path: path of the executable
"""
logger.error("%s not found", path)
e3.log.debug("PATH=%s", os.environ["PATH"])
raise OSError(errno.ENOENT,
f"No such file or directory, {path} not found")
# Try to send an helpful message if one of the executable has not
# been found.
for cmd in cmds:
if which(cmd[0], default=None) is None:
not_found(cmd[0])
def wait(self) -> int:
"""Wait until process ends and return its status.
:return: exit code of the process
"""
if self.status is not None:
# Wait has already been called
return self.status
# If there is no pipe in the loop then just do a wait. Otherwise
# in order to avoid blocked processes due to full pipes, use
# communicate.
if (self.output_file.fd != subprocess.PIPE
and self.error_file.fd != subprocess.PIPE
and self.input_file.fd != subprocess.PIPE):
self.status = self.internal.wait()
else:
tmp_input: Optional[str | bytes] = None
if self.input_file.fd == subprocess.PIPE:
tmp_input = self.input_file.get_command()
if isinstance(tmp_input, str):
tmp_input = tmp_input.encode("utf-8")
(self.raw_out, self.raw_err) = self.internal.communicate(tmp_input)
self.status = self.internal.returncode
self.close_files()
return self.status
def poll(self) -> Optional[int]:
"""Check the process status and set self.status if available.
This method checks whether the underlying process has exited
or not. If it hasn't, then it just returns None immediately.
Otherwise, it stores the process' exit code in self.status
and then returns it.
:return: None if the process is still alive; otherwise, returns
the process exit status.
"""
if self.status is not None:
# Process is already terminated and wait been called
return self.status
result = self.internal.poll()
if result is not None:
# Process is finished, call wait to finalize it (closing handles,
# ...)
return self.wait()
else:
return None
def kill(self, recursive: bool = True, timeout: int = 3) -> None:
"""Kill the process.
:param recursive: if True, try to kill the complete process tree
:param timeout: wait timeout (in seconds) after sending the kill
signal (when recursive=True)
"""
if recursive:
kill_process_tree(self.internal, timeout=timeout)
else:
self.internal.kill()
def interrupt(self) -> None:
"""Send SIGINT to the process, kill on Windows."""
if sys.platform == "win32":
self.kill() # Ctrl-C event is unreliable on Windows
else:
self.internal.send_signal(signal.SIGINT)
def is_running(self) -> bool:
"""Check whether the process is running."""
if psutil is None: # defensive code
# psutil not imported, use our is_running function
return is_running(self.pid)
else:
return self.internal.is_running()
def children(self) -> list[Any]:
"""Return list of child processes (using psutil)."""
if psutil is None: # defensive code
raise NotImplementedError("Run.children() require psutil")
return self.internal.children()
def sshcmd(self,
dst,
cmd,
inpt=None,
timeout=None,
log_cmd=True,
term=False,
display_errors=False,
parse_output=True): # noqa
if timeout is None:
timeout = conf['SSH_CONNECT_TIMEOUT']
output = []
errors = []
ex = [
'ssh', '-q', '-o UserKnownHostsFile=/dev/null',
'-o StrictHostKeyChecking=no',
'-o ConnectTimeout=%s' % timeout, dst
]
if term:
ex.insert(1, '-t')
ex.extend(cmd)
self.logger.debug("Execution list: %r", ex)
if log_cmd:
display_cmd(self.logger, ' '.join(ex))
sp = Popen(ex, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if inpt:
sp.stdin = subprocess.PIPE
out, err = sp.communicate(input=bytes(inpt, 'utf-8'))
else:
out, err = sp.communicate()
if parse_output and out:
output = [
self.replace_double_colon(
self.remove_ansi_esc_chars(
self.remove_non_printable_chars(l.strip())))
for l in out.decode('utf-8').split('\n') if len(l)
]
if log_cmd and len(output):
display_cmd_output(self.logger, output)
if err:
errors = [
l.strip() for l in err.decode('utf-8').split('\n')
if l.strip()
]
if display_errors:
display_cmd_output(self.logger, errors)
else:
self.logger.debug("Errors: %r", errors)
self.logger.debug("SSH CMD return code: %r", sp.returncode)
return sp.returncode, output, errors
