def popen(self, **kwargs):
"""Convenience method runs the command using Popen.
Args:
kwargs: passed through to Popen
Returns:
Popen instance
"""
if self._shell:
# on Windows, with shell=True Python interprets the args as NOT quoted
# and quotes them, but assumes a single string parameter is pre-quoted
# which is what we want.
# on Unix, with shell=True we interpret args[0]
# the same as a single string (it's the parameter after -c in sh -c)
# and anything after args[0] is passed as another flag to sh.
# (we never have anything after args[0])
# So if we always use the single string to popen when shell=True, things
# should work OK on all platforms.
assert len(self._args) == 1
args = self._args[0]
else:
args = self._args
return logged_subprocess.Popen(args=args,
env=py2_compat.env_without_unicode(self._env),
cwd=self._cwd,
shell=self._shell,
**kwargs)
def test_log_subprocess_Popen(monkeypatch):
recorded = dict(args=(), kwargs=())
def mock_Popen(*args, **kwargs):
recorded['args'] = args
recorded['kwargs'] = kwargs
monkeypatch.setattr('subprocess.Popen', mock_Popen)
logged_subprocess.Popen(['a', 'b'], foo='bar')
assert recorded == dict(args=(), kwargs=dict(args=['a', 'b'], foo='bar'))
def _call_pip(prefix, extra_args):
cmd_list = _get_pip_command(prefix, extra_args)
try:
p = logged_subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except OSError as e:
raise PipError("failed to run: %r: %r" % (" ".join(cmd_list), repr(e)))
(out, err) = p.communicate()
errstr = err.decode().strip()
if p.returncode != 0:
raise PipError('%s: %s' % (" ".join(cmd_list), errstr))
elif errstr != '':
for line in errstr.split("\n"):
print("%s %s: %s" % (cmd_list[0], cmd_list[1], line), file=sys.stderr)
return out
def test_log_subprocess_Popen_with_logging(monkeypatch):
logger = _test_logger()
verbose.push_verbose_logger(logger)
try:
recorded = dict(args=(), kwargs=())
def mock_Popen(*args, **kwargs):
recorded['args'] = args
recorded['kwargs'] = kwargs
monkeypatch.setattr('subprocess.Popen', mock_Popen)
logged_subprocess.Popen(['a', 'b'], foo='bar')
assert recorded == dict(args=(), kwargs=dict(args=['a', 'b'], foo='bar'))
assert logger.messages == ['$ a b']
finally:
verbose.pop_verbose_logger()
def _call_conda(extra_args, json_mode=False, platform=None):
assert len(extra_args) > 0 # we deref extra_args[0] below
(cmd_list,
command_in_errors) = _get_platform_hacked_conda_command(extra_args,
platform=platform)
try:
p = logged_subprocess.Popen(cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
raise CondaError("failed to run: %r: %r" %
(command_in_errors, repr(e)))
(out, err) = p.communicate()
errstr = err.decode().strip()
if p.returncode != 0:
parsed = None
message = errstr
if json_mode:
try:
parsed = json.loads(out.decode())
if parsed is not None and isinstance(parsed, dict):
# some versions of conda do 'error' and others
# both 'error' and 'message' and they appear to
# be the same.
for field in ('message', 'error'):
if field in parsed:
message = parsed[field]
break
except Exception:
pass
raise CondaError('%s: %s' % (command_in_errors, message), json=parsed)
elif errstr != '':
for line in errstr.split("\n"):
print("%s %s: %s" % ("conda", extra_args[0], line),
file=sys.stderr)
return out
def popen(args, stdout_callback, stderr_callback, **kwargs):
def ignore_line(line):
pass
if stdout_callback is None:
stdout_callback = ignore_line
if stderr_callback is None:
stderr_callback = ignore_line
p = logged_subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
queue = Queue()
# Create/destroy reader outside of the threads, since there
# have been threading bugs in related destructors such as TextIOWrapper
# https://bugs.python.org/issue28387
#
# In case the reader isn't thread-safe (likely) we only use it
# from one thread at a time, even though all creation/deletion
# is in this main thread, reading is always in the child
# threads.
#
# we use errors=replace primarily because with strict
# errors, TextIOWrapper can raise an exception
# "prematurely" (before returning all valid bytes).
# Arguably replace is nicer anyway for our purposes.
#
# We don't close these readers, because it seems to result
# in a double-close on the underlying file.
stdout_wrapper = codecs.getreader('utf-8')(p.stdout, errors='replace')
stderr_wrapper = codecs.getreader('utf-8')(p.stderr, errors='replace')
stdout_thread = _reader_thread(stdout_wrapper, queue)
stderr_thread = _reader_thread(stderr_wrapper, queue)
stdout_buffer = []
stderr_buffer = []
first_error = None
stdout_joined = False
stderr_joined = False
while not (queue.empty() and (stdout_joined and stderr_joined)):
(which, data, error) = queue.get()
if error is not None and first_error is None:
first_error = error
if data is None:
if which is stdout_wrapper:
stdout_thread.join()
stdout_joined = True
assert not stdout_thread.is_alive()
else:
assert which is stderr_wrapper
stderr_thread.join()
stderr_joined = True
assert not stderr_thread.is_alive()
else:
if which is stdout_wrapper:
stdout_callback(data)
stdout_buffer.append(data)
else:
assert which is stderr_wrapper
stderr_callback(data)
stderr_buffer.append(data)
assert queue.empty()
p.stdout.close()
p.stderr.close()
p.wait()
stdout_buffer = _combine_lines(stdout_buffer)
stderr_buffer = _combine_lines(stderr_buffer)
if first_error is not None:
raise first_error
return (p, stdout_buffer, stderr_buffer)
def ensure_redis(run_state):
# this is pretty lame, we'll want to get fancier at a
# future time (e.g. use Chalmers, stuff like
# that). The desired semantic is a new copy of Redis
# dedicated to this project directory; it should not
# require the user to have set up anything in advance,
# e.g. if we use Chalmers we should automatically take
# care of configuring/starting Chalmers itself.
url = context.status.analysis.existing_scoped_instance_url
if url is not None:
frontend.info(
"Using redis-server we started previously at {url}".format(
url=url))
return url
run_state.clear()
workdir = context.ensure_service_directory(requirement.env_var)
pidfile = os.path.join(workdir, "redis.pid")
logfile = os.path.join(workdir, "redis.log")
# 6379 is the default Redis port; leave that one free
# for a systemwide Redis. Try looking for a port above
# it. This is a pretty huge hack and a race condition,
# but Redis doesn't as far as I know have "let the OS
# pick the port" mode.
LOWER_PORT = config['lower_port']
UPPER_PORT = config['upper_port']
port = LOWER_PORT
while port <= UPPER_PORT:
if not network_util.can_connect_to_socket(host='localhost',
port=port):
break
port += 1
if port > UPPER_PORT:
frontend.error(
("All ports from {lower} to {upper} were in use, " +
"could not start redis-server on one of them.").format(
lower=LOWER_PORT, upper=UPPER_PORT))
return None
# be sure we don't get confused by an old log file
try:
os.remove(logfile)
except IOError: # pragma: no cover (py3 only)
pass
except OSError: # pragma: no cover (py2 only)
pass
command = [
'redis-server', '--pidfile', pidfile, '--logfile', logfile,
'--daemonize', 'yes', '--port',
str(port)
]
frontend.info("Starting " + repr(command))
# we don't close_fds=True because on Windows that is documented to
# keep us from collected stderr. But on Unix it's kinda broken not
# to close_fds. Hmm.
try:
popen = logged_subprocess.Popen(
args=command,
stderr=subprocess.PIPE,
env=py2_compat.env_without_unicode(context.environ))
except Exception as e:
frontend.error("Error executing redis-server: %s" % (str(e)))
return None
# communicate() waits for the process to exit, which
# is supposed to happen immediately due to --daemonize
(out, err) = popen.communicate()
assert out is None # because we didn't PIPE it
err = err.decode(errors='replace')
url = None
if popen.returncode == 0:
# now we need to wait for Redis to be ready; we
# are not sure whether it will create the port or
# pidfile first, so wait for both.
port_is_ready = False
pidfile_is_ready = False
MAX_WAIT_TIME = 10
so_far = 0
while so_far < MAX_WAIT_TIME:
increment = MAX_WAIT_TIME / 500.0
time.sleep(increment)
so_far += increment
if not port_is_ready:
if network_util.can_connect_to_socket(host='localhost',
port=port):
port_is_ready = True
if not pidfile_is_ready:
if os.path.exists(pidfile):
pidfile_is_ready = True
if port_is_ready and pidfile_is_ready:
break
# if we time out with no pidfile we forge ahead at this point
if port_is_ready:
run_state['port'] = port
url = "redis://localhost:{port}".format(port=port)
# note: --port doesn't work, only -p, and the failure with --port is silent.
run_state['shutdown_commands'] = [[
'redis-cli', '-p',
str(port), 'shutdown'
]]
else:
frontend.info(
"redis-server started successfully, but we timed out trying to connect to it on port %d"
% (port))
if url is None:
for line in err.split("\n"):
if line != "":
frontend.info(line)
try:
with codecs.open(logfile, 'r', 'utf-8') as log:
for line in log.readlines():
frontend.info(line)
except IOError as e:
# just be silent if redis-server failed before creating a log file,
# that's fine. Hopefully it had some stderr.
if e.errno != errno.ENOENT:
frontend.info(
"Failed to read {logfile}: {error}".format(
logfile=logfile, error=e))
frontend.error(
"redis-server process failed or timed out, exited with code {code}"
.format(code=popen.returncode))
return url
