def nice_local(command, nice=0, capture=False, shell=None):
"""
Exactly like Fabric's local but with an optional nice argument
"""
from fabric.operations import _prefix_env_vars, _prefix_commands, _AttributeString
from fabric.state import output, win32, env
from fabric.utils import error
given_command = command
# Apply cd(), path() etc
with_env = _prefix_env_vars(command, local=True)
wrapped_command = _prefix_commands(with_env, 'local')
if output.debug:
print("[localhost] local: %s" % (wrapped_command))
elif output.running:
print("[localhost] local: " + given_command)
# Tie in to global output controls as best we can; our capture argument
# takes precedence over the output settings.
dev_null = None
if capture:
out_stream = subprocess.PIPE
err_stream = subprocess.PIPE
else:
dev_null = open(os.devnull, 'w+')
# Non-captured, hidden streams are discarded.
out_stream = None if output.stdout else dev_null
err_stream = None if output.stderr else dev_null
try:
cmd_arg = wrapped_command if win32 else [wrapped_command]
p = subprocess.Popen(cmd_arg,
shell=True,
stdout=out_stream,
stderr=err_stream,
executable=shell,
preexec_fn=lambda: os.nice(nice),
close_fds=(not win32))
(stdout, stderr) = p.communicate()
finally:
if dev_null is not None:
dev_null.close()
# Handle error condition (deal with stdout being None, too)
out = _AttributeString(stdout.strip() if stdout else "")
err = _AttributeString(stderr.strip() if stderr else "")
out.command = given_command
out.real_command = wrapped_command
out.failed = False
out.return_code = p.returncode
out.stderr = err
if p.returncode not in env.ok_ret_codes:
out.failed = True
msg = "local() encountered an error (return code %s) while executing '%s'" % (
p.returncode, command)
error(message=msg, stdout=out, stderr=err)
out.succeeded = not out.failed
# If we were capturing, this will be a string; otherwise it will be None.
return out
def _run_host_command(command, shell=True, pty=True, combine_stderr=True):
"""
Run host wrapper command as root
(Modified from fabric.operations._run_command to ignore prefixes,
path(), cd(), and always use sudo.)
"""
# Set up new var so original argument can be displayed verbatim later.
given_command = command
# Handle context manager modifications, and shell wrapping
wrapped_command = _shell_wrap(
command,
shell,
_sudo_prefix(None)
)
# Execute info line
if output.debug:
print("[%s] %s: %s" % (env.host_string, 'sudo', wrapped_command))
elif output.running:
print("[%s] %s: %s" % (env.host_string, 'sudo', given_command))
# Actual execution, stdin/stdout/stderr handling, and termination
stdout, stderr, status = _execute(default_channel(), wrapped_command, pty,
combine_stderr)
# Assemble output string
out = _AttributeString(stdout)
err = _AttributeString(stderr)
# Error handling
out.failed = False
if status != 0:
out.failed = True
msg = "%s() received nonzero return code %s while executing" % (
'sudo', status
)
if env.warn_only:
msg += " '%s'!" % given_command
else:
msg += "!\n\nRequested: %s\nExecuted: %s" % (
given_command, wrapped_command
)
error(message=msg, stdout=out, stderr=err)
# Attach return code to output string so users who have set things to
# warn only, can inspect the error code.
out.return_code = status
# Convenience mirror of .failed
out.succeeded = not out.failed
# Attach stderr for anyone interested in that.
out.stderr = err
return out
def _run_host_command(command, shell=True, pty=True, combine_stderr=True):
"""
Run host wrapper command as root
(Modified from fabric.operations._run_command to ignore prefixes,
path(), cd(), and always use sudo.)
"""
# Set up new var so original argument can be displayed verbatim later.
given_command = command
# Handle context manager modifications, and shell wrapping
wrapped_command = _shell_wrap(
command,
shell,
_sudo_prefix(None)
)
# Execute info line
if output.debug:
print("[%s] %s: %s" % (env.host_string, 'sudo', wrapped_command))
elif output.running:
print("[%s] %s: %s" % (env.host_string, 'sudo', given_command))
# Actual execution, stdin/stdout/stderr handling, and termination
stdout, stderr, status = _execute(default_channel(), wrapped_command, pty,
combine_stderr)
# Assemble output string
out = _AttributeString(stdout)
err = _AttributeString(stderr)
# Error handling
out.failed = False
if status != 0:
out.failed = True
msg = "%s() received nonzero return code %s while executing" % (
'sudo', status
)
if env.warn_only:
msg += " '%s'!" % given_command
else:
msg += "!\n\nRequested: %s\nExecuted: %s" % (
given_command, wrapped_command
)
error(message=msg, stdout=out, stderr=err)
# Attach return code to output string so users who have set things to
# warn only, can inspect the error code.
out.return_code = status
# Convenience mirror of .failed
out.succeeded = not out.failed
# Attach stderr for anyone interested in that.
out.stderr = err
return out
def test_uninstall_is_called(self, mock_sudo, mock_version_check):
env.host = "any_host"
output1 = _AttributeString()
output1.succeeded = False
output2 = _AttributeString()
output2.succeeded = True
mock_sudo.side_effect = [output1, output2]
server.uninstall()
mock_version_check.assert_called_with()
mock_sudo.assert_any_call('rpm -e presto')
mock_sudo.assert_called_with('rpm -e presto-server-rpm')
def test_uninstall_is_called(self, mock_sudo, mock_version_check):
env.host = "any_host"
output1 = _AttributeString()
output1.succeeded = False
output2 = _AttributeString()
output2.succeeded = True
mock_sudo.side_effect = [output1, output2]
server.uninstall()
mock_version_check.assert_called_with()
mock_sudo.assert_any_call('rpm -e presto')
mock_sudo.assert_called_with('rpm -e presto-server-rpm')
def test_multiple_version_rpms(self, mock_run):
err_msg = 'Presto is not installed.'
output1 = _AttributeString(err_msg)
output1.succeeded = False
output2 = _AttributeString('0.111.SNAPSHOT')
output2.succeeded = True
mock_run.side_effect = [output1, output2]
expected = server.check_presto_version()
mock_run.assert_any_call('rpm -q --qf \"%{VERSION}\\n\" presto')
mock_run.assert_called_with(
'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm')
self.assertEqual(expected, '')
def test_lookup_string_config_not_in_file(self, run_mock):
run_mock.return_value = _AttributeString('')
run_mock.return_value.failed = False
run_mock.return_value.return_code = 1
config_value = lookup_string_config('config.to.lookup',
NODE_CONFIG_FILE, 'any_host')
self.assertEqual(config_value, '')
def test_td_presto_version(self, mock_run):
td_version = '101t'
output = _AttributeString(td_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
def test_lookup_string_config_not_in_file(self, run_mock):
run_mock.return_value = _AttributeString('')
run_mock.return_value.failed = False
run_mock.return_value.return_code = 1
config_value = lookup_string_config('config.to.lookup',
NODE_CONFIG_FILE, 'any_host')
self.assertEqual(config_value, '')
def test_td_presto_version(self, mock_run):
td_version = '101t'
output = _AttributeString(td_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
def test_lookup_port_not_integer_failure(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=hello')
run_mock.return_value.failed = False
self.assertRaisesRegexp(
ConfigurationError,
'Unable to coerce http-server.http.port \'hello\' to an int. '
'Failed to connect to any_host.', lookup_port, 'any_host')
def run_local(command, sudo=False, shell=True, pty=True, combine_stderr=None):
"""Local implementation of fabric.api.run() using subprocess.
.. note:: pty option exists for function signature compatibility and is
ignored.
"""
if combine_stderr is None:
combine_stderr = env.combine_stderr
# TODO: Pass the SUDO_PASSWORD variable to the command here
if sudo:
command = "sudo " + command
mico.output.debug(command)
stderr = subprocess.STDOUT if combine_stderr else subprocess.PIPE
process = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, stderr=stderr)
out, err = process.communicate()
# FIXME: Should stream the output, and only print it if fabric's properties allow it
# print out
# Wrap stdout string and add extra status attributes
result = operations._AttributeString(out.rstrip('\n'))
result.return_code = process.returncode
result.succeeded = (process.returncode == 0)
result.failed = not result.succeeded
result.stderr = StringIO(err)
return result
def test_multiple_version_rpms(self, mock_run):
output1 = _AttributeString('package presto is not installed')
output1.succeeded = False
output2 = _AttributeString('presto-server-rpm-0.115t-1.x86_64')
output2.succeeded = True
output3 = _AttributeString('Presto is not installed.')
output3.succeeded = False
output4 = _AttributeString('0.111.SNAPSHOT')
output4.succeeded = True
mock_run.side_effect = [output1, output2, output3, output4]
expected = server.check_presto_version()
mock_run.assert_has_calls(
[call('rpm -q presto'),
call('rpm -q presto-server-rpm')])
self.assertEqual(expected, '')
def test_lookup_string_config(self, sudo_mock):
sudo_mock.return_value = _AttributeString(
'config.to.lookup=/path/hello')
sudo_mock.return_value.failed = False
sudo_mock.return_value.return_code = 0
config_value = lookup_string_config('config.to.lookup',
NODE_CONFIG_FILE, 'any_host')
self.assertEqual(config_value, '/path/hello')
def test_init_success(self):
fabric_result = _AttributeString('Success')
fabric_result.succeeded = True
fabric_result.return_code = 0
success = Result('tox', result=fabric_result)
self.assertTrue(success.succeeded)
self.assertEquals(success.log, 'Success')
self.assertEquals(success.task, 'tox')
def test_init_failure(self):
fabric_result = _AttributeString('Oh snap')
fabric_result.succeeded = False
fabric_result.return_code = 1
failure = Result('tox', fabric_result)
self.assertFalse(failure.succeeded)
self.assertEquals(failure.log, 'Oh snap')
self.assertEquals(failure.task, 'tox')
def _blocal(command, capture=False):
"""
Slightly modified version of fabrics 'local' function designed
to make the underlying subprocess.Popen call use bash.
Taken from fabric operations.py file.
"""
given_command = command
# Apply cd(), path() etc
wrapped_command = _prefix_commands(_prefix_env_vars(command), 'local')
if output.debug:
print("[localhost] local: %s" % (wrapped_command))
elif output.running:
print("[localhost] local: " + given_command)
# Tie in to global output controls as best we can; our capture argument
# takes precedence over the output settings.
dev_null = None
if capture:
out_stream = subprocess.PIPE
err_stream = subprocess.PIPE
else:
dev_null = open(os.devnull, 'w+')
# Non-captured, hidden streams are discarded.
out_stream = None if output.stdout else dev_null
err_stream = None if output.stderr else dev_null
try:
cmd_arg = wrapped_command if win32 else [wrapped_command]
p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream,
stderr=err_stream,executable="/bin/bash")
(stdout, stderr) = p.communicate()
finally:
if dev_null is not None:
dev_null.close()
# Handle error condition (deal with stdout being None, too)
out = _AttributeString(stdout.strip() if stdout else "")
err = _AttributeString(stderr.strip() if stderr else "")
out.failed = False
out.return_code = p.returncode
out.stderr = err
if p.returncode != 0:
out.failed = True
msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command)
fabric.utils.error(message=msg, stdout=out, stderr=err)
out.succeeded = not out.failed
# If we were capturing, this will be a string; otherwise it will be None.
return out
def test_lookup_port_out_of_range(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=99999')
run_mock.return_value.failed = False
self.assertRaisesRegexp(
ConfigurationError,
'Invalid port number 99999: port must be between 1 and 65535 '
'for property http-server.http.port on host any_host.',
lookup_port, 'any_host')
def test_lookup_port_out_of_range(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=99999')
run_mock.return_value.failed = False
self.assertRaisesRegexp(
ConfigurationError,
'Invalid port number 99999: port must be between 1 and 65535 '
'for property http-server.http.port on host any_host.',
lookup_port, 'any_host'
)
def test_remove_failure(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
out = _AttributeString()
out.succeeded = False
sudo_mock.return_value = out
self.assertRaisesRegexp(
SystemExit, '\\[localhost\\] Failed to remove '
'connector tpch.', connector.remove, 'tpch')
def test_lookup_port_not_integer_failure(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=hello')
run_mock.return_value.failed = False
self.assertRaisesRegexp(
ConfigurationError,
'Unable to coerce http-server.http.port \'hello\' to an int. '
'Failed to connect to any_host.',
lookup_port, 'any_host'
)
def test_multiple_version_rpms(self, mock_run):
output1 = _AttributeString('package presto is not installed')
output1.succeeded = False
output2 = _AttributeString('presto-server-rpm-0.115t-1.x86_64')
output2.succeeded = True
output3 = _AttributeString('Presto is not installed.')
output3.succeeded = False
output4 = _AttributeString('0.111.SNAPSHOT')
output4.succeeded = True
mock_run.side_effect = [output1, output2, output3, output4]
expected = server.check_presto_version()
mock_run.assert_has_calls([
call('rpm -q presto'),
call('rpm -q presto-server-rpm')
])
self.assertEqual(expected, '')
def test_remove_no_such_file(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
error_msg = ('Could not remove catalog tpch: No such file ' +
os.path.join(get_catalog_directory(), 'tpch.properties'))
out = _AttributeString(error_msg)
out.succeeded = True
sudo_mock.return_value = out
self.assertRaisesRegexp(SystemExit, '\\[localhost\\] %s' % error_msg,
catalog.remove, 'tpch')
def test_lookup_port_failure(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=8080')
run_mock.return_value.failed = True
self.assertRaisesRegexp(
ConfigurationError,
'Configuration file /etc/presto/config.properties does not exist '
'on host any_host',
lookup_port, 'any_host'
)
def test_lookup_string_config_file_not_found(self, run_mock):
run_mock.return_value = _AttributeString(
'grep: /etc/presto/node.properties does not exist')
run_mock.return_value.return_code = 2
self.assertRaisesRegexp(
ConfigurationError,
'Configuration file /etc/presto/node.properties does not exist '
'on host any_host', lookup_string_config, 'config.to.lookup',
NODE_CONFIG_FILE, 'any_host')
def test_lookup_port_not_integer_failure(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=hello')
run_mock.return_value.failed = False
run_mock.return_value.return_code = 0
self.assertRaisesRegexp(
ConfigurationError,
'Invalid port number hello: port must be a number between 1 and'
' 65535 for property http-server.http.port on host any_host.',
lookup_port, 'any_host')
def test_remove_failure(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
out = _AttributeString()
out.succeeded = False
sudo_mock.return_value = out
self.assertRaisesRegexp(SystemExit,
'\\[localhost\\] Failed to remove catalog tpch.',
catalog.remove,
'tpch')
def test_remove_failure(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
out = _AttributeString()
out.succeeded = False
sudo_mock.return_value = out
connector.remove('tpch')
self.assertEqual('\nWarning: [localhost] Failed to remove connector '
'tpch.\n\t\n\n',
self.test_stderr.getvalue())
def test_remove_no_such_file(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
error_msg = ('Could not remove connector tpch: No such file '
'/etc/opt/prestoadmin/connectors/tpch.properties')
out = _AttributeString(error_msg)
out.succeeded = True
sudo_mock.return_value = out
self.assertRaisesRegexp(SystemExit, '\\[localhost\\] %s' % error_msg,
connector.remove, 'tpch')
def test_lookup_string_config_file_not_found(self, sudo_mock):
sudo_mock.return_value = _AttributeString(
'grep: /etc/presto/node.properties does not exist')
sudo_mock.return_value.return_code = 2
self.assertRaisesRegexp(
ConfigurationError,
'Could not access config file /etc/presto/node.properties on host any_host',
lookup_string_config, 'config.to.lookup', NODE_CONFIG_FILE,
'any_host'
)
def test_lookup_port_not_integer_failure(self, run_mock):
run_mock.return_value = _AttributeString('http-server.http.port=hello')
run_mock.return_value.failed = False
run_mock.return_value.return_code = 0
self.assertRaisesRegexp(
ConfigurationError,
'Invalid port number hello: port must be a number between 1 and'
' 65535 for property http-server.http.port on host any_host.',
lookup_port, 'any_host'
)
def test_remove_no_such_file(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
error_msg = ('Could not remove connector tpch: No such file '
'/etc/opt/prestoadmin/connectors/tpch.properties')
out = _AttributeString(error_msg)
out.succeeded = True
sudo_mock.return_value = out
connector.remove('tpch')
self.assertEqual('\nWarning: [localhost] %s\n\n' % error_msg,
self.test_stderr.getvalue())
def test_remove_no_such_file(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
error_msg = ('Could not remove connector tpch: No such file '
'/etc/opt/prestoadmin/connectors/tpch.properties')
out = _AttributeString(error_msg)
out.succeeded = True
sudo_mock.return_value = out
self.assertRaisesRegexp(SystemExit,
'\\[localhost\\] %s' % error_msg,
connector.remove,
'tpch')
def test_remove_no_such_file(self, exists_mock, sudo_mock):
exists_mock.return_value = False
fabric.api.env.host = 'localhost'
error_msg = ('Could not remove catalog tpch: No such file ' +
os.path.join(get_catalog_directory(), 'tpch.properties'))
out = _AttributeString(error_msg)
out.succeeded = True
sudo_mock.return_value = out
self.assertRaisesRegexp(SystemExit,
'\\[localhost\\] %s' % error_msg,
catalog.remove,
'tpch')
def test_rpm_upgrade(self, mock_sudo, mock_rpm_upgrade):
env.host = 'any_host'
env.nodeps = False
mock_sudo.return_value = _AttributeString('test_package_name')
mock_sudo.return_value.succeeded = True
package.rpm_upgrade('test.rpm')
mock_sudo.assert_any_call('rpm -qp --queryformat \'%{NAME}\' '
'/opt/prestoadmin/packages/test.rpm',
quiet=True)
mock_rpm_upgrade.assert_any_call('/opt/prestoadmin/packages/test.rpm')
def test_execute_query_get_port(self, run_mock, conn_mock):
client = PrestoClient('any_host', 'any_user')
client.rows = ['hello']
client.next_uri = 'hello'
client.response_from_server = {'hello': 'hello'}
run_mock.return_value = _AttributeString('http-server.http.port=8080')
run_mock.return_value.failed = False
client.execute_query('select * from nation')
self.assertEqual(client.port, 8080)
self.assertEqual(client.rows, [])
self.assertEqual(client.next_uri, '')
self.assertEqual(client.response_from_server, {})
def testrun_sql_get_port(self, sudo_mock, conn_mock, mock_presto_config):
client = PrestoClient('any_host', 'any_user')
client.rows = ['hello']
client.next_uri = 'hello'
client.response_from_server = {'hello': 'hello'}
sudo_mock.return_value = _AttributeString('http-server.http.port=8080')
sudo_mock.return_value.failed = False
sudo_mock.return_value.return_code = 0
client.run_sql('select * from nation')
self.assertEqual(client.port, 8080)
self.assertEqual(client.rows, [])
self.assertEqual(client.next_uri, '')
self.assertEqual(client.response_from_server, {})
def testrun_sql_get_port(self, sudo_mock, conn_mock, mock_presto_config):
client = PrestoClient('any_host', 'any_user')
client.rows = ['hello']
client.next_uri = 'hello'
client.response_from_server = {'hello': 'hello'}
sudo_mock.return_value = _AttributeString('http-server.http.port=8080')
sudo_mock.return_value.failed = False
sudo_mock.return_value.return_code = 0
client.run_sql('select * from nation')
self.assertEqual(client.port, 8080)
self.assertEqual(client.rows, [])
self.assertEqual(client.next_uri, '')
self.assertEqual(client.response_from_server, {})
def test_version_with_snapshot(self, mock_run):
snapshot_version = '0.107.SNAPSHOT'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
snapshot_version = '0.107.SNAPSHOT-1.x86_64'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
snapshot_version = '0.107-SNAPSHOT'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
def test_version_with_snapshot(self, mock_run):
snapshot_version = '0.107.SNAPSHOT'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
snapshot_version = '0.107.SNAPSHOT-1.x86_64'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
snapshot_version = '0.107-SNAPSHOT'
output = _AttributeString(snapshot_version)
output.succeeded = True
mock_run.return_value = output
expected = server.check_presto_version()
self.assertEqual(expected, '')
def test_rpm_upgrade(self, mock_sudo, mock_rpm_upgrade):
env.host = 'any_host'
env.nodeps = False
mock_sudo.return_value = _AttributeString('test_package_name')
mock_sudo.return_value.succeeded = True
package.rpm_upgrade('test.rpm')
mock_sudo.assert_any_call(
'rpm -qp --queryformat \'%{NAME}\' '
'/opt/prestoadmin/packages/test.rpm',
quiet=True)
mock_rpm_upgrade.assert_any_call('/opt/prestoadmin/packages/test.rpm')
def test_warning_presto_version_not_installed(self, mock_warn, mock_run,
mock_run_sql, mock_presto_config):
env.host = 'node1'
env.roledefs['coordinator'] = ['node1']
env.roledefs['worker'] = ['node1']
env.roledefs['all'] = ['node1']
env.hosts = env.roledefs['all']
output = _AttributeString('package presto is not installed')
output.succeeded = False
mock_run.return_value = output
env.host = 'node1'
server.collect_node_information()
installation_warning = 'Presto is not installed.'
mock_warn.assert_called_with(installation_warning)
def test_warning_presto_version_not_installed(self, mock_warn, mock_run,
mock_run_sql):
env.host = 'node1'
env.roledefs['coordinator'] = ['node1']
env.roledefs['worker'] = ['node1']
env.roledefs['all'] = ['node1']
env.hosts = env.roledefs['all']
output = _AttributeString('package presto is not installed')
output.succeeded = False
mock_run.return_value = output
env.host = 'node1'
server.collect_node_information()
installation_warning = 'Presto is not installed.'
mock_warn.assert_called_with(installation_warning)
def test_warning_presto_version_wrong(self, mock_warn, mock_run,
mock_run_sql):
env.host = 'node1'
env.roledefs['coordinator'] = ['node1']
env.roledefs['worker'] = ['node1']
env.roledefs['all'] = ['node1']
env.hosts = env.roledefs['all']
old_version = '0.97'
output = _AttributeString(old_version)
output.succeeded = True
mock_run.return_value = output
server.collect_node_information()
version_warning = 'Presto version is %s, version >= 0.%d required.'\
% (old_version, PRESTO_RPM_MIN_REQUIRED_VERSION)
mock_warn.assert_called_with(version_warning)
def test_is_pip_installed_pythonbrew(self, mock_run):
from fabric.operations import _AttributeString
from fabtools.python import is_pip_installed
fake_result = _AttributeString(
'\x1b[32mSwitched to Python-2.7.5'
'\x1b[0mpip 1.4.1 from /home/vagrant/.pythonbrew/pythons/Python-2.7.5/lib/python2.7/site-packages/pip-1.4.1-py2.7.egg (python 2.7)'
)
fake_result.failed = False
fake_result.succeeded = True
mock_run.return_value = fake_result
res = is_pip_installed(version='1.3.1')
self.assertTrue(res)
def test_warning_presto_version_wrong(self, mock_warn, mock_run,
mock_run_sql):
env.host = 'node1'
env.roledefs['coordinator'] = ['node1']
env.roledefs['worker'] = ['node1']
env.roledefs['all'] = ['node1']
env.hosts = env.roledefs['all']
old_version = '0.97'
output = _AttributeString(old_version)
output.succeeded = True
mock_run.return_value = output
server.collect_node_information()
version_warning = 'Presto version is %s, version >= 0.%d required.'\
% (old_version, PRESTO_RPM_MIN_REQUIRED_VERSION)
mock_warn.assert_called_with(version_warning)
def test_is_pip_installed_pythonbrew(self, mock_run):
from fabric.operations import _AttributeString
from fabtools.python import is_pip_installed
fake_result = _AttributeString(
'\x1b[32mSwitched to Python-2.7.5'
'\x1b[0mpip 1.4.1 from /home/vagrant/.pythonbrew/pythons/Python-2.7.5/lib/python2.7/site-packages/pip-1.4.1-py2.7.egg (python 2.7)'
)
fake_result.failed = False
fake_result.succeeded = True
mock_run.return_value = fake_result
res = is_pip_installed(version='1.3.1')
self.assertTrue(res)
def test_rpm_upgrade_nodeps(self, mock_sudo):
env.host = 'any_host'
env.nodeps = True
mock_sudo.return_value = _AttributeString('/opt/prestoadmin/packages/'
'test.rpm')
mock_sudo.return_value.succeeded = True
package.rpm_upgrade('test.rpm')
mock_sudo.assert_any_call('rpm -qp --queryformat \'%{NAME}\' '
'/opt/prestoadmin/packages/test.rpm',
quiet=True)
mock_sudo.assert_any_call('rpm -i --nodeps '
'/opt/prestoadmin/packages/test.rpm')
mock_sudo.assert_any_call('rpm -e --nodeps '
'/opt/prestoadmin/packages/test.rpm')
self.assertEqual(3, mock_sudo.call_count)
def _run_host_command(command,
shell=True,
pty=True,
combine_stderr=True,
quiet=False,
warn_only=False,
stdout=None,
stderr=None,
timeout=None):
"""
Run host wrapper command as root
(Modified from fabric.operations._run_command to ignore prefixes,
path(), cd(), and always use sudo.)
"""
manager = _noop
if warn_only:
manager = warn_only_manager
# Quiet's behavior is a superset of warn_only's, so it wins.
if quiet:
manager = quiet_manager
with manager():
# Set up new var so original argument can be displayed verbatim later.
given_command = command
# Handle context manager modifications, and shell wrapping
wrapped_command = _shell_wrap(
command, # !! removed _prefix_commands() & _prefix_env_vars()
shell,
_sudo_prefix(None) # !! always use sudo
)
# Execute info line
which = 'sudo' # !! always use sudo
if output.debug:
print(("[%s] %s: %s" % (env.host_string, which, wrapped_command)))
elif output.running:
print(("[%s] %s: %s" % (env.host_string, which, given_command)))
# Actual execution, stdin/stdout/stderr handling, and termination
result_stdout, result_stderr, status = _execute(
channel=default_channel(),
command=wrapped_command,
pty=pty,
combine_stderr=combine_stderr,
invoke_shell=False,
stdout=stdout,
stderr=stderr,
timeout=timeout)
# Assemble output string
out = _AttributeString(result_stdout)
err = _AttributeString(result_stderr)
# Error handling
out.failed = False
out.command = given_command
out.real_command = wrapped_command
if status not in env.ok_ret_codes:
out.failed = True
msg = "%s() received nonzero return code %s while executing" % (
which, status)
if env.warn_only:
msg += " '%s'!" % given_command
else:
msg += "!\n\nRequested: %s\nExecuted: %s" % (given_command,
wrapped_command)
error(message=msg, stdout=out, stderr=err)
# Attach return code to output string so users who have set things to
# warn only, can inspect the error code.
out.return_code = status
# Convenience mirror of .failed
out.succeeded = not out.failed
# Attach stderr for anyone interested in that.
out.stderr = err
return out
def get_mock_ssh_text(text, code):
result = _AttributeString(text)
result.return_code = code
return result
def mock_fail_then_succeed(self):
output1 = _AttributeString()
output1.succeeded = False
output2 = _AttributeString()
output2.succeeded = True
return [output1, output2]
def local(command, capture=False, shell=None, ignore_errors=False, env=None):
"""
Run a command on the local system.
`local` is simply a convenience wrapper around the use of the builtin
Python ``subprocess`` module with ``shell=True`` activated. If you need to
do anything special, consider using the ``subprocess`` module directly.
``shell`` is passed directly to `subprocess.Popen
<http://docs.python.org/library/subprocess.html#subprocess.Popen>`_'s
``execute`` argument (which determines the local shell to use.) As per the
linked documentation, on Unix the default behavior is to use ``/bin/sh``,
so this option is useful for setting that value to e.g. ``/bin/bash``.
`local` is not currently capable of simultaneously printing and
capturing output, as `~fabric.operations.run`/`~fabric.operations.sudo`
do. The ``capture`` kwarg allows you to switch between printing and
capturing as necessary, and defaults to ``False``.
When ``capture=False``, the local subprocess' stdout and stderr streams are
hooked up directly to your terminal, though you may use the global
:doc:`output controls </usage/output_controls>` ``output.stdout`` and
``output.stderr`` to hide one or both if desired. In this mode, the return
value's stdout/stderr values are always empty.
When ``capture=True``, you will not see any output from the subprocess in
your terminal, but the return value will contain the captured
stdout/stderr.
In either case, as with `~fabric.operations.run` and
`~fabric.operations.sudo`, this return value exhibits the ``return_code``,
``stderr``, ``failed``, ``succeeded``, ``command`` and ``real_command``
attributes. See `run` for details.
`~fabric.operations.local` will honor the `~fabric.context_managers.lcd`
context manager, allowing you to control its current working directory
independently of the remote end (which honors
`~fabric.context_managers.cd`).
.. versionchanged:: 1.0
Added the ``succeeded`` and ``stderr`` attributes.
.. versionchanged:: 1.0
Now honors the `~fabric.context_managers.lcd` context manager.
.. versionchanged:: 1.0
Changed the default value of ``capture`` from ``True`` to ``False``.
.. versionadded:: 1.9
The return value attributes ``.command`` and ``.real_command``.
"""
given_command = command
# Apply cd(), path() etc
with_env = _prefix_env_vars(command, local=True)
wrapped_command = _prefix_commands(with_env, 'local')
# if output.debug:
# print("[localhost] local: %s" % (wrapped_command))
# elif output.running:
# print("[localhost] local: " + given_command)
# Tie in to global output controls as best we can; our capture argument
# takes precedence over the output settings.
dev_null = None
if capture:
out_stream = subprocess.PIPE
err_stream = subprocess.PIPE
else:
dev_null = open(os.devnull, 'w+')
# Non-captured, hidden streams are discarded.
out_stream = None if output.stdout else dev_null
err_stream = None if output.stderr else dev_null
if env is None:
env = os.environ
try:
cmd_arg = wrapped_command if win32 else [wrapped_command]
p = subprocess.Popen(cmd_arg,
shell=True,
stdout=out_stream,
stderr=err_stream,
executable=shell,
close_fds=(not win32),
env=env)
(stdout, stderr) = p.communicate()
finally:
if dev_null is not None:
dev_null.close()
# Handle error condition (deal with stdout being None, too)
out = _AttributeString(stdout.decode().strip() if stdout else "")
err = _AttributeString(stderr.decode().strip() if stderr else "")
out.command = given_command
out.real_command = wrapped_command
out.failed = False
out.return_code = p.returncode
out.stderr = err
if p.returncode not in _env.ok_ret_codes and not ignore_errors:
out.failed = True
# msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command)
# print('\n\n')
# if out:
# print('{}'.format(out))
# print('\n')
# if err:
# print('{}'.format(err))
# sys.exit(1)
raise CommandError(out)
# error(message=msg, stdout=out, stderr=err)
out.succeeded = not out.failed
# If we were capturing, this will be a string; otherwise it will be None.
return out
def handle_complex_command(self, command, job_name=""):
"""
Wrap subprocess.Popen in such a way that the stderr and stdout from running a shell command will
be captured and logged in nearly real time. This is similar to fabric.local, but allows us to
retain control over the process. This method is named "complex" because it uses queues and
threads to execute a command while capturing and displaying the output.
"""
# We define a "local logger" here such that we can give it a slightly
# different name. We use the package name as part of the logger to
# allow admins to easily distinguish between which package is currently
# being installed.
llog_name = __name__
if len(job_name) > 0:
llog_name += ':' + job_name
llog = logging.getLogger(llog_name)
# Print the command we're about to execute, ``set -x`` style.
llog.debug('+ ' + str(command))
# Launch the command as subprocess. A bufsize of 1 means line buffered.
process_handle = subprocess.Popen(str(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1,
close_fds=False,
shell=True,
cwd=state.env['lcwd'])
pid = process_handle.pid
# Launch the asynchronous readers of the process' stdout and stderr.
stdout_queue = queue.Queue()
stdout_reader = asynchronous_reader.AsynchronousReader(
process_handle.stdout, stdout_queue)
stdout_reader.start()
stderr_queue = queue.Queue()
stderr_reader = asynchronous_reader.AsynchronousReader(
process_handle.stderr, stderr_queue)
stderr_reader.start()
# Place streamed stdout and stderr into a threaded IPC queue target so it can
# be printed and stored for later retrieval when generating the INSTALLATION.log.
stdio_thread = threading.Thread(
target=self.enqueue_output,
args=(process_handle.stdout, stdout_queue, process_handle.stderr,
stderr_queue))
thread_lock = threading.Lock()
thread_lock.acquire()
stdio_thread.start()
# Check the queues for output until there is nothing more to get.
start_timer = time.time()
while not stdout_reader.installation_complete(
) or not stderr_reader.installation_complete():
# Show what we received from standard output.
while not stdout_queue.empty():
try:
line = stdout_queue.get()
except queue.Empty:
line = None
break
if line:
llog.debug(line)
start_timer = time.time()
else:
break
# Show what we received from standard error.
while not stderr_queue.empty():
try:
line = stderr_queue.get()
except queue.Empty:
line = None
break
if line:
llog.debug(line)
start_timer = time.time()
else:
stderr_queue.task_done()
break
# Sleep a bit before asking the readers again.
time.sleep(.1)
current_wait_time = time.time() - start_timer
if stdout_queue.empty() and stderr_queue.empty(
) and current_wait_time > basic_util.NO_OUTPUT_TIMEOUT:
err_msg = "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" % \
(pid, basic_util.NO_OUTPUT_TIMEOUT)
stderr_reader.lines.append(err_msg)
process_handle.kill()
break
thread_lock.release()
# Wait until each of the threads we've started terminate. The following calls will block each thread
# until it terminates either normally, through an unhandled exception, or until the timeout occurs.
stdio_thread.join(basic_util.NO_OUTPUT_TIMEOUT)
stdout_reader.join(basic_util.NO_OUTPUT_TIMEOUT)
stderr_reader.join(basic_util.NO_OUTPUT_TIMEOUT)
# Close subprocess' file descriptors.
self.close_file_descriptor(process_handle.stdout)
self.close_file_descriptor(process_handle.stderr)
stdout = '\n'.join(stdout_reader.lines)
stderr = '\n'.join(stderr_reader.lines)
# Handle error condition (deal with stdout being None, too)
output = _AttributeString(stdout.strip() if stdout else "")
errors = _AttributeString(stderr.strip() if stderr else "")
# Make sure the process has finished.
process_handle.poll()
output.return_code = process_handle.returncode
output.stderr = errors
return output
def bash_local(command, capture=False):
"""
Run a command on the local system.
`local` is simply a convenience wrapper around the use of the builtin
Python ``subprocess`` module with ``shell=True`` activated. If you need to
do anything special, consider using the ``subprocess`` module directly.
`local` is not currently capable of simultaneously printing and
capturing output, as `~fabric.operations.run`/`~fabric.operations.sudo`
do. The ``capture`` kwarg allows you to switch between printing and
capturing as necessary, and defaults to ``False``.
When ``capture=False``, the local subprocess' stdout and stderr streams are
hooked up directly to your terminal, though you may use the global
:doc:`output controls </usage/output_controls>` ``output.stdout`` and
``output.stderr`` to hide one or both if desired. In this mode,
`~fabric.operations.local` returns None.
When ``capture=True``, this function will return the contents of the
command's stdout as a string-like object; as with `~fabric.operations.run`
and `~fabric.operations.sudo`, this return value exhibits the
``return_code``, ``stderr``, ``failed`` and ``succeeded`` attributes. See
`run` for details.
`~fabric.operations.local` will honor the `~fabric.context_managers.lcd`
context manager, allowing you to control its current working directory
independently of the remote end (which honors
`~fabric.context_managers.cd`).
.. versionchanged:: 1.0
Added the ``succeeded`` and ``stderr`` attributes.
.. versionchanged:: 1.0
Now honors the `~fabric.context_managers.lcd` context manager.
.. versionchanged:: 1.0
Changed the default value of ``capture`` from ``True`` to ``False``.
"""
given_command = command
# Apply cd(), path() etc
wrapped_command = _prefix_commands(_prefix_env_vars(command), 'local')
if output.debug:
print("[localhost] local: %s" % (wrapped_command))
elif output.running:
print("[localhost] local: " + given_command)
# Tie in to global output controls as best we can; our capture argument
# takes precedence over the output settings.
dev_null = None
if capture:
out_stream = subprocess.PIPE
err_stream = subprocess.PIPE
else:
dev_null = open(os.devnull, 'w+')
# Non-captured, hidden streams are discarded.
out_stream = None if output.stdout else dev_null
err_stream = None if output.stderr else dev_null
try:
cmd_arg = wrapped_command if win32 else [wrapped_command]
p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream,
stderr=err_stream, executable="/bin/bash")
(stdout, stderr) = p.communicate()
finally:
if dev_null is not None:
dev_null.close()
# Handle error condition (deal with stdout being None, too)
out = _AttributeString(stdout.strip() if stdout else "")
err = _AttributeString(stderr.strip() if stderr else "")
out.failed = False
out.return_code = p.returncode
out.stderr = err
if p.returncode != 0:
out.failed = True
msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command)
error(message=msg, stdout=out, stderr=err)
out.succeeded = not out.failed
# If we were capturing, this will be a string; otherwise it will be None.
return out
def test_lookup_port_not_in_file(self, run_mock):
run_mock.return_value = _AttributeString('')
run_mock.return_value.failed = False
port = lookup_port('any_host')
self.assertEqual(port, 8080)
def mock_fail_then_succeed(self):
output1 = _AttributeString()
output1.succeeded = False
output2 = _AttributeString()
output2.succeeded = True
return [output1, output2]
