def nice_local(command, nice=0, capture=False, shell=None): """ Exactly like Fabric's local but with an optional nice argument """ from fabric.operations import _prefix_env_vars, _prefix_commands, _AttributeString from fabric.state import output, win32, env from fabric.utils import error given_command = command # Apply cd(), path() etc with_env = _prefix_env_vars(command, local=True) wrapped_command = _prefix_commands(with_env, 'local') if output.debug: print("[localhost] local: %s" % (wrapped_command)) elif output.running: print("[localhost] local: " + given_command) # Tie in to global output controls as best we can; our capture argument # takes precedence over the output settings. dev_null = None if capture: out_stream = subprocess.PIPE err_stream = subprocess.PIPE else: dev_null = open(os.devnull, 'w+') # Non-captured, hidden streams are discarded. out_stream = None if output.stdout else dev_null err_stream = None if output.stderr else dev_null try: cmd_arg = wrapped_command if win32 else [wrapped_command] p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream, stderr=err_stream, executable=shell, preexec_fn=lambda: os.nice(nice), close_fds=(not win32)) (stdout, stderr) = p.communicate() finally: if dev_null is not None: dev_null.close() # Handle error condition (deal with stdout being None, too) out = _AttributeString(stdout.strip() if stdout else "") err = _AttributeString(stderr.strip() if stderr else "") out.command = given_command out.real_command = wrapped_command out.failed = False out.return_code = p.returncode out.stderr = err if p.returncode not in env.ok_ret_codes: out.failed = True msg = "local() encountered an error (return code %s) while executing '%s'" % ( p.returncode, command) error(message=msg, stdout=out, stderr=err) out.succeeded = not out.failed # If we were capturing, this will be a string; otherwise it will be None. return out
def _run_host_command(command, shell=True, pty=True, combine_stderr=True): """ Run host wrapper command as root (Modified from fabric.operations._run_command to ignore prefixes, path(), cd(), and always use sudo.) """ # Set up new var so original argument can be displayed verbatim later. given_command = command # Handle context manager modifications, and shell wrapping wrapped_command = _shell_wrap( command, shell, _sudo_prefix(None) ) # Execute info line if output.debug: print("[%s] %s: %s" % (env.host_string, 'sudo', wrapped_command)) elif output.running: print("[%s] %s: %s" % (env.host_string, 'sudo', given_command)) # Actual execution, stdin/stdout/stderr handling, and termination stdout, stderr, status = _execute(default_channel(), wrapped_command, pty, combine_stderr) # Assemble output string out = _AttributeString(stdout) err = _AttributeString(stderr) # Error handling out.failed = False if status != 0: out.failed = True msg = "%s() received nonzero return code %s while executing" % ( 'sudo', status ) if env.warn_only: msg += " '%s'!" % given_command else: msg += "!\n\nRequested: %s\nExecuted: %s" % ( given_command, wrapped_command ) error(message=msg, stdout=out, stderr=err) # Attach return code to output string so users who have set things to # warn only, can inspect the error code. out.return_code = status # Convenience mirror of .failed out.succeeded = not out.failed # Attach stderr for anyone interested in that. out.stderr = err return out
def test_uninstall_is_called(self, mock_sudo, mock_version_check): env.host = "any_host" output1 = _AttributeString() output1.succeeded = False output2 = _AttributeString() output2.succeeded = True mock_sudo.side_effect = [output1, output2] server.uninstall() mock_version_check.assert_called_with() mock_sudo.assert_any_call('rpm -e presto') mock_sudo.assert_called_with('rpm -e presto-server-rpm')
def test_multiple_version_rpms(self, mock_run): err_msg = 'Presto is not installed.' output1 = _AttributeString(err_msg) output1.succeeded = False output2 = _AttributeString('0.111.SNAPSHOT') output2.succeeded = True mock_run.side_effect = [output1, output2] expected = server.check_presto_version() mock_run.assert_any_call('rpm -q --qf \"%{VERSION}\\n\" presto') mock_run.assert_called_with( 'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm') self.assertEqual(expected, '')
def test_lookup_string_config_not_in_file(self, run_mock): run_mock.return_value = _AttributeString('') run_mock.return_value.failed = False run_mock.return_value.return_code = 1 config_value = lookup_string_config('config.to.lookup', NODE_CONFIG_FILE, 'any_host') self.assertEqual(config_value, '')
def test_td_presto_version(self, mock_run): td_version = '101t' output = _AttributeString(td_version) output.succeeded = True mock_run.return_value = output expected = server.check_presto_version() self.assertEqual(expected, '')
def test_lookup_port_not_integer_failure(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=hello') run_mock.return_value.failed = False self.assertRaisesRegexp( ConfigurationError, 'Unable to coerce http-server.http.port \'hello\' to an int. ' 'Failed to connect to any_host.', lookup_port, 'any_host')
def run_local(command, sudo=False, shell=True, pty=True, combine_stderr=None): """Local implementation of fabric.api.run() using subprocess. .. note:: pty option exists for function signature compatibility and is ignored. """ if combine_stderr is None: combine_stderr = env.combine_stderr # TODO: Pass the SUDO_PASSWORD variable to the command here if sudo: command = "sudo " + command mico.output.debug(command) stderr = subprocess.STDOUT if combine_stderr else subprocess.PIPE process = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, stderr=stderr) out, err = process.communicate() # FIXME: Should stream the output, and only print it if fabric's properties allow it # print out # Wrap stdout string and add extra status attributes result = operations._AttributeString(out.rstrip('\n')) result.return_code = process.returncode result.succeeded = (process.returncode == 0) result.failed = not result.succeeded result.stderr = StringIO(err) return result
def test_multiple_version_rpms(self, mock_run): output1 = _AttributeString('package presto is not installed') output1.succeeded = False output2 = _AttributeString('presto-server-rpm-0.115t-1.x86_64') output2.succeeded = True output3 = _AttributeString('Presto is not installed.') output3.succeeded = False output4 = _AttributeString('0.111.SNAPSHOT') output4.succeeded = True mock_run.side_effect = [output1, output2, output3, output4] expected = server.check_presto_version() mock_run.assert_has_calls( [call('rpm -q presto'), call('rpm -q presto-server-rpm')]) self.assertEqual(expected, '')
def test_lookup_string_config(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'config.to.lookup=/path/hello') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 config_value = lookup_string_config('config.to.lookup', NODE_CONFIG_FILE, 'any_host') self.assertEqual(config_value, '/path/hello')
def test_init_success(self): fabric_result = _AttributeString('Success') fabric_result.succeeded = True fabric_result.return_code = 0 success = Result('tox', result=fabric_result) self.assertTrue(success.succeeded) self.assertEquals(success.log, 'Success') self.assertEquals(success.task, 'tox')
def test_init_failure(self): fabric_result = _AttributeString('Oh snap') fabric_result.succeeded = False fabric_result.return_code = 1 failure = Result('tox', fabric_result) self.assertFalse(failure.succeeded) self.assertEquals(failure.log, 'Oh snap') self.assertEquals(failure.task, 'tox')
def _blocal(command, capture=False): """ Slightly modified version of fabrics 'local' function designed to make the underlying subprocess.Popen call use bash. Taken from fabric operations.py file. """ given_command = command # Apply cd(), path() etc wrapped_command = _prefix_commands(_prefix_env_vars(command), 'local') if output.debug: print("[localhost] local: %s" % (wrapped_command)) elif output.running: print("[localhost] local: " + given_command) # Tie in to global output controls as best we can; our capture argument # takes precedence over the output settings. dev_null = None if capture: out_stream = subprocess.PIPE err_stream = subprocess.PIPE else: dev_null = open(os.devnull, 'w+') # Non-captured, hidden streams are discarded. out_stream = None if output.stdout else dev_null err_stream = None if output.stderr else dev_null try: cmd_arg = wrapped_command if win32 else [wrapped_command] p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream, stderr=err_stream,executable="/bin/bash") (stdout, stderr) = p.communicate() finally: if dev_null is not None: dev_null.close() # Handle error condition (deal with stdout being None, too) out = _AttributeString(stdout.strip() if stdout else "") err = _AttributeString(stderr.strip() if stderr else "") out.failed = False out.return_code = p.returncode out.stderr = err if p.returncode != 0: out.failed = True msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command) fabric.utils.error(message=msg, stdout=out, stderr=err) out.succeeded = not out.failed # If we were capturing, this will be a string; otherwise it will be None. return out
def test_lookup_port_out_of_range(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=99999') run_mock.return_value.failed = False self.assertRaisesRegexp( ConfigurationError, 'Invalid port number 99999: port must be between 1 and 65535 ' 'for property http-server.http.port on host any_host.', lookup_port, 'any_host')
def test_lookup_port_out_of_range(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=99999') run_mock.return_value.failed = False self.assertRaisesRegexp( ConfigurationError, 'Invalid port number 99999: port must be between 1 and 65535 ' 'for property http-server.http.port on host any_host.', lookup_port, 'any_host' )
def test_remove_failure(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' out = _AttributeString() out.succeeded = False sudo_mock.return_value = out self.assertRaisesRegexp( SystemExit, '\\[localhost\\] Failed to remove ' 'connector tpch.', connector.remove, 'tpch')
def test_lookup_port_not_integer_failure(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=hello') run_mock.return_value.failed = False self.assertRaisesRegexp( ConfigurationError, 'Unable to coerce http-server.http.port \'hello\' to an int. ' 'Failed to connect to any_host.', lookup_port, 'any_host' )
def test_multiple_version_rpms(self, mock_run): output1 = _AttributeString('package presto is not installed') output1.succeeded = False output2 = _AttributeString('presto-server-rpm-0.115t-1.x86_64') output2.succeeded = True output3 = _AttributeString('Presto is not installed.') output3.succeeded = False output4 = _AttributeString('0.111.SNAPSHOT') output4.succeeded = True mock_run.side_effect = [output1, output2, output3, output4] expected = server.check_presto_version() mock_run.assert_has_calls([ call('rpm -q presto'), call('rpm -q presto-server-rpm') ]) self.assertEqual(expected, '')
def test_remove_no_such_file(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' error_msg = ('Could not remove catalog tpch: No such file ' + os.path.join(get_catalog_directory(), 'tpch.properties')) out = _AttributeString(error_msg) out.succeeded = True sudo_mock.return_value = out self.assertRaisesRegexp(SystemExit, '\\[localhost\\] %s' % error_msg, catalog.remove, 'tpch')
def test_lookup_port_failure(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=8080') run_mock.return_value.failed = True self.assertRaisesRegexp( ConfigurationError, 'Configuration file /etc/presto/config.properties does not exist ' 'on host any_host', lookup_port, 'any_host' )
def test_lookup_string_config_file_not_found(self, run_mock): run_mock.return_value = _AttributeString( 'grep: /etc/presto/node.properties does not exist') run_mock.return_value.return_code = 2 self.assertRaisesRegexp( ConfigurationError, 'Configuration file /etc/presto/node.properties does not exist ' 'on host any_host', lookup_string_config, 'config.to.lookup', NODE_CONFIG_FILE, 'any_host')
def test_lookup_port_not_integer_failure(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=hello') run_mock.return_value.failed = False run_mock.return_value.return_code = 0 self.assertRaisesRegexp( ConfigurationError, 'Invalid port number hello: port must be a number between 1 and' ' 65535 for property http-server.http.port on host any_host.', lookup_port, 'any_host')
def test_remove_failure(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' out = _AttributeString() out.succeeded = False sudo_mock.return_value = out self.assertRaisesRegexp(SystemExit, '\\[localhost\\] Failed to remove catalog tpch.', catalog.remove, 'tpch')
def test_remove_failure(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' out = _AttributeString() out.succeeded = False sudo_mock.return_value = out connector.remove('tpch') self.assertEqual('\nWarning: [localhost] Failed to remove connector ' 'tpch.\n\t\n\n', self.test_stderr.getvalue())
def test_remove_no_such_file(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' error_msg = ('Could not remove connector tpch: No such file ' '/etc/opt/prestoadmin/connectors/tpch.properties') out = _AttributeString(error_msg) out.succeeded = True sudo_mock.return_value = out self.assertRaisesRegexp(SystemExit, '\\[localhost\\] %s' % error_msg, connector.remove, 'tpch')
def test_lookup_string_config_file_not_found(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'grep: /etc/presto/node.properties does not exist') sudo_mock.return_value.return_code = 2 self.assertRaisesRegexp( ConfigurationError, 'Could not access config file /etc/presto/node.properties on host any_host', lookup_string_config, 'config.to.lookup', NODE_CONFIG_FILE, 'any_host' )
def test_lookup_port_not_integer_failure(self, run_mock): run_mock.return_value = _AttributeString('http-server.http.port=hello') run_mock.return_value.failed = False run_mock.return_value.return_code = 0 self.assertRaisesRegexp( ConfigurationError, 'Invalid port number hello: port must be a number between 1 and' ' 65535 for property http-server.http.port on host any_host.', lookup_port, 'any_host' )
def test_remove_no_such_file(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' error_msg = ('Could not remove connector tpch: No such file ' '/etc/opt/prestoadmin/connectors/tpch.properties') out = _AttributeString(error_msg) out.succeeded = True sudo_mock.return_value = out connector.remove('tpch') self.assertEqual('\nWarning: [localhost] %s\n\n' % error_msg, self.test_stderr.getvalue())
def test_rpm_upgrade(self, mock_sudo, mock_rpm_upgrade): env.host = 'any_host' env.nodeps = False mock_sudo.return_value = _AttributeString('test_package_name') mock_sudo.return_value.succeeded = True package.rpm_upgrade('test.rpm') mock_sudo.assert_any_call('rpm -qp --queryformat \'%{NAME}\' ' '/opt/prestoadmin/packages/test.rpm', quiet=True) mock_rpm_upgrade.assert_any_call('/opt/prestoadmin/packages/test.rpm')
def test_execute_query_get_port(self, run_mock, conn_mock): client = PrestoClient('any_host', 'any_user') client.rows = ['hello'] client.next_uri = 'hello' client.response_from_server = {'hello': 'hello'} run_mock.return_value = _AttributeString('http-server.http.port=8080') run_mock.return_value.failed = False client.execute_query('select * from nation') self.assertEqual(client.port, 8080) self.assertEqual(client.rows, []) self.assertEqual(client.next_uri, '') self.assertEqual(client.response_from_server, {})
def testrun_sql_get_port(self, sudo_mock, conn_mock, mock_presto_config): client = PrestoClient('any_host', 'any_user') client.rows = ['hello'] client.next_uri = 'hello' client.response_from_server = {'hello': 'hello'} sudo_mock.return_value = _AttributeString('http-server.http.port=8080') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 client.run_sql('select * from nation') self.assertEqual(client.port, 8080) self.assertEqual(client.rows, []) self.assertEqual(client.next_uri, '') self.assertEqual(client.response_from_server, {})
def test_version_with_snapshot(self, mock_run): snapshot_version = '0.107.SNAPSHOT' output = _AttributeString(snapshot_version) output.succeeded = True mock_run.return_value = output expected = server.check_presto_version() self.assertEqual(expected, '') snapshot_version = '0.107.SNAPSHOT-1.x86_64' output = _AttributeString(snapshot_version) output.succeeded = True mock_run.return_value = output expected = server.check_presto_version() self.assertEqual(expected, '') snapshot_version = '0.107-SNAPSHOT' output = _AttributeString(snapshot_version) output.succeeded = True mock_run.return_value = output expected = server.check_presto_version() self.assertEqual(expected, '')
def test_rpm_upgrade(self, mock_sudo, mock_rpm_upgrade): env.host = 'any_host' env.nodeps = False mock_sudo.return_value = _AttributeString('test_package_name') mock_sudo.return_value.succeeded = True package.rpm_upgrade('test.rpm') mock_sudo.assert_any_call( 'rpm -qp --queryformat \'%{NAME}\' ' '/opt/prestoadmin/packages/test.rpm', quiet=True) mock_rpm_upgrade.assert_any_call('/opt/prestoadmin/packages/test.rpm')
def test_warning_presto_version_not_installed(self, mock_warn, mock_run, mock_run_sql, mock_presto_config): env.host = 'node1' env.roledefs['coordinator'] = ['node1'] env.roledefs['worker'] = ['node1'] env.roledefs['all'] = ['node1'] env.hosts = env.roledefs['all'] output = _AttributeString('package presto is not installed') output.succeeded = False mock_run.return_value = output env.host = 'node1' server.collect_node_information() installation_warning = 'Presto is not installed.' mock_warn.assert_called_with(installation_warning)
def test_warning_presto_version_not_installed(self, mock_warn, mock_run, mock_run_sql): env.host = 'node1' env.roledefs['coordinator'] = ['node1'] env.roledefs['worker'] = ['node1'] env.roledefs['all'] = ['node1'] env.hosts = env.roledefs['all'] output = _AttributeString('package presto is not installed') output.succeeded = False mock_run.return_value = output env.host = 'node1' server.collect_node_information() installation_warning = 'Presto is not installed.' mock_warn.assert_called_with(installation_warning)
def test_warning_presto_version_wrong(self, mock_warn, mock_run, mock_run_sql): env.host = 'node1' env.roledefs['coordinator'] = ['node1'] env.roledefs['worker'] = ['node1'] env.roledefs['all'] = ['node1'] env.hosts = env.roledefs['all'] old_version = '0.97' output = _AttributeString(old_version) output.succeeded = True mock_run.return_value = output server.collect_node_information() version_warning = 'Presto version is %s, version >= 0.%d required.'\ % (old_version, PRESTO_RPM_MIN_REQUIRED_VERSION) mock_warn.assert_called_with(version_warning)
def test_is_pip_installed_pythonbrew(self, mock_run): from fabric.operations import _AttributeString from fabtools.python import is_pip_installed fake_result = _AttributeString( '\x1b[32mSwitched to Python-2.7.5' '\x1b[0mpip 1.4.1 from /home/vagrant/.pythonbrew/pythons/Python-2.7.5/lib/python2.7/site-packages/pip-1.4.1-py2.7.egg (python 2.7)' ) fake_result.failed = False fake_result.succeeded = True mock_run.return_value = fake_result res = is_pip_installed(version='1.3.1') self.assertTrue(res)
def test_rpm_upgrade_nodeps(self, mock_sudo): env.host = 'any_host' env.nodeps = True mock_sudo.return_value = _AttributeString('/opt/prestoadmin/packages/' 'test.rpm') mock_sudo.return_value.succeeded = True package.rpm_upgrade('test.rpm') mock_sudo.assert_any_call('rpm -qp --queryformat \'%{NAME}\' ' '/opt/prestoadmin/packages/test.rpm', quiet=True) mock_sudo.assert_any_call('rpm -i --nodeps ' '/opt/prestoadmin/packages/test.rpm') mock_sudo.assert_any_call('rpm -e --nodeps ' '/opt/prestoadmin/packages/test.rpm') self.assertEqual(3, mock_sudo.call_count)
def _run_host_command(command, shell=True, pty=True, combine_stderr=True, quiet=False, warn_only=False, stdout=None, stderr=None, timeout=None): """ Run host wrapper command as root (Modified from fabric.operations._run_command to ignore prefixes, path(), cd(), and always use sudo.) """ manager = _noop if warn_only: manager = warn_only_manager # Quiet's behavior is a superset of warn_only's, so it wins. if quiet: manager = quiet_manager with manager(): # Set up new var so original argument can be displayed verbatim later. given_command = command # Handle context manager modifications, and shell wrapping wrapped_command = _shell_wrap( command, # !! removed _prefix_commands() & _prefix_env_vars() shell, _sudo_prefix(None) # !! always use sudo ) # Execute info line which = 'sudo' # !! always use sudo if output.debug: print(("[%s] %s: %s" % (env.host_string, which, wrapped_command))) elif output.running: print(("[%s] %s: %s" % (env.host_string, which, given_command))) # Actual execution, stdin/stdout/stderr handling, and termination result_stdout, result_stderr, status = _execute( channel=default_channel(), command=wrapped_command, pty=pty, combine_stderr=combine_stderr, invoke_shell=False, stdout=stdout, stderr=stderr, timeout=timeout) # Assemble output string out = _AttributeString(result_stdout) err = _AttributeString(result_stderr) # Error handling out.failed = False out.command = given_command out.real_command = wrapped_command if status not in env.ok_ret_codes: out.failed = True msg = "%s() received nonzero return code %s while executing" % ( which, status) if env.warn_only: msg += " '%s'!" % given_command else: msg += "!\n\nRequested: %s\nExecuted: %s" % (given_command, wrapped_command) error(message=msg, stdout=out, stderr=err) # Attach return code to output string so users who have set things to # warn only, can inspect the error code. out.return_code = status # Convenience mirror of .failed out.succeeded = not out.failed # Attach stderr for anyone interested in that. out.stderr = err return out
def get_mock_ssh_text(text, code): result = _AttributeString(text) result.return_code = code return result
def mock_fail_then_succeed(self): output1 = _AttributeString() output1.succeeded = False output2 = _AttributeString() output2.succeeded = True return [output1, output2]
def local(command, capture=False, shell=None, ignore_errors=False, env=None): """ Run a command on the local system. `local` is simply a convenience wrapper around the use of the builtin Python ``subprocess`` module with ``shell=True`` activated. If you need to do anything special, consider using the ``subprocess`` module directly. ``shell`` is passed directly to `subprocess.Popen <http://docs.python.org/library/subprocess.html#subprocess.Popen>`_'s ``execute`` argument (which determines the local shell to use.) As per the linked documentation, on Unix the default behavior is to use ``/bin/sh``, so this option is useful for setting that value to e.g. ``/bin/bash``. `local` is not currently capable of simultaneously printing and capturing output, as `~fabric.operations.run`/`~fabric.operations.sudo` do. The ``capture`` kwarg allows you to switch between printing and capturing as necessary, and defaults to ``False``. When ``capture=False``, the local subprocess' stdout and stderr streams are hooked up directly to your terminal, though you may use the global :doc:`output controls </usage/output_controls>` ``output.stdout`` and ``output.stderr`` to hide one or both if desired. In this mode, the return value's stdout/stderr values are always empty. When ``capture=True``, you will not see any output from the subprocess in your terminal, but the return value will contain the captured stdout/stderr. In either case, as with `~fabric.operations.run` and `~fabric.operations.sudo`, this return value exhibits the ``return_code``, ``stderr``, ``failed``, ``succeeded``, ``command`` and ``real_command`` attributes. See `run` for details. `~fabric.operations.local` will honor the `~fabric.context_managers.lcd` context manager, allowing you to control its current working directory independently of the remote end (which honors `~fabric.context_managers.cd`). .. versionchanged:: 1.0 Added the ``succeeded`` and ``stderr`` attributes. .. versionchanged:: 1.0 Now honors the `~fabric.context_managers.lcd` context manager. .. versionchanged:: 1.0 Changed the default value of ``capture`` from ``True`` to ``False``. .. versionadded:: 1.9 The return value attributes ``.command`` and ``.real_command``. """ given_command = command # Apply cd(), path() etc with_env = _prefix_env_vars(command, local=True) wrapped_command = _prefix_commands(with_env, 'local') # if output.debug: # print("[localhost] local: %s" % (wrapped_command)) # elif output.running: # print("[localhost] local: " + given_command) # Tie in to global output controls as best we can; our capture argument # takes precedence over the output settings. dev_null = None if capture: out_stream = subprocess.PIPE err_stream = subprocess.PIPE else: dev_null = open(os.devnull, 'w+') # Non-captured, hidden streams are discarded. out_stream = None if output.stdout else dev_null err_stream = None if output.stderr else dev_null if env is None: env = os.environ try: cmd_arg = wrapped_command if win32 else [wrapped_command] p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream, stderr=err_stream, executable=shell, close_fds=(not win32), env=env) (stdout, stderr) = p.communicate() finally: if dev_null is not None: dev_null.close() # Handle error condition (deal with stdout being None, too) out = _AttributeString(stdout.decode().strip() if stdout else "") err = _AttributeString(stderr.decode().strip() if stderr else "") out.command = given_command out.real_command = wrapped_command out.failed = False out.return_code = p.returncode out.stderr = err if p.returncode not in _env.ok_ret_codes and not ignore_errors: out.failed = True # msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command) # print('\n\n') # if out: # print('{}'.format(out)) # print('\n') # if err: # print('{}'.format(err)) # sys.exit(1) raise CommandError(out) # error(message=msg, stdout=out, stderr=err) out.succeeded = not out.failed # If we were capturing, this will be a string; otherwise it will be None. return out
def handle_complex_command(self, command, job_name=""): """ Wrap subprocess.Popen in such a way that the stderr and stdout from running a shell command will be captured and logged in nearly real time. This is similar to fabric.local, but allows us to retain control over the process. This method is named "complex" because it uses queues and threads to execute a command while capturing and displaying the output. """ # We define a "local logger" here such that we can give it a slightly # different name. We use the package name as part of the logger to # allow admins to easily distinguish between which package is currently # being installed. llog_name = __name__ if len(job_name) > 0: llog_name += ':' + job_name llog = logging.getLogger(llog_name) # Print the command we're about to execute, ``set -x`` style. llog.debug('+ ' + str(command)) # Launch the command as subprocess. A bufsize of 1 means line buffered. process_handle = subprocess.Popen(str(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, close_fds=False, shell=True, cwd=state.env['lcwd']) pid = process_handle.pid # Launch the asynchronous readers of the process' stdout and stderr. stdout_queue = queue.Queue() stdout_reader = asynchronous_reader.AsynchronousReader( process_handle.stdout, stdout_queue) stdout_reader.start() stderr_queue = queue.Queue() stderr_reader = asynchronous_reader.AsynchronousReader( process_handle.stderr, stderr_queue) stderr_reader.start() # Place streamed stdout and stderr into a threaded IPC queue target so it can # be printed and stored for later retrieval when generating the INSTALLATION.log. stdio_thread = threading.Thread( target=self.enqueue_output, args=(process_handle.stdout, stdout_queue, process_handle.stderr, stderr_queue)) thread_lock = threading.Lock() thread_lock.acquire() stdio_thread.start() # Check the queues for output until there is nothing more to get. start_timer = time.time() while not stdout_reader.installation_complete( ) or not stderr_reader.installation_complete(): # Show what we received from standard output. while not stdout_queue.empty(): try: line = stdout_queue.get() except queue.Empty: line = None break if line: llog.debug(line) start_timer = time.time() else: break # Show what we received from standard error. while not stderr_queue.empty(): try: line = stderr_queue.get() except queue.Empty: line = None break if line: llog.debug(line) start_timer = time.time() else: stderr_queue.task_done() break # Sleep a bit before asking the readers again. time.sleep(.1) current_wait_time = time.time() - start_timer if stdout_queue.empty() and stderr_queue.empty( ) and current_wait_time > basic_util.NO_OUTPUT_TIMEOUT: err_msg = "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" % \ (pid, basic_util.NO_OUTPUT_TIMEOUT) stderr_reader.lines.append(err_msg) process_handle.kill() break thread_lock.release() # Wait until each of the threads we've started terminate. The following calls will block each thread # until it terminates either normally, through an unhandled exception, or until the timeout occurs. stdio_thread.join(basic_util.NO_OUTPUT_TIMEOUT) stdout_reader.join(basic_util.NO_OUTPUT_TIMEOUT) stderr_reader.join(basic_util.NO_OUTPUT_TIMEOUT) # Close subprocess' file descriptors. self.close_file_descriptor(process_handle.stdout) self.close_file_descriptor(process_handle.stderr) stdout = '\n'.join(stdout_reader.lines) stderr = '\n'.join(stderr_reader.lines) # Handle error condition (deal with stdout being None, too) output = _AttributeString(stdout.strip() if stdout else "") errors = _AttributeString(stderr.strip() if stderr else "") # Make sure the process has finished. process_handle.poll() output.return_code = process_handle.returncode output.stderr = errors return output
def bash_local(command, capture=False): """ Run a command on the local system. `local` is simply a convenience wrapper around the use of the builtin Python ``subprocess`` module with ``shell=True`` activated. If you need to do anything special, consider using the ``subprocess`` module directly. `local` is not currently capable of simultaneously printing and capturing output, as `~fabric.operations.run`/`~fabric.operations.sudo` do. The ``capture`` kwarg allows you to switch between printing and capturing as necessary, and defaults to ``False``. When ``capture=False``, the local subprocess' stdout and stderr streams are hooked up directly to your terminal, though you may use the global :doc:`output controls </usage/output_controls>` ``output.stdout`` and ``output.stderr`` to hide one or both if desired. In this mode, `~fabric.operations.local` returns None. When ``capture=True``, this function will return the contents of the command's stdout as a string-like object; as with `~fabric.operations.run` and `~fabric.operations.sudo`, this return value exhibits the ``return_code``, ``stderr``, ``failed`` and ``succeeded`` attributes. See `run` for details. `~fabric.operations.local` will honor the `~fabric.context_managers.lcd` context manager, allowing you to control its current working directory independently of the remote end (which honors `~fabric.context_managers.cd`). .. versionchanged:: 1.0 Added the ``succeeded`` and ``stderr`` attributes. .. versionchanged:: 1.0 Now honors the `~fabric.context_managers.lcd` context manager. .. versionchanged:: 1.0 Changed the default value of ``capture`` from ``True`` to ``False``. """ given_command = command # Apply cd(), path() etc wrapped_command = _prefix_commands(_prefix_env_vars(command), 'local') if output.debug: print("[localhost] local: %s" % (wrapped_command)) elif output.running: print("[localhost] local: " + given_command) # Tie in to global output controls as best we can; our capture argument # takes precedence over the output settings. dev_null = None if capture: out_stream = subprocess.PIPE err_stream = subprocess.PIPE else: dev_null = open(os.devnull, 'w+') # Non-captured, hidden streams are discarded. out_stream = None if output.stdout else dev_null err_stream = None if output.stderr else dev_null try: cmd_arg = wrapped_command if win32 else [wrapped_command] p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream, stderr=err_stream, executable="/bin/bash") (stdout, stderr) = p.communicate() finally: if dev_null is not None: dev_null.close() # Handle error condition (deal with stdout being None, too) out = _AttributeString(stdout.strip() if stdout else "") err = _AttributeString(stderr.strip() if stderr else "") out.failed = False out.return_code = p.returncode out.stderr = err if p.returncode != 0: out.failed = True msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command) error(message=msg, stdout=out, stderr=err) out.succeeded = not out.failed # If we were capturing, this will be a string; otherwise it will be None. return out
def test_lookup_port_not_in_file(self, run_mock): run_mock.return_value = _AttributeString('') run_mock.return_value.failed = False port = lookup_port('any_host') self.assertEqual(port, 8080)