def test_setters(self): result = exec_result.ExecResult(cmd=cmd) self.assertEqual(result.exit_code, ExitCodes.EX_INVALID) result.exit_code = 0 self.assertEqual(result.exit_code, 0) self.assertEqual(result.exit_code, result['exit_code']) tst_stdout = [ b'Test\n', b'long\n', b'stdout\n', b'data\n', b' \n', b'5\n', b'6\n', b'7\n', b'8\n', b'end!\n' ] tst_stderr = [b'test\n'] * 10 result['stdout'] = tst_stdout self.assertEqual(result.stdout, tst_stdout) self.assertEqual(result.stdout, result['stdout']) result['stderr'] = tst_stderr self.assertEqual(result.stderr, tst_stderr) self.assertEqual(result.stderr, result['stderr']) with self.assertRaises(TypeError): result.exit_code = 'code' with self.assertRaises(error.DevopsError): result['stdout_brief'] = 'test' with self.assertRaises(IndexError): result['test'] = True with self.assertRaises(TypeError): result.stdout = 'stdout' self.assertEqual(result.stdout, tst_stdout) with self.assertRaises(TypeError): result.stderr = 'stderr' self.assertEqual(result.stderr, tst_stderr) self.assertEqual(result.stdout_bin, bytearray(b''.join(tst_stdout))) self.assertEqual(result.stderr_bin, bytearray(b''.join(tst_stderr))) stdout_br = tst_stdout[:3] + [b'...\n'] + tst_stdout[-3:] stderr_br = tst_stderr[:3] + [b'...\n'] + tst_stderr[-3:] stdout_brief = b''.join(stdout_br).strip().decode(encoding='utf-8') stderr_brief = b''.join(stderr_br).strip().decode(encoding='utf-8') self.assertEqual(result.stdout_brief, stdout_brief) self.assertEqual(result.stderr_brief, stderr_brief)
def test_not_implemented(self, logger): """Test assertion on non implemented deserializer""" result = exec_result.ExecResult(cmd=cmd) deserialize = getattr(result, '_ExecResult__deserialize') with self.assertRaises(error.DevopsNotImplementedError): deserialize('tst') logger.assert_has_calls((mock.call.error( '{fmt} deserialize target is not implemented'.format( fmt='tst')), ))
def test_deprecations(self, logger): result = exec_result.ExecResult('test', stdout=[b'{"test": true}']) for deprecated in ('stdout_json', 'stdout_yaml'): result['{}'.format(deprecated)] = {'test': False} logger.assert_has_calls((mock.call.warning( '{key} is read-only and calculated automatically'.format( key='{}'.format(deprecated))), )) self.assertEqual(result[deprecated], {'test': True}) logger.reset_mock()
def prepare_close(popen, stderr_val=None, ec=0): stdout_lines = stdout_list stderr_lines = stderr_list if stderr_val is None else [] stdout = FakeFileStream(*stdout_lines) stderr = FakeFileStream(*stderr_lines) popen_obj = mock.Mock() popen_obj.attach_mock(stdout, 'stdout') popen_obj.attach_mock(stderr, 'stderr') popen_obj.configure_mock(returncode=ec) popen.return_value = popen_obj # noinspection PyTypeChecker exp_result = exec_result.ExecResult(cmd=command, stderr=stderr_lines, stdout=stdout_lines, exit_code=ec) return popen_obj, exp_result
def __exec_command(cls, command, cwd=None, env=None, timeout=None, verbose=False): """Command executor helper :type command: str :type cwd: str :type env: dict :type timeout: int :rtype: ExecResult """ def poll_stream(src, verb_logger=None): dst = [] try: for line in src: dst.append(line) if verb_logger is not None: verb_logger( line.decode('utf-8', errors='backslashreplace').rstrip()) except IOError: pass return dst def poll_streams(result, stdout, stderr, verbose): rlist, _, _ = select.select([stdout, stderr], [], []) if rlist: if stdout in rlist: result.stdout += poll_stream( src=stdout, verb_logger=logger.info if verbose else logger.debug) if stderr in rlist: result.stderr += poll_stream( src=stderr, verb_logger=logger.error if verbose else logger.debug) @decorators.threaded(started=True) def poll_pipes(proc, result, stop): """Polling task for FIFO buffers :type proc: subprocess.Popen :type result: ExecResult :type stop: threading.Event """ # Get file descriptors for stdout and stderr streams fd_stdout = proc.stdout.fileno() fd_stderr = proc.stderr.fileno() # Get flags of stdout and stderr streams fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL) fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL) # Set nonblock mode for stdout and stderr streams fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK) fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK) while not stop.isSet(): time.sleep(0.1) poll_streams(result=result, stdout=proc.stdout, stderr=proc.stderr, verbose=verbose) proc.poll() if proc.returncode is not None: result.exit_code = proc.returncode result.stdout += poll_stream( src=proc.stdout, verb_logger=logger.info if verbose else logger.debug) result.stderr += poll_stream( src=proc.stderr, verb_logger=logger.error if verbose else logger.debug) stop.set() # 1 Command per run with cls.__lock: result = exec_result.ExecResult(cmd=command) stop_event = threading.Event() if verbose: logger.info("\nExecuting command: {!r}".format( command.rstrip())) else: logger.debug("\nExecuting command: {!r}".format( command.rstrip())) # Run process = subprocess.Popen(args=[command], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=cwd, env=env, universal_newlines=False) # Poll output poll_pipes(process, result, stop_event) # wait for process close stop_event.wait(timeout) # Process closed? if stop_event.isSet(): stop_event.clear() return result # Kill not ended process and wait for close try: process.kill() # kill -9 stop_event.wait(5) except OSError: # Nothing to kill logger.warning("{!r} has been completed just after timeout: " "please validate timeout.".format(command)) wait_err_msg = ( 'Wait for {0!r} during {1}s: no return code!\n'.format( command, timeout)) output_brief_msg = ('\tSTDOUT:\n' '{0}\n' '\tSTDERR"\n' '{1}'.format(result.stdout_brief, result.stderr_brief)) logger.debug(wait_err_msg) raise error.TimeoutError(wait_err_msg + output_brief_msg)
def __exec_command(cls, command, channel, stdout, stderr, timeout, verbose=False): """Get exit status from channel with timeout :type command: str :type channel: paramiko.channel.Channel :type stdout: paramiko.channel.ChannelFile :type stderr: paramiko.channel.ChannelFile :type timeout: int :type verbose: bool :rtype: ExecResult :raises: TimeoutError """ def poll_stream(src, verbose): dst = [] try: for line in src: dst.append(line) if verbose: print(line.decode('utf-8', errors='backslashreplace'), end="") except IOError: pass return dst def poll_streams(result, channel, stdout, stderr, verbose): if channel.recv_ready(): result.stdout += poll_stream(src=stdout, verbose=verbose) if channel.recv_stderr_ready(): result.stderr += poll_stream(src=stderr, verbose=verbose) @decorators.threaded(started=True) def poll_pipes(stdout, stderr, result, stop, channel): """Polling task for FIFO buffers :type stdout: paramiko.channel.ChannelFile :type stderr: paramiko.channel.ChannelFile :type result: ExecResult :type stop: Event :type channel: paramiko.channel.Channel """ while not stop.isSet(): time.sleep(0.1) poll_streams(result=result, channel=channel, stdout=stdout, stderr=stderr, verbose=verbose) if channel.status_event.is_set(): result.exit_code = result.exit_code = channel.exit_status result.stdout += poll_stream(src=stdout, verbose=verbose) result.stderr += poll_stream(src=stderr, verbose=verbose) stop.set() # channel.status_event.wait(timeout) result = exec_result.ExecResult(cmd=command) stop_event = threading.Event() if verbose: print("\nExecuting command: {!r}".format(command.rstrip())) poll_pipes(stdout=stdout, stderr=stderr, result=result, stop=stop_event, channel=channel) stop_event.wait(timeout) # Process closed? if stop_event.isSet(): stop_event.clear() channel.close() return result stop_event.set() channel.close() status_tmpl = ('Wait for {0!r} during {1}s: no return code!\n' '\tSTDOUT:\n' '{2}\n' '\tSTDERR"\n' '{3}') logger.debug( status_tmpl.format(command, timeout, result.stdout, result.stderr)) raise error.TimeoutError( status_tmpl.format(command, timeout, result.stdout_brief, result.stderr_brief))
def test_create_minimal(self, logger): """Test defaults""" result = exec_result.ExecResult(cmd=cmd) self.assertEqual(result.cmd, cmd) self.assertEqual(result.cmd, result['cmd']) self.assertEqual(result.stdout, []) self.assertEqual(result.stdout, result['stdout']) self.assertEqual(result.stderr, []) self.assertEqual(result.stderr, result['stderr']) self.assertEqual(result.stdout_bin, bytearray()) self.assertEqual(result.stderr_bin, bytearray()) self.assertEqual(result.stdout_str, '') self.assertEqual(result.stdout_str, result['stdout_str']) self.assertEqual(result.stderr_str, '') self.assertEqual(result.stderr_str, result['stderr_str']) self.assertEqual(result.stdout_brief, '') self.assertEqual(result.stdout_brief, result['stdout_brief']) self.assertEqual(result.stderr_brief, '') self.assertEqual(result.stderr_brief, result['stderr_brief']) self.assertEqual(result.exit_code, ExitCodes.EX_INVALID) self.assertEqual(result.exit_code, result['exit_code']) self.assertEqual( repr(result), '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, ' 'exit_code={exit_code!s})'.format( cls=exec_result.ExecResult.__name__, cmd=cmd, stdout=[], stderr=[], exit_code=ExitCodes.EX_INVALID ) ) self.assertEqual( str(result), "{cls}(\n\tcmd={cmd!r}," "\n\t stdout=\n'{stdout_brief}'," "\n\tstderr=\n'{stderr_brief}', " '\n\texit_code={exit_code!s}\n)'.format( cls=exec_result.ExecResult.__name__, cmd=cmd, stdout_brief='', stderr_brief='', exit_code=ExitCodes.EX_INVALID ) ) with self.assertRaises(IndexError): # pylint: disable=pointless-statement # noinspection PyStatementEffect result['nonexistent'] # pylint: enable=pointless-statement with self.assertRaises(error.DevopsError): # pylint: disable=pointless-statement # noinspection PyStatementEffect result['stdout_json'] # pylint: enable=pointless-statement logger.assert_has_calls(( mock.call.exception( "'{cmd}' stdout is not valid json:\n" "{stdout_str!r}\n".format(cmd=cmd, stdout_str='')), )) self.assertIsNone(result['stdout_yaml']) self.assertEqual( hash(result), hash((exec_result.ExecResult, cmd, '', '', ExitCodes.EX_INVALID)) )
def test_json(self): result = exec_result.ExecResult('test', stdout=[b'{"test": true}']) self.assertEqual(result.stdout_json, {'test': True})
def __exec_command( cls, command, channel, stdout, stderr, timeout, verbose=False): """Get exit status from channel with timeout :type command: str :type channel: paramiko.channel.Channel :type stdout: paramiko.channel.ChannelFile :type stderr: paramiko.channel.ChannelFile :type timeout: int :type verbose: bool :rtype: ExecResult :raises: TimeoutError """ def poll_stream(src, verb_logger=None): dst = [] try: for line in src: dst.append(line) if verb_logger is not None: verb_logger( line.decode('utf-8', errors='backslashreplace').rstrip() ) except IOError: pass return dst def poll_streams(result, channel, stdout, stderr, verbose): if channel.recv_ready(): result.stdout += poll_stream( src=stdout, verb_logger=logger.info if verbose else logger.debug) if channel.recv_stderr_ready(): result.stderr += poll_stream( src=stderr, verb_logger=logger.error if verbose else logger.debug) @decorators.threaded(started=True) def poll_pipes(stdout, stderr, result, stop, channel): """Polling task for FIFO buffers :type stdout: paramiko.channel.ChannelFile :type stderr: paramiko.channel.ChannelFile :type result: ExecResult :type stop: Event :type channel: paramiko.channel.Channel """ while not stop.isSet(): time.sleep(0.1) poll_streams( result=result, channel=channel, stdout=stdout, stderr=stderr, verbose=verbose ) if channel.status_event.is_set(): result.exit_code = result.exit_code = channel.exit_status result.stdout += poll_stream( src=stdout, verb_logger=logger.info if verbose else logger.debug) result.stderr += poll_stream( src=stderr, verb_logger=logger.error if verbose else logger.debug) stop.set() # channel.status_event.wait(timeout) result = exec_result.ExecResult(cmd=command) stop_event = threading.Event() message = log_templates.CMD_EXEC.format(cmd=command.rstrip()) if verbose: logger.info(message) else: logger.debug(message) poll_pipes( stdout=stdout, stderr=stderr, result=result, stop=stop_event, channel=channel ) stop_event.wait(timeout) # Process closed? if stop_event.isSet(): stop_event.clear() channel.close() return result stop_event.set() channel.close() wait_err_msg = log_templates.CMD_WAIT_ERROR.format( cmd=command.rstrip(), timeout=timeout) output_brief_msg = ('\tSTDOUT:\n' '{0}\n' '\tSTDERR"\n' '{1}'.format(result.stdout_brief, result.stderr_brief)) logger.debug(wait_err_msg) raise error.TimeoutError(wait_err_msg + output_brief_msg)
class TestHelpersHelpers(unittest.TestCase): @mock.patch('devops.helpers.helpers.tcp_ping', return_value=False, autospec=True) def test_get_free_port(self, ping): result = helpers.get_free_port() self.assertEqual(result, 32000) ping.assert_called_once_with('localhost', 32000) ping.reset_mock() ping.return_value = True self.assertRaises(error.DevopsError, helpers.get_free_port) ping.assert_has_calls( [mock.call('localhost', port) for port in xrange(32000, 32100)]) @mock.patch('devops.helpers.subprocess_runner.Subprocess.execute', return_value=exec_result.ExecResult( cmd="ping -c 1 -W '{timeout:d}' '{host:s}'".format( host='127.0.0.1', timeout=1, ), exit_code=0, )) def test_icmp_ping(self, caller): host = '127.0.0.1' timeout = 1 result = helpers.icmp_ping(host=host) caller.assert_called_once_with( "ping -c 1 -W '{timeout:d}' '{host:s}'".format(host=host, timeout=timeout)) self.assertTrue(result, 'Unexpected result of validation') @mock.patch('socket.socket') def test_tcp_ping_(self, sock): s = mock.Mock() settimeout = mock.Mock() connect = mock.Mock() close = mock.Mock() s.configure_mock(settimeout=settimeout, connect=connect, close=close) sock.return_value = s host = '127.0.0.1' port = 65535 timeout = 1 helpers.tcp_ping_(host, port) sock.assert_called_once() settimeout.assert_not_called() connect.assert_called_once_with((str(host), int(port))) close.assert_called_once() sock.reset_mock() s.reset_mock() settimeout.reset_mock() close.reset_mock() connect.reset_mock() helpers.tcp_ping_(host, port, timeout) sock.assert_called_once() settimeout.assert_called_once_with(timeout) connect.assert_called_once_with((str(host), int(port))) close.assert_called_once() @mock.patch('devops.helpers.helpers.tcp_ping_', autospec=True) def test_tcp_ping(self, ping): host = '127.0.0.1' port = 65535 timeout = 1 result = helpers.tcp_ping(host, port, timeout) ping.assert_called_once_with(host, port, timeout) self.assertTrue(result) ping.reset_mock() ping.side_effect = socket.error result = helpers.tcp_ping(host, port, timeout) ping.assert_called_once_with(host, port, timeout) self.assertFalse(result) def test__check_wait_args(self): helpers._check_wait_args(lambda: None, [], {}, 1, 1) def test__check_wait_args_wrong_predicate(self): with self.assertRaises(TypeError): helpers._check_wait_args('predicate', [], {}, 1, 1) def test__check_wait_args_wrong_predicate_args(self): with self.assertRaises(TypeError): helpers._check_wait_args(lambda: None, 12, {}, 1, 1) def test__check_wait_args_wrong_predicate_kwargs(self): with self.assertRaises(TypeError): helpers._check_wait_args(lambda: None, [], {'one', 'two'}, 1, 1) def test__check_wait_args_wrong_interval(self): with self.assertRaises(ValueError): helpers._check_wait_args(lambda: None, ['one'], {'two': 2}, -2, 1) def test__check_wait_args_wrong_timeout(self): with self.assertRaises(ValueError): helpers._check_wait_args(lambda: None, (1, 2, 3), {}, 10, 0) @mock.patch('time.sleep', autospec=True) def test_wait(self, sleep): predicate = mock.Mock(return_value=True) result = helpers.wait(predicate, interval=1, timeout=1) self.assertTrue(result) predicate.assert_called_once() sleep.assert_not_called() sleep.reset_mock() predicate.reset_mock() predicate.return_value = 2 result = helpers.wait(predicate, interval=2, timeout=2) self.assertEqual(result, 2) predicate.assert_called_once() sleep.assert_not_called() sleep.reset_mock() predicate.reset_mock() predicate.return_value = False self.assertRaises(error.TimeoutError, helpers.wait, predicate, interval=1, timeout=1) predicate.assert_called() @mock.patch('time.sleep', autospec=True) def test_wait_pass(self, sleep): predicate = mock.Mock(return_value=True) result = helpers.wait_pass(predicate) self.assertTrue(result) predicate.reset_mock() predicate.side_effect = ValueError self.assertRaises(error.TimeoutError, helpers.wait_pass, predicate, timeout=1) @mock.patch('devops.helpers.helpers.tcp_ping', autospec=True) def test_wait_tcp(self, ping): host = '127.0.0.1' port = 65535 timeout = 1 helpers.wait_tcp(host, port, timeout) ping.assert_called_once_with(host=host, port=port) @mock.patch('devops.helpers.ssh_client.SSHClient', autospec=True) @mock.patch('devops.helpers.helpers.wait') def test_wait_ssh_cmd(self, wait, ssh): host = '127.0.0.1' port = 65535 check_cmd = 'ls ~' username = '******' password = '******' timeout = 0 helpers.wait_ssh_cmd(host, port, check_cmd, username, password, timeout) ssh.assert_called_once_with(host=host, port=port, auth=ssh_client.SSHAuth(username=username, password=password)) wait.assert_called_once() # Todo: cover ssh_client.execute @mock.patch('six.moves.http_client.HTTPConnection', autospec=True) def test_http(self, connection): host = 'localhost' port = 80 method = 'GET' url = '/' waited_code = 200 class Res(object): status = waited_code conn = mock.Mock() connection.return_value = conn request = mock.Mock() getresponse = mock.Mock(return_value=Res()) conn.configure_mock(getresponse=getresponse, request=request) result = helpers.http() self.assertTrue(result) connection.assert_called_once_with(host, port) request.assert_called_once_with(method, url) getresponse.assert_called_once() connection.reset_mock() request.reset_mock() getresponse.reset_mock() conn.reset_mock() conn.configure_mock(getresponse=getresponse, request=request) getresponse.return_value = Res() result = helpers.http(waited_code=404) self.assertFalse(result) connection.assert_called_once_with(host, port) request.assert_called_once_with(method, url) getresponse.assert_called_once() connection.reset_mock() request.reset_mock() getresponse.reset_mock() conn.reset_mock() conn.configure_mock(getresponse=getresponse, request=request) getresponse.return_value = Res() getresponse.side_effect = Exception result = helpers.http() self.assertFalse(result) connection.assert_called_once_with(host, port) request.assert_called_once_with(method, url) getresponse.assert_called_once() @mock.patch('six.moves.xmlrpc_client.Server', autospec=True) def test_xmlrpctoken(self, srv): server = mock.Mock() login = mock.Mock(return_value=True) server.configure_mock(login=login) srv.return_value = server uri = 'http://127.0.0.1' user = '******' password = '******' result = helpers.xmlrpctoken(uri, user, password) self.assertTrue(result) srv.assert_called_once_with(uri) login.assert_called_once_with(user, password) srv.reset_mock() server.reset_mock() login.reset_mock() server.configure_mock(login=login) srv.return_value = server login.side_effect = Exception self.assertRaises(error.AuthenticationError, helpers.xmlrpctoken, uri, user, password) srv.assert_called_once_with(uri) login.assert_called_once_with(user, password) @mock.patch('six.moves.xmlrpc_client.Server', autospec=True) def test_xmlrpcmethod(self, srv): class Success(object): success = True class Fail(object): def __getattr__(self, item): raise Exception() uri = 'http://127.0.0.1' srv.side_effect = [Success(), Success(), Fail()] result = helpers.xmlrpcmethod(uri, 'success') self.assertTrue(result) srv.assert_called_once_with(uri) srv.reset_mock() self.assertRaises(AttributeError, helpers.xmlrpcmethod, uri, 'failure') srv.assert_called_once_with(uri) srv.reset_mock() self.assertRaises(AttributeError, helpers.xmlrpcmethod, uri, 'success') srv.assert_called_once_with(uri) @mock.patch('os.urandom', autospec=True) def test_generate_mac(self, rand): rand.return_value = b'\x01\x02\x03\x04\x05' result = helpers.generate_mac() self.assertEqual(result, '64:01:02:03:04:05') rand.assert_called_once_with(5) def test_deepgetattr(self): # pylint: disable=attribute-defined-outside-init class Tst(object): one = 1 tst = Tst() tst2 = Tst() tst2.two = Tst() # pylint: enable=attribute-defined-outside-init result = helpers.deepgetattr(tst, 'one') self.assertEqual(result, 1) result = helpers.deepgetattr(tst2, 'two.one') self.assertEqual(result, 1) result = helpers.deepgetattr(tst, 'two.one') self.assertIsNone(result) result = helpers.deepgetattr(tst, 'two.one', default=0) self.assertEqual(result, 0) result = helpers.deepgetattr(tst2, 'two_one', splitter='_') self.assertEqual(result, 1) self.assertRaises(AttributeError, helpers.deepgetattr, tst, 'two.one', do_raise=True) def test_underscored(self): result = helpers.underscored('single') self.assertEqual(result, 'single') result = helpers.underscored('m', 'u', 'l', 't', 'i', 'p', 'l', 'e') self.assertEqual(result, 'm_u_l_t_i_p_l_e')