def test_setters(self):
result = exec_result.ExecResult(cmd=cmd)
self.assertEqual(result.exit_code, ExitCodes.EX_INVALID)
result.exit_code = 0
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.exit_code, result['exit_code'])
tst_stdout = [
b'Test\n',
b'long\n',
b'stdout\n',
b'data\n',
b' \n',
b'5\n',
b'6\n',
b'7\n',
b'8\n',
b'end!\n'
]
tst_stderr = [b'test\n'] * 10
result['stdout'] = tst_stdout
self.assertEqual(result.stdout, tst_stdout)
self.assertEqual(result.stdout, result['stdout'])
result['stderr'] = tst_stderr
self.assertEqual(result.stderr, tst_stderr)
self.assertEqual(result.stderr, result['stderr'])
with self.assertRaises(TypeError):
result.exit_code = 'code'
with self.assertRaises(error.DevopsError):
result['stdout_brief'] = 'test'
with self.assertRaises(IndexError):
result['test'] = True
with self.assertRaises(TypeError):
result.stdout = 'stdout'
self.assertEqual(result.stdout, tst_stdout)
with self.assertRaises(TypeError):
result.stderr = 'stderr'
self.assertEqual(result.stderr, tst_stderr)
self.assertEqual(result.stdout_bin, bytearray(b''.join(tst_stdout)))
self.assertEqual(result.stderr_bin, bytearray(b''.join(tst_stderr)))
stdout_br = tst_stdout[:3] + [b'...\n'] + tst_stdout[-3:]
stderr_br = tst_stderr[:3] + [b'...\n'] + tst_stderr[-3:]
stdout_brief = b''.join(stdout_br).strip().decode(encoding='utf-8')
stderr_brief = b''.join(stderr_br).strip().decode(encoding='utf-8')
self.assertEqual(result.stdout_brief, stdout_brief)
self.assertEqual(result.stderr_brief, stderr_brief)
def test_not_implemented(self, logger):
"""Test assertion on non implemented deserializer"""
result = exec_result.ExecResult(cmd=cmd)
deserialize = getattr(result, '_ExecResult__deserialize')
with self.assertRaises(error.DevopsNotImplementedError):
deserialize('tst')
logger.assert_has_calls((mock.call.error(
'{fmt} deserialize target is not implemented'.format(
fmt='tst')), ))
def test_deprecations(self, logger):
result = exec_result.ExecResult('test', stdout=[b'{"test": true}'])
for deprecated in ('stdout_json', 'stdout_yaml'):
result['{}'.format(deprecated)] = {'test': False}
logger.assert_has_calls((mock.call.warning(
'{key} is read-only and calculated automatically'.format(
key='{}'.format(deprecated))), ))
self.assertEqual(result[deprecated], {'test': True})
logger.reset_mock()
def prepare_close(popen, stderr_val=None, ec=0):
stdout_lines = stdout_list
stderr_lines = stderr_list if stderr_val is None else []
stdout = FakeFileStream(*stdout_lines)
stderr = FakeFileStream(*stderr_lines)
popen_obj = mock.Mock()
popen_obj.attach_mock(stdout, 'stdout')
popen_obj.attach_mock(stderr, 'stderr')
popen_obj.configure_mock(returncode=ec)
popen.return_value = popen_obj
# noinspection PyTypeChecker
exp_result = exec_result.ExecResult(cmd=command,
stderr=stderr_lines,
stdout=stdout_lines,
exit_code=ec)
return popen_obj, exp_result
def __exec_command(cls,
command,
cwd=None,
env=None,
timeout=None,
verbose=False):
"""Command executor helper
:type command: str
:type cwd: str
:type env: dict
:type timeout: int
:rtype: ExecResult
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip())
except IOError:
pass
return dst
def poll_streams(result, stdout, stderr, verbose):
rlist, _, _ = select.select([stdout, stderr], [], [])
if rlist:
if stdout in rlist:
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if stderr in rlist:
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(proc, result, stop):
"""Polling task for FIFO buffers
:type proc: subprocess.Popen
:type result: ExecResult
:type stop: threading.Event
"""
# Get file descriptors for stdout and stderr streams
fd_stdout = proc.stdout.fileno()
fd_stderr = proc.stderr.fileno()
# Get flags of stdout and stderr streams
fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL)
fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL)
# Set nonblock mode for stdout and stderr streams
fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK)
while not stop.isSet():
time.sleep(0.1)
poll_streams(result=result,
stdout=proc.stdout,
stderr=proc.stderr,
verbose=verbose)
proc.poll()
if proc.returncode is not None:
result.exit_code = proc.returncode
result.stdout += poll_stream(
src=proc.stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=proc.stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# 1 Command per run
with cls.__lock:
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
if verbose:
logger.info("\nExecuting command: {!r}".format(
command.rstrip()))
else:
logger.debug("\nExecuting command: {!r}".format(
command.rstrip()))
# Run
process = subprocess.Popen(args=[command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=cwd,
env=env,
universal_newlines=False)
# Poll output
poll_pipes(process, result, stop_event)
# wait for process close
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
return result
# Kill not ended process and wait for close
try:
process.kill() # kill -9
stop_event.wait(5)
except OSError:
# Nothing to kill
logger.warning("{!r} has been completed just after timeout: "
"please validate timeout.".format(command))
wait_err_msg = (
'Wait for {0!r} during {1}s: no return code!\n'.format(
command, timeout))
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
def __exec_command(cls,
command,
channel,
stdout,
stderr,
timeout,
verbose=False):
"""Get exit status from channel with timeout
:type command: str
:type channel: paramiko.channel.Channel
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type timeout: int
:type verbose: bool
:rtype: ExecResult
:raises: TimeoutError
"""
def poll_stream(src, verbose):
dst = []
try:
for line in src:
dst.append(line)
if verbose:
print(line.decode('utf-8', errors='backslashreplace'),
end="")
except IOError:
pass
return dst
def poll_streams(result, channel, stdout, stderr, verbose):
if channel.recv_ready():
result.stdout += poll_stream(src=stdout, verbose=verbose)
if channel.recv_stderr_ready():
result.stderr += poll_stream(src=stderr, verbose=verbose)
@decorators.threaded(started=True)
def poll_pipes(stdout, stderr, result, stop, channel):
"""Polling task for FIFO buffers
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type result: ExecResult
:type stop: Event
:type channel: paramiko.channel.Channel
"""
while not stop.isSet():
time.sleep(0.1)
poll_streams(result=result,
channel=channel,
stdout=stdout,
stderr=stderr,
verbose=verbose)
if channel.status_event.is_set():
result.exit_code = result.exit_code = channel.exit_status
result.stdout += poll_stream(src=stdout, verbose=verbose)
result.stderr += poll_stream(src=stderr, verbose=verbose)
stop.set()
# channel.status_event.wait(timeout)
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
if verbose:
print("\nExecuting command: {!r}".format(command.rstrip()))
poll_pipes(stdout=stdout,
stderr=stderr,
result=result,
stop=stop_event,
channel=channel)
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
channel.close()
return result
stop_event.set()
channel.close()
status_tmpl = ('Wait for {0!r} during {1}s: no return code!\n'
'\tSTDOUT:\n'
'{2}\n'
'\tSTDERR"\n'
'{3}')
logger.debug(
status_tmpl.format(command, timeout, result.stdout, result.stderr))
raise error.TimeoutError(
status_tmpl.format(command, timeout, result.stdout_brief,
result.stderr_brief))
def test_create_minimal(self, logger):
"""Test defaults"""
result = exec_result.ExecResult(cmd=cmd)
self.assertEqual(result.cmd, cmd)
self.assertEqual(result.cmd, result['cmd'])
self.assertEqual(result.stdout, [])
self.assertEqual(result.stdout, result['stdout'])
self.assertEqual(result.stderr, [])
self.assertEqual(result.stderr, result['stderr'])
self.assertEqual(result.stdout_bin, bytearray())
self.assertEqual(result.stderr_bin, bytearray())
self.assertEqual(result.stdout_str, '')
self.assertEqual(result.stdout_str, result['stdout_str'])
self.assertEqual(result.stderr_str, '')
self.assertEqual(result.stderr_str, result['stderr_str'])
self.assertEqual(result.stdout_brief, '')
self.assertEqual(result.stdout_brief, result['stdout_brief'])
self.assertEqual(result.stderr_brief, '')
self.assertEqual(result.stderr_brief, result['stderr_brief'])
self.assertEqual(result.exit_code, ExitCodes.EX_INVALID)
self.assertEqual(result.exit_code, result['exit_code'])
self.assertEqual(
repr(result),
'{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, '
'exit_code={exit_code!s})'.format(
cls=exec_result.ExecResult.__name__,
cmd=cmd,
stdout=[],
stderr=[],
exit_code=ExitCodes.EX_INVALID
)
)
self.assertEqual(
str(result),
"{cls}(\n\tcmd={cmd!r},"
"\n\t stdout=\n'{stdout_brief}',"
"\n\tstderr=\n'{stderr_brief}', "
'\n\texit_code={exit_code!s}\n)'.format(
cls=exec_result.ExecResult.__name__,
cmd=cmd,
stdout_brief='',
stderr_brief='',
exit_code=ExitCodes.EX_INVALID
)
)
with self.assertRaises(IndexError):
# pylint: disable=pointless-statement
# noinspection PyStatementEffect
result['nonexistent']
# pylint: enable=pointless-statement
with self.assertRaises(error.DevopsError):
# pylint: disable=pointless-statement
# noinspection PyStatementEffect
result['stdout_json']
# pylint: enable=pointless-statement
logger.assert_has_calls((
mock.call.exception(
"'{cmd}' stdout is not valid json:\n"
"{stdout_str!r}\n".format(cmd=cmd, stdout_str='')),
))
self.assertIsNone(result['stdout_yaml'])
self.assertEqual(
hash(result),
hash((exec_result.ExecResult, cmd, '', '', ExitCodes.EX_INVALID))
)
def test_json(self):
result = exec_result.ExecResult('test', stdout=[b'{"test": true}'])
self.assertEqual(result.stdout_json, {'test': True})
def __exec_command(
cls, command, channel, stdout, stderr, timeout, verbose=False):
"""Get exit status from channel with timeout
:type command: str
:type channel: paramiko.channel.Channel
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type timeout: int
:type verbose: bool
:rtype: ExecResult
:raises: TimeoutError
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip()
)
except IOError:
pass
return dst
def poll_streams(result, channel, stdout, stderr, verbose):
if channel.recv_ready():
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if channel.recv_stderr_ready():
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(stdout, stderr, result, stop, channel):
"""Polling task for FIFO buffers
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type result: ExecResult
:type stop: Event
:type channel: paramiko.channel.Channel
"""
while not stop.isSet():
time.sleep(0.1)
poll_streams(
result=result,
channel=channel,
stdout=stdout,
stderr=stderr,
verbose=verbose
)
if channel.status_event.is_set():
result.exit_code = result.exit_code = channel.exit_status
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# channel.status_event.wait(timeout)
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
message = log_templates.CMD_EXEC.format(cmd=command.rstrip())
if verbose:
logger.info(message)
else:
logger.debug(message)
poll_pipes(
stdout=stdout,
stderr=stderr,
result=result,
stop=stop_event,
channel=channel
)
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
channel.close()
return result
stop_event.set()
channel.close()
wait_err_msg = log_templates.CMD_WAIT_ERROR.format(
cmd=command.rstrip(),
timeout=timeout)
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
class TestHelpersHelpers(unittest.TestCase):
@mock.patch('devops.helpers.helpers.tcp_ping',
return_value=False,
autospec=True)
def test_get_free_port(self, ping):
result = helpers.get_free_port()
self.assertEqual(result, 32000)
ping.assert_called_once_with('localhost', 32000)
ping.reset_mock()
ping.return_value = True
self.assertRaises(error.DevopsError, helpers.get_free_port)
ping.assert_has_calls(
[mock.call('localhost', port) for port in xrange(32000, 32100)])
@mock.patch('devops.helpers.subprocess_runner.Subprocess.execute',
return_value=exec_result.ExecResult(
cmd="ping -c 1 -W '{timeout:d}' '{host:s}'".format(
host='127.0.0.1',
timeout=1,
),
exit_code=0,
))
def test_icmp_ping(self, caller):
host = '127.0.0.1'
timeout = 1
result = helpers.icmp_ping(host=host)
caller.assert_called_once_with(
"ping -c 1 -W '{timeout:d}' '{host:s}'".format(host=host,
timeout=timeout))
self.assertTrue(result, 'Unexpected result of validation')
@mock.patch('socket.socket')
def test_tcp_ping_(self, sock):
s = mock.Mock()
settimeout = mock.Mock()
connect = mock.Mock()
close = mock.Mock()
s.configure_mock(settimeout=settimeout, connect=connect, close=close)
sock.return_value = s
host = '127.0.0.1'
port = 65535
timeout = 1
helpers.tcp_ping_(host, port)
sock.assert_called_once()
settimeout.assert_not_called()
connect.assert_called_once_with((str(host), int(port)))
close.assert_called_once()
sock.reset_mock()
s.reset_mock()
settimeout.reset_mock()
close.reset_mock()
connect.reset_mock()
helpers.tcp_ping_(host, port, timeout)
sock.assert_called_once()
settimeout.assert_called_once_with(timeout)
connect.assert_called_once_with((str(host), int(port)))
close.assert_called_once()
@mock.patch('devops.helpers.helpers.tcp_ping_', autospec=True)
def test_tcp_ping(self, ping):
host = '127.0.0.1'
port = 65535
timeout = 1
result = helpers.tcp_ping(host, port, timeout)
ping.assert_called_once_with(host, port, timeout)
self.assertTrue(result)
ping.reset_mock()
ping.side_effect = socket.error
result = helpers.tcp_ping(host, port, timeout)
ping.assert_called_once_with(host, port, timeout)
self.assertFalse(result)
def test__check_wait_args(self):
helpers._check_wait_args(lambda: None, [], {}, 1, 1)
def test__check_wait_args_wrong_predicate(self):
with self.assertRaises(TypeError):
helpers._check_wait_args('predicate', [], {}, 1, 1)
def test__check_wait_args_wrong_predicate_args(self):
with self.assertRaises(TypeError):
helpers._check_wait_args(lambda: None, 12, {}, 1, 1)
def test__check_wait_args_wrong_predicate_kwargs(self):
with self.assertRaises(TypeError):
helpers._check_wait_args(lambda: None, [], {'one', 'two'}, 1, 1)
def test__check_wait_args_wrong_interval(self):
with self.assertRaises(ValueError):
helpers._check_wait_args(lambda: None, ['one'], {'two': 2}, -2, 1)
def test__check_wait_args_wrong_timeout(self):
with self.assertRaises(ValueError):
helpers._check_wait_args(lambda: None, (1, 2, 3), {}, 10, 0)
@mock.patch('time.sleep', autospec=True)
def test_wait(self, sleep):
predicate = mock.Mock(return_value=True)
result = helpers.wait(predicate, interval=1, timeout=1)
self.assertTrue(result)
predicate.assert_called_once()
sleep.assert_not_called()
sleep.reset_mock()
predicate.reset_mock()
predicate.return_value = 2
result = helpers.wait(predicate, interval=2, timeout=2)
self.assertEqual(result, 2)
predicate.assert_called_once()
sleep.assert_not_called()
sleep.reset_mock()
predicate.reset_mock()
predicate.return_value = False
self.assertRaises(error.TimeoutError,
helpers.wait,
predicate,
interval=1,
timeout=1)
predicate.assert_called()
@mock.patch('time.sleep', autospec=True)
def test_wait_pass(self, sleep):
predicate = mock.Mock(return_value=True)
result = helpers.wait_pass(predicate)
self.assertTrue(result)
predicate.reset_mock()
predicate.side_effect = ValueError
self.assertRaises(error.TimeoutError,
helpers.wait_pass,
predicate,
timeout=1)
@mock.patch('devops.helpers.helpers.tcp_ping', autospec=True)
def test_wait_tcp(self, ping):
host = '127.0.0.1'
port = 65535
timeout = 1
helpers.wait_tcp(host, port, timeout)
ping.assert_called_once_with(host=host, port=port)
@mock.patch('devops.helpers.ssh_client.SSHClient', autospec=True)
@mock.patch('devops.helpers.helpers.wait')
def test_wait_ssh_cmd(self, wait, ssh):
host = '127.0.0.1'
port = 65535
check_cmd = 'ls ~'
username = '******'
password = '******'
timeout = 0
helpers.wait_ssh_cmd(host, port, check_cmd, username, password,
timeout)
ssh.assert_called_once_with(host=host,
port=port,
auth=ssh_client.SSHAuth(username=username,
password=password))
wait.assert_called_once()
# Todo: cover ssh_client.execute
@mock.patch('six.moves.http_client.HTTPConnection', autospec=True)
def test_http(self, connection):
host = 'localhost'
port = 80
method = 'GET'
url = '/'
waited_code = 200
class Res(object):
status = waited_code
conn = mock.Mock()
connection.return_value = conn
request = mock.Mock()
getresponse = mock.Mock(return_value=Res())
conn.configure_mock(getresponse=getresponse, request=request)
result = helpers.http()
self.assertTrue(result)
connection.assert_called_once_with(host, port)
request.assert_called_once_with(method, url)
getresponse.assert_called_once()
connection.reset_mock()
request.reset_mock()
getresponse.reset_mock()
conn.reset_mock()
conn.configure_mock(getresponse=getresponse, request=request)
getresponse.return_value = Res()
result = helpers.http(waited_code=404)
self.assertFalse(result)
connection.assert_called_once_with(host, port)
request.assert_called_once_with(method, url)
getresponse.assert_called_once()
connection.reset_mock()
request.reset_mock()
getresponse.reset_mock()
conn.reset_mock()
conn.configure_mock(getresponse=getresponse, request=request)
getresponse.return_value = Res()
getresponse.side_effect = Exception
result = helpers.http()
self.assertFalse(result)
connection.assert_called_once_with(host, port)
request.assert_called_once_with(method, url)
getresponse.assert_called_once()
@mock.patch('six.moves.xmlrpc_client.Server', autospec=True)
def test_xmlrpctoken(self, srv):
server = mock.Mock()
login = mock.Mock(return_value=True)
server.configure_mock(login=login)
srv.return_value = server
uri = 'http://127.0.0.1'
user = '******'
password = '******'
result = helpers.xmlrpctoken(uri, user, password)
self.assertTrue(result)
srv.assert_called_once_with(uri)
login.assert_called_once_with(user, password)
srv.reset_mock()
server.reset_mock()
login.reset_mock()
server.configure_mock(login=login)
srv.return_value = server
login.side_effect = Exception
self.assertRaises(error.AuthenticationError, helpers.xmlrpctoken, uri,
user, password)
srv.assert_called_once_with(uri)
login.assert_called_once_with(user, password)
@mock.patch('six.moves.xmlrpc_client.Server', autospec=True)
def test_xmlrpcmethod(self, srv):
class Success(object):
success = True
class Fail(object):
def __getattr__(self, item):
raise Exception()
uri = 'http://127.0.0.1'
srv.side_effect = [Success(), Success(), Fail()]
result = helpers.xmlrpcmethod(uri, 'success')
self.assertTrue(result)
srv.assert_called_once_with(uri)
srv.reset_mock()
self.assertRaises(AttributeError, helpers.xmlrpcmethod, uri, 'failure')
srv.assert_called_once_with(uri)
srv.reset_mock()
self.assertRaises(AttributeError, helpers.xmlrpcmethod, uri, 'success')
srv.assert_called_once_with(uri)
@mock.patch('os.urandom', autospec=True)
def test_generate_mac(self, rand):
rand.return_value = b'\x01\x02\x03\x04\x05'
result = helpers.generate_mac()
self.assertEqual(result, '64:01:02:03:04:05')
rand.assert_called_once_with(5)
def test_deepgetattr(self):
# pylint: disable=attribute-defined-outside-init
class Tst(object):
one = 1
tst = Tst()
tst2 = Tst()
tst2.two = Tst()
# pylint: enable=attribute-defined-outside-init
result = helpers.deepgetattr(tst, 'one')
self.assertEqual(result, 1)
result = helpers.deepgetattr(tst2, 'two.one')
self.assertEqual(result, 1)
result = helpers.deepgetattr(tst, 'two.one')
self.assertIsNone(result)
result = helpers.deepgetattr(tst, 'two.one', default=0)
self.assertEqual(result, 0)
result = helpers.deepgetattr(tst2, 'two_one', splitter='_')
self.assertEqual(result, 1)
self.assertRaises(AttributeError,
helpers.deepgetattr,
tst,
'two.one',
do_raise=True)
def test_underscored(self):
result = helpers.underscored('single')
self.assertEqual(result, 'single')
result = helpers.underscored('m', 'u', 'l', 't', 'i', 'p', 'l', 'e')
self.assertEqual(result, 'm_u_l_t_i_p_l_e')
