def raise_if_needed(self):
"""
Raise an error for the command failure. If the command succeeded, do nothing
"""
if self.aborted_by == AbortedBy.timeout:
self.raise_exception(exception_cls=CommandTimeoutError,
timeout=Duration(self.timeout))
if self.aborted_by == AbortedBy.reboot:
self.raise_exception(exception_cls=CommandAbortedByReboot)
if self.aborted_by == AbortedBy.overflowed:
self.raise_exception(
exception_cls=CommandAbortedByOverflow,
max_output_per_channel=self.max_output_per_channel)
if self.aborted_by == AbortedBy.hanging:
self.raise_exception(exception_cls=CommandHanging)
if self.aborted_by == AbortedBy.orphaned:
self.raise_exception(exception_cls=CommandOrphaned)
if self.aborted_by == AbortedBy.line_timeout:
self.raise_exception(exception_cls=CommandLineTimeout,
line_timeout=self.line_timeout)
if self.aborted_by == AbortedBy.error:
_, traceback = self.get_output(new=False)
traceback = self._decode_output(traceback, safe=True).strip()
exception = traceback.splitlines()[-1]
raise TalkerError(host_id=self.host_id,
hostname=self.hostname,
fulltb=traceback,
_exception=exception)
if self.retcode is None:
return
if self.raise_on_failure and self.retcode not in self.good_codes:
self.raise_exception()
def since_started(self):
"""
Get the time since the command acked
:rtype: Duration
"""
return Duration(time.time() - self.ack) if self.ack else None
def formatted(line, current_ts, last_ts):
fmt = "{:>7}{}"
if (current_ts and last_ts):
return fmt.format(
Duration(current_ts - last_ts).render(show_intervals),
line)
else:
return fmt.format("", line)
def test_wait_log_predicate(get_log):
def pred():
raise PredicateNotSatisfied('bad attempt')
with pytest.raises(TimeoutException):
wait(pred=pred, timeout=.5, sleep=.1, message=False, log_interval=0.2)
durations = re.findall('Still waiting after (.*?): bad attempt', get_log())
rounded_durations = [round(Duration(d), 2) for d in durations]
assert rounded_durations == [
0.2, 0.4
], 'expected logs at 200ms and 400ms, got %s' % (durations, )
def test_client_reboot(self):
timeout = 60.0
cmd = self.client.reboot(self.host_id,
timeout=Duration(timeout),
force=False,
raise_on_failure=True)
sleep(0.1) # write to redis must be completed before we pop from redis
result = self.get_command_from_redis()
expected_value = {
'id': cmd.job_id,
'cmd': 'reboot',
'force': False,
'timeout': None
}
self.assertEqual(result, expected_value)
def __init__(
self,
host_id,
talker,
raise_on_failure=True,
retcode=(0, ),
args=None,
timeout=HOUR,
server_timeout=True,
line_timeout=None,
log_file=None,
name=None,
max_output_per_channel=None,
set_new_logpath=None,
):
self.name = name
self.job_id = str(uuid4())
self.talker = talker
self.retcode = None
self.aborted_by = None
self.ack = None
self.stdout = []
self.stderr = []
self.raise_on_failure = raise_on_failure
self.good_codes = retcode
self.args = args
self.host_id = host_id
self.hostname = host_id.partition('.')[-1]
self.ack_timer = None
self.handling_timer = None
self.timeout = timeout or HOUR # No command should run over hour, unless specified explicitly
self.server_timeout = server_timeout
self.line_timeout = Duration(line_timeout) if line_timeout else None
self.log_file = log_file
self.max_output_per_channel = max_output_per_channel or MAX_OUTPUT_PER_CHANNEL
self.set_new_logpath = set_new_logpath
self.client_timer = Timer(
expiration=self.timeout if not server_timeout else self.timeout +
5)
self._line_timeout_synced = False
self.attempts = 0 # in case of TalkerCommandLost
# this is the version from which the agent acknowledges receipt of command
self.ack_supported = self.talker.agent_version >= V1_3_1
# this is the version from which the agent is ignores params it does not support
self.kwargs_resilient = self.talker.agent_version >= V1_3_1
# this is the version from which the agent is ignores params it does not support
self.pid_supported = self.talker.agent_version >= V1_6_0
def iter_results(self, line_timeout=DAY, timeout=DAY):
"""
Iterate return code and output results
:param [int, float] line_timeout: Timeout for new output to be recieved in seconds
:param [int, float] timeout: Timout for command to finish in seconds
:returns: A Tuple with the channel name and value. Channel names are `retcode`, `stdout` and `stderr`
:rtype: Tuple[str, str]
"""
timeout = timeout or DAY
line_timeout = line_timeout or DAY
blpop_timeout = min(line_timeout, timeout // 10, self.timeout // 10,
AGENT_ACK_TIMEOUT // 10)
session_timer = Timer(
expiration=timeout
) # timeout for this invocation (not entire command duration)
line_timer = Timer(expiration=line_timeout)
timeout_reset_timer = Timer(expiration=max(line_timeout // 10, 1))
self._sync_timeout(line_timeout=line_timeout)
while self.retcode is None:
if line_timer.expired:
self.raise_exception(exception_cls=CommandLineTimeout,
line_timeout=Duration(line_timeout))
if session_timer.expired:
self.raise_exception(exception_cls=CommandTimeout,
timeout=timeout)
channels = {
self._ack_key: 'ack',
self._stderr_key: 'stderr',
self._stdout_key: 'stdout',
self._exit_code_key: 'retcode'
}
res = self.talker.reactor.blpop([
self._ack_key, self._stderr_key, self._stdout_key,
self._exit_code_key
],
timeout=blpop_timeout)
if not res:
if not self.ack:
# no point checking these timers timeout if the command hasn't been acknowledged yet
line_timer.reset()
session_timer.reset()
self.check_client_timeout()
continue
else:
channel_name, value = res
channel_name = channels[channel_name.decode('utf-8')]
line_timer.reset()
if timeout_reset_timer.expired:
timeout_reset_timer.reset()
self.reset_server_timeout()
if channel_name == 'stderr':
self.on_output(self.stderr, (value, ))
elif channel_name == 'stdout':
self.on_output(self.stdout, (value, ))
elif channel_name == 'retcode':
# see if any leftovers after receiving exit code
stdout, stderr = self.get_output()
for data_channel_name, values in (('stdout', stdout),
('stderr', stderr)):
for data_value in values:
yield (data_channel_name, data_value)
value = self.set_retcode(value)
elif channel_name == 'ack':
self.on_ack(value)
continue
yield (channel_name, value)
self.raise_if_needed()
def remain(self):
return Duration(max(0, self.expiration-self.elapsed)) if self.expiration is not None else None
def expired(self):
return Duration(max(0, self.elapsed-self.expiration) if self.expiration is not None else 0)
def elapsed(self):
return Duration(self.elapsed_delta.total_seconds())
def duration(self):
return Duration(self.duration_delta.total_seconds())