def _run_command(protocol_client, shell_id, command,
command_type=util.POWERSHELL,
upper_timeout=CONFIG.argus.upper_timeout):
command_id = None
bare_command = command
thread_pool = pool.ThreadPool(processes=THREADS)
command = util.get_command(command, command_type)
try:
command_id = protocol_client.run_command(shell_id, command)
result = thread_pool.apply_async(
protocol_client.get_command_output,
args=(shell_id, command_id))
stdout, stderr, exit_code = result.get(
timeout=upper_timeout)
if exit_code:
output = "\n\n".join([out for out in (stdout, stderr) if out])
raise exceptions.ArgusError(
"Executing command {command!r} with encoded Command"
"{encoded_command!r} failed with exit code {exit_code!r}"
" and output {output!r}."
.format(command=bare_command,
encoded_command=command,
exit_code=exit_code,
output=output))
return util.sanitize_command_output(stdout), stderr, exit_code
except multiprocessing.TimeoutError:
raise exceptions.ArgusTimeoutError(
"The command '{cmd}' has timed out.".format(cmd=bare_command))
finally:
thread_pool.terminate()
protocol_client.cleanup_command(shell_id, command_id)
def exec_with_retry(action, retry_count, retry_count_interval):
i = 0
while True:
try:
return action()
except Exception:
if i < retry_count:
i += 1
time.sleep(retry_count_interval)
else:
raise exceptions.ArgusTimeoutError(
"{!r} failed too many times.".format(action))
def run_command_until_condition(self, cmd, cond,
retry_count=CONFIG.argus.retry_count,
delay=CONFIG.argus.retry_delay,
command_type=util.POWERSHELL,
upper_timeout=CONFIG.argus.upper_timeout):
"""Run the given `cmd` until a condition `cond` occurs.
:param cond:
A callable which receives the standard output returned by
executing the command. It should return a boolean value,
which tells to this function to stop execution.
:raises:
`ArgusCLIError` if there is output found in the standard error.
This method uses and behaves like `run_command_with_retry` but
with an additional condition parameter.
"""
# countdown normalization
if not retry_count or retry_count < 0:
retry_count = 0
while True:
try:
stdout, stderr, exit_code = self.run_command(
cmd, command_type=command_type,
upper_timeout=upper_timeout)
except Exception as exc: # pylint: disable=broad-except
LOG.debug("Command failed with %r.", exc)
else:
if stderr and exit_code:
raise exceptions.ArgusCLIError(
("Executing command {!r} failed with {!r}"
" and exit code {}.")
.format(cmd, stderr, exit_code))
elif cond(stdout):
return
else:
LOG.debug("Condition not met, retrying...")
if retry_count > 0:
retry_count -= 1
LOG.debug("Retrying '%s'", cmd)
time.sleep(delay)
else:
raise exceptions.ArgusTimeoutError(
"Command {!r} failed too many times."
.format(cmd))
def _run_cmd_until_condition(self,
cmd,
cond,
retry_count=None,
retry_count_interval=5):
"""Run the given `cmd` until a condition *cond* occurs.
:param cmd:
A string, representing a command which needs to
be executed on the underlying remote client.
:param cond:
A callable which receives the stdout returned by
executing the command. It should return a boolean value,
which tells to this function to stop execution.
:param retry_count:
The number of retries which this function has.
If the value is ``None``, then the function will run *forever*.
:param retry_count_interval:
The number of seconds to sleep when retrying a command.
"""
count = 0
while True:
try:
std_out, std_err = self._execute(cmd)
except Exception: # pylint: disable=broad-except
LOG.debug("Command %r failed while waiting for condition", cmd)
count += 1
if retry_count and count >= retry_count:
raise exceptions.ArgusTimeoutError(
"Command {!r} failed too many times.".format(cmd))
time.sleep(retry_count_interval)
else:
if std_err:
raise exceptions.ArgusCLIError(
"Executing command {!r} failed with {!r}".format(
cmd, std_err))
elif cond(std_out):
break
else:
time.sleep(retry_count_interval)
def run_command_with_retry(self, cmd, count=CONFIG.argus.retry_count,
delay=CONFIG.argus.retry_delay,
command_type=util.POWERSHELL,
upper_timeout=CONFIG.argus.upper_timeout):
"""Run the given `cmd` until succeeds.
:param cmd:
A string, representing a command which needs to
be executed on the underlying remote client.
:param count:
The number of retries which this function has.
If the value is ``None``, then the function will retry *forever*.
:param delay:
The number of seconds to sleep when retrying a command.
:rtype: tuple
:returns: stdout, stderr, exit_code
"""
# Countdown normalization.
if not count or count < 0:
count = 0
while True:
try:
return self.run_command(cmd, command_type=command_type,
upper_timeout=upper_timeout)
except Exception as exc: # pylint: disable=broad-except
LOG.debug("Command failed with %r.", exc)
# A negative `count` means no count at all.
if count >= 0:
count -= 1
if count == 0:
raise exceptions.ArgusTimeoutError(
"Command {!r} failed too many times."
.format(cmd))
LOG.debug("Retrying '%s'", cmd)
time.sleep(delay)