def __init__(self, manager, **manager_options):
super(StatefulManagerProxy, self).__init__(manager)
min_polling_interval = float(manager_options.get("min_polling_interval", DEFAULT_MIN_POLLING_INTERVAL))
preprocess_retry_action_kwds = filter_destination_params(manager_options, "preprocess_action_")
postprocess_retry_action_kwds = filter_destination_params(manager_options, "postprocess_action_")
self.__preprocess_action_executor = RetryActionExecutor(**preprocess_retry_action_kwds)
self.__postprocess_action_executor = RetryActionExecutor(**postprocess_retry_action_kwds)
self.min_polling_interval = datetime.timedelta(0, min_polling_interval)
self.active_jobs = ActiveJobs.from_manager(manager)
self.__state_change_callback = self._default_status_change_callback
self.__monitor = None
def __init__(self,
username,
hostname,
password=None,
private_key=None,
port=22,
timeout=60,
**kwargs):
self.username = username
self.hostname = hostname
self.password = password
self.private_key = private_key
self.port = int(port) if port else port
self.timeout = int(timeout) if timeout else timeout
self.ssh = None
self.retry_action_executor = RetryActionExecutor(max_retries=100,
interval_max=300)
self.connect()
def test_exception_passthrough():
action_tracker = ActionTracker(fail_count=1)
exception_raised = False
try:
RetryActionExecutor().execute(action_tracker.execute)
except Exception:
exception_raised = True
assert action_tracker.count == 1
assert exception_raised
def __init__(self, username, hostname, password=None, private_key=None, port=22, timeout=60, **kwargs):
self.username = username
self.hostname = hostname
self.password = password
self.private_key = private_key
self.port = int(port) if port else port
self.timeout = int(timeout) if timeout else timeout
self.ssh = None
self.retry_action_executor = RetryActionExecutor(max_retries=100, interval_max=300)
self.connect()
class ParamikoShell(object):
def __init__(self,
username,
hostname,
password=None,
private_key=None,
port=22,
timeout=60,
strict_host_key_checking=True,
**kwargs):
self.username = username
self.hostname = hostname
self.password = password
self.private_key = private_key
self.port = int(port) if port else port
self.timeout = int(timeout) if timeout else timeout
self.strict_host_key_checking = string_as_bool(
strict_host_key_checking)
self.ssh = None
self.retry_action_executor = RetryActionExecutor(max_retries=100,
interval_max=300)
self.connect()
def connect(self):
log.info("Attempting establishment of new paramiko SSH channel")
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.RejectPolicy(
) if self.strict_host_key_checking else paramiko.WarningPolicy())
self.ssh.load_system_host_keys()
self.ssh.connect(hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
key_filename=self.private_key,
timeout=self.timeout)
def execute(self, cmd, timeout=60):
def retry():
try:
_, stdout, stderr = self._execute(cmd, timeout)
except paramiko.SSHException as e:
log.error(e)
time.sleep(10)
self.connect()
_, stdout, stderr = self._execute(cmd, timeout)
return stdout, stderr
stdout, stderr = self.retry_action_executor.execute(retry)
return_code = stdout.channel.recv_exit_status()
return Bunch(stdout=unicodify(stdout.read()),
stderr=unicodify(stderr.read()),
returncode=return_code)
def _execute(self, cmd, timeout):
return self.ssh.exec_command(smart_str(cmd), timeout=timeout)
def test_third_execution_fine():
action_tracker = ActionTracker(fail_count=2)
exception_raised = False
try:
RetryActionExecutor(max_retries=2,
interval_start=.01,
interval_step=.01).execute(action_tracker.execute)
except Exception:
exception_raised = True
assert action_tracker.count == 3, action_tracker.count
assert not exception_raised
class ParamikoShell(object):
def __init__(self, username, hostname, password=None, private_key=None, port=22, timeout=60, **kwargs):
self.username = username
self.hostname = hostname
self.password = password
self.private_key = private_key
self.port = int(port) if port else port
self.timeout = int(timeout) if timeout else timeout
self.ssh = None
self.retry_action_executor = RetryActionExecutor(max_retries=100, interval_max=300)
self.connect()
def connect(self):
log.info("Attempting establishment of new paramiko SSH channel")
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
key_filename=self.private_key,
timeout=self.timeout)
def execute(self, cmd, timeout=60):
def retry():
try:
_, stdout, stderr = self._execute(cmd, timeout)
except paramiko.SSHException as e:
log.error(e)
time.sleep(10)
self.connect()
_, stdout, stderr = self._execute(cmd, timeout)
return stdout, stderr
stdout, stderr = self.retry_action_executor.execute(retry)
return_code = stdout.channel.recv_exit_status()
return Bunch(stdout=stdout.read(), stderr=stderr.read(), returncode=return_code)
def _execute(self, cmd, timeout):
return self.ssh.exec_command(cmd, timeout=timeout)
def test_retry_defaults():
action_tracker = ActionTracker()
assert RetryActionExecutor().execute(action_tracker.execute) == 42
assert action_tracker.count == 1
