def cibadmin(command_args, timeout=120):
assert timeout > 0, 'timeout must be greater than zero'
# I think these are "errno" values, but I'm not positive
# but going forward, any additions to this should try to be informative
# about the type of exit code and why it's OK to retry
RETRY_CODES = {
10: "something unknown",
41: "something unknown",
62: "Timer expired",
107: "Transport endpoint is not connected"
}
command_args.insert(0, 'cibadmin')
# NB: This isn't a "true" timeout, in that it won't forcibly stop the
# subprocess after a timeout. We'd need more invasive changes to
# shell._run() for that.
for _ in util.wait(timeout):
result = AgentShell.run(command_args)
if result.rc == 0:
return result
elif result.rc not in RETRY_CODES:
break
# Add some harmless diagnostics which will be visible in the logs.
AgentShell.run(['service', 'corosync', 'status'])
AgentShell.run(['service', 'pacemaker', 'status'])
if result.rc in RETRY_CODES:
raise PacemakerError(
"%s timed out after %d seconds: rc: %s, stderr: %s" %
(" ".join(command_args), timeout, result.rc, result.stderr))
else:
raise AgentShell.CommandExecutionError(result, command_args)
def _wait_for_command(self, command_id, timeout):
"""Wait for at least timeout"""
for _ in util.wait(timeout):
command = self._get_command(command_id)
if command.complete:
return command
for _ in util.wait(RABBITMQ_LONGWAIT_PERIOD):
command = self._get_command(command_id)
if command.complete:
break
if command.complete:
raise AssertionError("Command didn't complete after %s seconds but did after %s seconds" % (timeout, RABBITMQ_LONGWAIT_PERIOD))
else:
raise AssertionError("Command didn't complete even after %s seconds" % RABBITMQ_LONGWAIT_PERIOD)
def _cibadmin(command_args, timeout=120, raise_on_timeout=False):
assert timeout > 0, "timeout must be greater than zero"
# I think these are "errno" values, but I'm not positive
# but going forward, any additions to this should try to be informative
# about the type of exit code and why it's OK to retry
RETRY_CODES = {
10: "something unknown",
41: "something unknown",
62: "Timer expired",
107: "Transport endpoint is not connected",
}
command_args.insert(0, "cibadmin")
# NB: This isn't a "true" timeout, in that it won't forcibly stop the
# subprocess after a timeout. We'd need more invasive changes to
# shell._run() for that.
for _ in util.wait(timeout):
result = AgentShell.run(command_args)
if result.rc == 0:
return result
elif result.rc not in RETRY_CODES:
break
if raise_on_timeout and result.rc in RETRY_CODES:
raise PacemakerError(
"%s timed out after %d seconds: rc: %s, stderr: %s" %
(" ".join(command_args), timeout, result.rc, result.stderr))
return result
def _wait_for_port(self, port):
log.info("Waiting for port %s..." % port)
for _ in util.wait(self.TIMEOUT):
try:
return socket.socket().connect(("localhost", port))
except socket.error:
pass
raise
def _assert_fs_stat(self, fs_id, name, value):
""" Wait until filesystem statistic matches """
initial = self.get_filesystem(fs_id).get(name)
for _ in util.wait(timeout=constants.TEST_TIMEOUT):
fs = self.get_filesystem(fs_id)
if fs.get(name) == value:
return fs
self.assertEqual(fs.get(name), value,
"initial {0}: {1}".format(name, initial))
def wait_alerts(self, expected_alerts, **filters):
"Wait and assert correct number of matching alerts."
expected_alerts.sort()
for _ in util.wait(TEST_TIMEOUT):
alerts = [alert["alert_type"] for alert in self.get_list("/api/alert/", filters)]
alerts.sort()
if alerts == expected_alerts:
return alerts
raise AssertionError(alerts)
def wait_for_action(self, victim, timeout=TEST_TIMEOUT, **filters):
"""
Check victim's available_actions until the desired action is available
or the timeout is reached, filtering on action keys: class_name, state.
"""
for _ in util.wait(timeout):
actions = self.get_json_by_uri(
victim['resource_uri'])['available_actions']
for action in actions:
if all(action.get(key) == filters[key] for key in filters):
return action
actions = [
dict((key, action.get(key)) for key in filters)
for action in actions
]
raise AssertionError('{0} not found in {1}'.format(filters, actions))
def wait_for_action(self, victim, timeout=TEST_TIMEOUT, **filters):
"""
Check victim's available_actions until the desired action is available
or the timeout is reached, filtering on action keys: class_name, state.
"""
for _ in util.wait(timeout):
obj = self.get_json_by_uri(victim["resource_uri"])
actions = obj.get("available_actions", None)
if actions is None:
actions = get_actions(self.chroma_manager, [obj]).json["objects"]
for action in actions:
if all(action.get(key) == filters[key] for key in filters):
return action
actions = [dict((key, action.get(key)) for key in filters) for action in actions]
raise AssertionError("{0} not found in {1}".format(filters, actions))
