def check_vm_is_accessible_with_ssh(self, vm_ip, username=None,
password=None, pkeys=None):
"""Check that instance is accessible with ssh via floating_ip.
:param vm_ip: floating_ip of instance
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
error_msg = 'Instance with ip {0} is not accessible with ssh.'\
.format(vm_ip)
def is_accessible():
try:
with self.env.get_ssh_to_vm(
vm_ip, username, password, pkeys) as vm_remote:
vm_remote.execute("date")
return True
except ssh_exception.SSHException:
return False
except ssh_exception.NoValidConnectionsError:
return False
wait(is_accessible,
sleep_seconds=10, timeout_seconds=60,
waiting_for=error_msg)
def check_vm_is_available(self, vm,
username=None, password=None, pkeys=None):
"""Check that instance is available for connect from controller.
:param vm: instance to ping from it compute node
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
vm = self.os_conn.get_instance_detail(vm)
srv_host = self.env.find_node_by_fqdn(
self.os_conn.get_srv_hypervisor_name(vm)).data['ip']
vm_ip = self.os_conn.get_nova_instance_ips(vm)['floating']
with self.env.get_ssh_to_node(srv_host) as remote:
cmd = "ping -c1 {0}".format(vm_ip)
waiting_for_msg = (
'Waiting for instance with ip {0} has '
'connectivity from node with ip {1}.').format(vm_ip, srv_host)
wait(lambda: remote.execute(cmd)['exit_code'] == 0,
sleep_seconds=10, timeout_seconds=3 * 10,
waiting_for=waiting_for_msg)
return self.check_vm_is_accessible_with_ssh(
vm_ip, username=username, password=password, pkeys=pkeys)
def reset_computes(self, hostnames, env_name):
def get_hipervisors():
return [x for x in self.os_conn.nova.hypervisors.list()
if x.hypervisor_hostname in hostnames]
node_states = defaultdict(list)
def is_nodes_started():
for hypervisor in get_hipervisors():
state = hypervisor.state
prev_states = node_states[hypervisor.hypervisor_hostname]
if len(prev_states) == 0 or state != prev_states[-1]:
prev_states.append(state)
return all(x[-2:] == ['down', 'up'] for x in node_states.values())
logger.info('Resetting computes {}'.format(hostnames))
for hostname in hostnames:
node = self.env.find_node_by_fqdn(hostname)
devops_node = DevopsClient.get_node_by_mac(env_name=env_name,
mac=node.data['mac'])
devops_node.reset()
wait(is_nodes_started, timeout_seconds=10 * 60)
def assign_floating_ip(self, srv, use_neutron=False):
if use_neutron:
# Find external net id for tenant
nets = self.neutron.list_networks()['networks']
err_msg = "Active external network not found in nets:{}"
ext_net_ids = [
net['id'] for net in nets
if net['router:external'] and net['status'] == "ACTIVE"]
assert ext_net_ids, err_msg.format(nets)
net_id = ext_net_ids[0]
# Find instance port
ports = self.neutron.list_ports(device_id=srv.id)['ports']
err_msg = "Not found active ports for instance:{}"
assert ports, err_msg.format(srv.id)
port = ports[0]
# Create floating IP
body = {'floatingip': {'floating_network_id': net_id,
'port_id': port['id']}}
flip = self.neutron.create_floatingip(body)
# Wait active state for port
port_id = flip['floatingip']['port_id']
state = lambda: self.neutron.show_port(port_id)['port']['status']
wait(lambda: state() == "ACTIVE")
return flip['floatingip']
fl_ips_pool = self.nova.floating_ip_pools.list()
if fl_ips_pool:
floating_ip = self.nova.floating_ips.create(
pool=fl_ips_pool[0].name)
self.nova.servers.add_floating_ip(srv, floating_ip)
return floating_ip
def clear_l3_agent(self, _ip, router_name, node, wait_for_alive=False):
"""Clear L3 agent ban and wait until router moved to this node
Clear previously banned L3 agent on node wait until router moved
to this node
:param _ip: ip of server to to execute clear command
:param router_name: name of router to wait until it move to node
:param node: name of node to clear
:param wait_for_alive:
"""
router = self.os_conn.neutron.list_routers(
name=router_name)['routers'][0]
with self.env.get_ssh_to_node(_ip) as remote:
remote.execute(
"pcs resource clear p_neutron-l3-agent {0}".format(node))
logger.info("Clear L3 agent on node {0}".format(node))
# wait for l3 agent alive
if wait_for_alive:
wait(
lambda: self.os_conn.get_l3_for_router(
router['id'])['agents'][0]['alive'] is True,
timeout_seconds=60 * 3, waiting_for="L3 agent is alive",
sleep_seconds=(1, 60)
)
def test_set_image_disk_and_ram_size(self, horizon, create_image):
"""Verify that image limits has influence to flavor choice."""
ram_size = 1024
disk_size = 4
image_name = next(generate_ids('image', length=20))
create_image(image_name, min_disk=disk_size, min_ram=ram_size)
with horizon.page_images as page:
page.table_images.row(
name=image_name).dropdown_menu.item_default.click()
with page.form_launch_instance as form:
form.item_flavor.click()
wait(lambda: form.tab_flavor.table_available_flavors.rows,
timeout_seconds=30, sleep_seconds=0.1)
for row in form.tab_flavor.table_available_flavors.rows:
ram_cell = row.cell('ram')
if get_size(ram_cell.value, to='mb') < ram_size:
assert ram_cell.label_alert.is_present
else:
assert not ram_cell.label_alert.is_present
disk_cell = row.cell('root_disk')
if get_size(disk_cell.value, to='gb') < disk_size:
assert disk_cell.label_alert.is_present
else:
assert not disk_cell.label_alert.is_present
form.cancel()
def run_in_split(self, cmd, resize, background):
tree = self.i3.get_tree()
focused = tree.find_focused()
ws = focused.workspace_of()
ws.focus()
ws.split_vertical()
proc = subproc.popen(cmd, async_=True)
def focus_changed():
new_focused = self.i3.get_tree().find_focused()
return (new_focused != focused and new_focused != ws
and new_focused.is_window)
waiting.wait(
focus_changed,
timeout_seconds=2,
sleep_seconds=0.1)
new = self.i3.get_tree().find_focused()
assert new != focused
if resize:
new.resize_set_ppt(height=resize)
if background:
focused.focus()
status = proc.wait()
print(status)
sys.exit(status)
return status
def test_predicate_sleeps_coelesce(timeline, do_sleep):
timeline.predicate_sleep = 0.5 if do_sleep else 0
sleep_seconds = 2
expected_sleep = sleep_seconds - timeline.predicate_sleep
timeline.satisfy_after_time = 10
waiting.wait(timeline.predicate, sleep_seconds=2)
assert timeline.sleeps_performed == [expected_sleep for i in range(5)]
def wait_for_status(self, status, timeout=EVENT_TIMEOUT):
"""Wait status value after transit statuses."""
self.wait_for_presence()
with self.cell('status') as cell:
wait(lambda: cell.value not in self.transit_statuses,
timeout_seconds=EVENT_TIMEOUT, sleep_seconds=0.1)
assert cell.value == status
def go_to(self, item_names):
"""Go to page via navigate menu.
Arguments:
- item_names: list of items of navigate menu.
"""
container = self
last_name = item_names[-1]
for item_name in item_names:
item = ui.UI(
By.XPATH, './li/a[contains(., "{}")]'.format(item_name))
item.container = container
if item_name == last_name:
item.click()
break
sub_menu = ui.Block(
By.XPATH,
('./li/ul[contains(@class, "collapse") and preceding-sibling'
'::a[contains(., "{}")]]'.format(item_name)))
sub_menu.container = container
if not _is_expanded(sub_menu):
item.click()
wait(lambda: _is_expanded(sub_menu),
timeout_seconds=10, sleep_seconds=0.1)
container = sub_menu
def __enter__(self):
try:
waiting.wait(self._try_acquire)
return self
except:
self.__exit__(*sys.exc_info())
raise
def _prepare_neutron_server_and_env(self, net_count):
"""Prepares neutron service network count on dhcp agent
and prepares env.
:param net_count: how many networks musth dhcp agent handle
"""
def _check_neutron_restart():
try:
self.os_conn.list_networks()['networks']
except Exception as e:
logger.debug(e)
return False
return True
all_controllers = self.env.get_nodes_by_role('controller')
for controller in all_controllers:
with controller.ssh() as remote:
res = self._apply_new_neutron_param_value(remote, net_count)
error_msg = (
'Neutron service restart with new value failed, '
'exit code {exit_code},'
'stdout {stdout}, stderr {stderr}').format(**res)
assert 0 == res['exit_code'], error_msg
wait_msg = "Waiting for neutron is up"
wait(
lambda: _check_neutron_restart(),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for=wait_msg)
self._prepare_openstack_state()
def test_wait_any_satisfied_on_start(predicates, forge, timeline):
for index, p in enumerate(predicates):
works = (index == 2)
p().and_return(works)
if works:
break
forge.replay()
waiting.wait(waiting.ANY(predicates))
assert timeline.virtual_time == 0
def test__wait_any_satisfied_on_start(self):
for index, p in enumerate(self.predicates):
works = (index == 2)
p().and_return(works)
if works:
break
self.forge.replay()
waiting.wait(waiting.ANY(self.predicates))
self.assertEquals(self.virtual_time, 0)
def test_nexted_stop_iteration_preserve_exception():
expected_exception = StopIteration('foo')
def predicate():
raise expected_exception
with pytest.raises(StopIteration) as ex:
waiting.wait(predicate, timeout_seconds=0)
assert ex.value is expected_exception
def warm_start_nodes(self, devops_nodes):
for node in devops_nodes:
logger.info('Starting node {}'.format(node.name))
node.create()
wait(self.check_nodes_get_online_state, timeout_seconds=10 * 60)
logger.info('wait until the nodes get online state')
for node in self.get_all_nodes():
logger.info('online state of node {0} now is {1}'
.format(node.data['name'], node.data['online']))
def wait_for_absence(self, timeout=None):
"""Wait for ui element absence."""
timeout = timeout or self.timeout
try:
wait(lambda: not self.is_present,
timeout_seconds=timeout, sleep_seconds=0.1)
except TimeoutExpired:
raise Exception(
"{!r} is still present after {} sec".format(self, timeout))
def test_nexted_stop_iteration_preserve_traceback():
expected_exception = StopIteration('foo')
def predicate():
raise expected_exception
with pytest.raises(StopIteration) as ex:
waiting.wait(predicate, timeout_seconds=0)
assert ' raise expected_exception' in ex.traceback[-1].statement.lines
def test__wait_all(self):
for iteration in range(len(self.predicates) * 2, 0, -1):
iteration -= 1
for index, p in reversed(list(enumerate(self.predicates))):
if index > iteration:
continue
p().and_return(index == iteration)
self.forge.replay()
waiting.wait(waiting.ALL(self.predicates))
self.assertEquals(self.virtual_time, len(self.predicates) * 2 - 1)
def test_wait_all(predicates, forge, timeline):
for iteration in range(len(predicates) * 2, 0, -1):
iteration -= 1
for index, p in reversed(list(enumerate(predicates))):
if index > iteration:
continue
p().and_return(index == iteration)
forge.replay()
waiting.wait(waiting.ALL(predicates))
assert timeline.virtual_time == ((len(predicates) * 2) - 1)
def wait_for_all_volumes(self):
from time import sleep
from waiting import wait
def predicate():
try:
_ = [partition.get_volume().get_volume_guid() for partition in self.get_partitions()]
return True
except:
logger.exception("Exception in wait_for_all_volumes")
return False
wait(predicate, timeout_seconds=30)
def destroy_nodes(self, devops_nodes):
logger.info('wait until the nodes get offline state')
node_ips = [node.get_ip_address_by_network_name('admin')
for node in devops_nodes]
for node in devops_nodes:
node.destroy()
wait(lambda: self.check_nodes_get_offline_state(node_ips),
timeout_seconds=10 * 60)
for node in self.get_all_nodes():
logger.info('online state of node {0} now is {1}'
.format(node.data['name'], node.data['online']))
def wait_for_controller(self, cluster, nodes):
cluster.download_kubeconfig_no_ingress()
self.update_oc_config(nodes, cluster)
def check_status():
res = utils.get_assisted_controller_status(cluster.kubeconfig_path)
return "Running" in str(res, "utf-8")
waiting.wait(
lambda: check_status(),
timeout_seconds=900,
sleep_seconds=30,
waiting_for="controller to be running",
)
def wait_for_event(self, event_to_find, reference_time, params_list=None, host_id="", timeout=10):
logging.info(f"Searching for event: {event_to_find}")
if params_list is None:
params_list = list()
try:
waiting.wait(
lambda: self._find_event(event_to_find, reference_time, params_list, host_id),
timeout_seconds=timeout,
sleep_seconds=2,
waiting_for="Event: %s" % event_to_find,
)
except waiting.exceptions.TimeoutExpired:
logging.error(f"Event: {event_to_find} did't found")
raise
def main():
deploy_options = handle_arguments()
service_url = utils.get_service_url(SERVICE, deploy_options.target,
deploy_options.domain,
deploy_options.namespace)
health_url = f'{service_url}/health'
print(f'Wait for {health_url}')
waiting.wait(lambda: wait_for_request(health_url),
timeout_seconds=TIMEOUT,
expected_exceptions=(requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout),
sleep_seconds=SLEEP,
waiting_for="assisted-service to be healthy")
def test_heartbeat(temp_workdir):
with open("replicate.yaml", "w") as f:
f.write("repository: file://.replicate/")
experiment = replicate.init()
heartbeat_path = f".replicate/metadata/heartbeats/{experiment.id}.json"
wait(lambda: os.path.exists(heartbeat_path), timeout_seconds=1, sleep_seconds=0.01)
assert json.load(open(heartbeat_path))["experiment_id"] == experiment.id
experiment.stop()
assert not os.path.exists(heartbeat_path)
# check starting and stopping immediately doesn't do anything weird
experiment = replicate.init()
experiment.stop()
def wait_till_cluster_is_in_status(client,
cluster_id,
statuses,
timeout=consts.NODES_REGISTERED_TIMEOUT,
interval=30):
log.info("Wait till cluster %s is in status %s", cluster_id, statuses)
try:
waiting.wait(lambda: client.cluster_get(cluster_id).status in statuses,
timeout_seconds=timeout,
sleep_seconds=interval,
waiting_for="Cluster to be in status %s" % statuses)
except:
log.info("Cluster: %s", client.cluster_get(cluster_id))
raise
def wait_for_health(self, name, health_check=None, interval=1, timeout=60):
"""Container stopped context manager.
:param str name: container name as it appears in the docker compose file.
:param callable health_check: a callable used to determine if the service has recovered.
:param int interval: interval (in seconds) between checks.
:param int timeout: timeout (in seconds) for all checks to pass.
"""
log.debug("Waiting for %s container to be healthy", name)
health_check = health_check if health_check else lambda: self.is_container_ready(
name)
waiting.wait(health_check,
sleep_seconds=interval,
timeout_seconds=timeout)
def _remote_ping(self, count):
cmd = ' '.join(self._prepare_cmd(count))
output_file = tempfile.mktemp()
pid = self.remote.background_call(cmd, stdout=output_file)
result = PingResult()
yield result
if count:
waiting.wait(lambda: not self.remote.execute('ps -o pid | grep {}'.
format(pid)).is_ok,
timeout_seconds=count * 10)
self.remote.execute('kill -SIGINT {}'.format(pid))
result.stdout = self.remote.check_call(
"cat {}".format(output_file)).stdout
self.remote.execute('rm {}'.format(output_file))
def main():
deploy_options = handle_arguments()
if not deploy_options.apply_manifest:
return
waiting.wait(
lambda: is_assisted_service_ready(
target=deploy_options.target,
domain=deploy_options.domain,
namespace=deploy_options.namespace,
disable_tls=deploy_options.disable_tls),
timeout_seconds=TIMEOUT,
expected_exceptions=(requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout),
sleep_seconds=SLEEP, waiting_for="assisted-service to be healthy")
def wait_for_controller_logs(client, cluster_id, timeout, interval=60):
try:
# if logs_info has any content, the conroller is alive and healthy
waiting.wait(
lambda: client.cluster_get(cluster_id).logs_info,
timeout_seconds=timeout,
sleep_seconds=interval,
waiting_for="controller logs_info to be filled",
)
except BaseException:
log.exception(
"Failed to wait on start of controller logs on cluster %s",
cluster_id)
return False
def run_install_flow(client,
cluster_id,
kubeconfig_path,
pull_secret,
tf=None):
log.info("Verifying cluster exists")
cluster = client.cluster_get(cluster_id)
log.info("Verifying pull secret")
verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret)
log.info("Wait till cluster is ready")
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster_id,
statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING],
)
cluster = client.cluster_get(cluster_id)
if cluster.status == consts.ClusterStatus.READY:
log.info("Install cluster %s", cluster_id)
_install_cluster(client=client, cluster=cluster)
else:
log.info(
"Cluster is already in installing status, skipping install command"
)
log.info("Download kubeconfig-noingress")
client.download_kubeconfig_no_ingress(cluster_id=cluster_id,
kubeconfig_path=kubeconfig_path)
wait_till_installed(client=client, cluster=cluster)
log.info("Download kubeconfig")
waiting.wait(
lambda: client.download_kubeconfig(
cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) is None,
timeout_seconds=240,
sleep_seconds=20,
expected_exceptions=Exception,
waiting_for="Kubeconfig",
)
# set new vips
if tf:
cluster_info = client.cluster_get(cluster.id)
if not cluster_info.api_vip:
cluster_info.api_vip = helper_cluster.get_api_vip_from_cluster(
client, cluster_info)
tf.set_new_vip(cluster_info.api_vip)
def ban_dhcp_agent(self, node_to_ban, host, network_name=None,
wait_for_die=True, wait_for_rescheduling=True):
"""Ban DHCP agent and wait until agents rescheduling.
Ban dhcp agent on same node as network placed and wait until agents
rescheduling.
:param node_to_ban: dhcp-agent host to ban
:param host: host or ip of controller onto execute ban command
:param network_name: name of network to determine node with dhcp agents
:param wait_for_die: wait until dhcp-agent die
:param wait_for_rescheduling: wait new dhcp-agent starts
:returns: str, name of banned node
"""
list_dhcp_agents = lambda: self.os_conn.list_all_neutron_agents(
agent_type='dhcp', filter_attr='host')
if network_name:
network = self.os_conn.neutron.list_networks(
name=network_name)['networks'][0]
list_dhcp_agents = (
lambda: self.os_conn.get_node_with_dhcp_for_network(
network['id']))
current_agents = list_dhcp_agents()
# ban dhcp agent on provided node
with self.env.get_ssh_to_node(host) as remote:
remote.execute(
"pcs resource ban p_neutron-dhcp-agent {0}".format(
node_to_ban))
logger.info("Ban DHCP agent on node {0}".format(node_to_ban))
# Wait to die banned dhcp agent
if wait_for_die:
err_msg = "Awainting ban of DHCP agent: {0}"
wait(
lambda: (node_to_ban not in list_dhcp_agents()),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for=err_msg.format(node_to_ban))
# Wait to reschedule dhcp agent
if wait_for_rescheduling:
err_msg = "New DHCP agent wasn't rescheduled"
wait(
lambda: (set(list_dhcp_agents()) - set(current_agents)),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for=err_msg)
return node_to_ban
def wait_for_networking(
self,
timeout=3 * consts.MINUTE,
interval=consts.DEFAULT_CHECK_STATUSES_INTERVAL,
):
log.info("Wait till %s nodes have MAC and IP address", len(self.nodes))
# Best effort
with SuppressAndLog(waiting.TimeoutExpired):
waiting.wait(
lambda: self._are_nodes_network_prepared(),
timeout_seconds=timeout,
sleep_seconds=interval,
waiting_for="nodes to have IP and MAC addresses",
)
def download_logs_from_all_hosts(client, cluster_id):
output_folder = f'build/{cluster_id}'
utils.recreate_folder(output_folder)
hosts = client.get_cluster_hosts(cluster_id=cluster_id)
for host in hosts:
output_file = os.path.join(output_folder, f'{host["id"]}_logs.tar.gz')
waiting.wait(
lambda: client.download_host_logs(cluster_id=cluster_id,
host_id=host["id"],
output_file=output_file) is None,
timeout_seconds=240,
sleep_seconds=20,
expected_exceptions=Exception,
waiting_for="Logs",
)
def create_nodes_and_wait_till_registered(inventory_client, cluster, image_path, storage_path,
master_count, nodes_details):
nodes_count = master_count + nodes_details["worker_count"]
create_nodes(image_path, storage_path=storage_path, master_count=master_count, nodes_details=nodes_details)
utils.wait_till_nodes_are_ready(nodes_count=nodes_count, cluster_name=nodes_details["cluster_name"])
if not inventory_client:
log.info("No inventory url, will not wait till nodes registration")
return
log.info("Wait till nodes will be registered")
waiting.wait(lambda: len(inventory_client.get_cluster_hosts(cluster.id)) >= nodes_count,
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
sleep_seconds=5, waiting_for="Nodes to be registered in inventory service")
log.info("Registered nodes are:")
pprint.pprint(inventory_client.get_cluster_hosts(cluster.id))
def send_tokens_to_unchanging_user(address, tokens, fee, external_api,
internal_api):
def get_balance(k):
return get_account_balance(external_api, internal_api, k).balance
bal0 = get_balance(address)
spend_tx_obj = SpendTx(recipient_pubkey=address,
amount=tokens,
fee=fee,
ttl=100,
payload="sending tokens")
internal_api.post_spend_tx(spend_tx_obj)
wait(lambda: get_balance(address) == (bal0 + tokens),
timeout_seconds=120,
sleep_seconds=0.25)
def wait_for_health(health_check, interval=1, timeout=60):
try:
return waiting.wait(health_check,
sleep_seconds=interval,
timeout_seconds=timeout)
except waiting.TimeoutExpired:
return False
def start_server(port, app_module=None, log_file_name=None, procname='hammock-test'):
args = app.command(port, app_module, procname)
kwargs = {}
log_file = None
if log_file_name:
log_file = open(log_file_name, 'w')
args += ['--logger', 'file:{}'.format(log_file_name)]
kwargs = {'stderr': log_file, 'stdout': log_file}
server = subprocess.Popen(args, **kwargs)
waiting.wait(
functools.partial(testing.test_connection, ("localhost", port)),
timeout_seconds=10,
waiting_for="server to start listening",
)
return server, log_file
def run(self, n_works):
while n_works:
try:
waiting.wait(lambda: len(self.queue) > 0, timeout_seconds=240)
except TimeoutExpired:
break
input = self.queue.popleft()
result = self.process(input)
if self.next_unit:
self.next_unit.enqueue(result)
else:
self.result_queue.append(result)
n_works = n_works - 1
def wait_done(self, name, build_number, output_progress=False):
info = {'out': 0}
def is_build_done():
build_info = self.get_build_info(name, build_number)
if output_progress:
output = self.get_build_console_output(name, build_number)
rest = output[info['out']:]
if rest.strip():
logging.info(rest)
info['out'] += len(rest)
if build_info['result']:
return build_info
return None
init_build_info = self.get_build_info(name, build_number)
logging.info('waiting for done: {}console'.format(
init_build_info['url']))
try:
return waiting.wait(is_build_done)
except KeyboardInterrupt:
c = raw_input('choose: [s]top build, else deattach').strip()
if c == 's':
self.stop_build(name, build_number)
raise
def get_ip_for_single_node(cluster_deployment, is_ipv4, timeout=300):
agents = cluster_deployment.list_agents()
assert len(agents) == 1
agent = agents[0]
def get_bmc_address():
interfaces = agent.status().get('inventory', {}).get('interfaces')
if not interfaces:
return
ip_addresses = interfaces[0].get(
'ipV4Addresses' if is_ipv4 else 'ipV6Addresses')
if not ip_addresses:
return
ip_addr = ip_addresses[0]
ip_interface = IPv4Interface(ip_addr) if is_ipv4 else IPv6Interface(
ip_addr)
return str(ip_interface.ip)
return waiting.wait(
get_bmc_address,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'single node ip of agent {agent.ref}',
)
def wait_for_state(
self,
required_state: str,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT) -> None:
required_state = required_state.lower()
def _has_required_state() -> bool:
state, _ = self.state(timeout=0.5)
return state.lower() == required_state
waiting.wait(
_has_required_state,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} state to be {required_state}',
expected_exceptions=waiting.exceptions.TimeoutExpired)
def create_fraud_warning(api):
charge = api.charges.create(
valid_charge_req(card=fraud_warning_card_req()))
# fmt: off
wait(
lambda: api.charges.get(charge["id"])["fraudDetails"]["status"] !=
"in_progress",
timeout_seconds=30,
expected_exceptions=KeyError,
)
# fmt: on
fraud_warnings = api.fraud_warnings.list({"limit": 100})
for fraud_warning in fraud_warnings["list"]:
if fraud_warning.get("charge") == charge["id"]:
return [fraud_warning, charge]
raise AssertionError("Expected fraud_warning to be created")
def start_node(name, config_filename=None):
if should_start_node(name):
print("\nNode " + name + " starting")
config_prefix = ""
if config_filename != None:
if config_filename[0] == "/": # absolute path
config_prefix = 'ERL_FLAGS="-config ' + config_filename + '" '
else:
config_prefix = 'ERL_FLAGS="-config `pwd`/' + config_filename + '" '
p = os.popen(config_prefix + "make " + name + "-start", "r")
while 1:
line = p.readline()
if not line: break
api = external_api(name)
wait(lambda: node_online(api), timeout_seconds=10, sleep_seconds=0.5)
def wait_till_nodes_are_ready(nodes_count, cluster_name):
log.info("Wait till %s nodes will be ready and have ips", nodes_count)
cmd = "%s %s| grep %s | wc -l" % (VIRSH_LEASES_COMMAND,
consts.TEST_NETWORK, cluster_name)
try:
waiting.wait(
lambda: int(run_command(cmd, shell=True).strip()) >= nodes_count,
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT * nodes_count,
sleep_seconds=10,
waiting_for="Nodes to have ips")
log.info("All nodes have booted and got ips")
except:
cmd = "%s %s" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK)
log.error("Not all nodes are ready. Current dhcp leases are %s",
run_command(cmd, shell=False).strip())
raise
def _wait_started(self, job_in_queue_url):
def get_queue_info():
js = self.jenkins_open(jenkins.Request(job_in_queue_url))
return json.loads(js)
q_info = get_queue_info()
def is_build_started():
progress = get_queue_info()
logging.info('waiting for started: {}'.format(
_get_queue_info_short(progress)))
if ('executable' in progress
and 'number' in progress['executable']):
build_number = progress['executable']['number']
return build_number
else:
return None
try:
return waiting.wait(is_build_started)
except KeyboardInterrupt:
c = raw_input('choose: [s]top queue, else: deattach').strip()
if c == 's':
self.cancel_queue(q_info['id'])
raise
def wait_till_nodes_are_ready(nodes_count, network_name):
log.info("Wait till %s nodes will be ready and have ips", nodes_count)
try:
waiting.wait(
lambda: len(get_network_leases(network_name)) >= nodes_count,
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT * nodes_count,
sleep_seconds=10,
waiting_for="Nodes to have ips",
)
log.info("All nodes have booted and got ips")
except:
log.error(
"Not all nodes are ready. Current dhcp leases are %s",
get_network_leases(network_name),
)
raise
def wait_for_all_volumes(self):
from time import sleep
from waiting import wait
def predicate():
try:
_ = [
partition.get_volume().get_volume_guid()
for partition in self.get_partitions()
]
return True
except:
logger.exception("Exception in wait_for_all_volumes")
return False
wait(predicate, timeout_seconds=30)
def _set_host_name_from_node(cls, nodes: List[Node], agent: Agent, is_ipv4: bool) -> None:
"""
Use the MAC address that is listed in the virt node object to find the interface entry
in the host's inventory and take the host name from there
The setting of the hostname is required for IPv6 only, because the nodes are booted with
hostname equal to localhost which is neither unique not legal name for AI host
"""
if is_ipv4:
return
def find_matching_node_and_return_name():
inventory = agent.status().get("inventory", {})
for node in nodes:
for mac_address in node.macs:
for interface in inventory.get("interfaces", []):
if interface["macAddress"].lower() == mac_address.lower():
return node.name
hostname = waiting.wait(
find_matching_node_and_return_name,
timeout_seconds=60,
sleep_seconds=1,
waiting_for=f"agent={agent.ref} to find a hostname",
)
log.info(f"patching agent {agent.ref} with hostname {hostname}")
agent.patch(hostname=hostname)
def wait_for_output(self, text, timeout_seconds=10, regex=False, **kwargs):
"""
:return: line with text or raises TimeoutExpired exception
"""
seen = self.stdout_offset
def finder():
nonlocal seen
for line in self.output[seen:]:
if regex:
match = re.search(text, line)
else:
match = text in line
if match:
return line
seen += 1
return ""
waiting_for = 'Text "%s" is present inside output of "%s" cmd' % (
text, self.cmd)
return wait(lambda: self.output and finder(),
timeout_seconds,
waiting_for=waiting_for,
sleep_seconds=0.01,
**kwargs)
def start_node(name, config_filename):
if should_start_node(name):
print("\nNode " + name + " starting")
config_prefix = ""
if config_filename[0] == "/": # absolute path
config_prefix = 'EPOCH_CONFIG="' + config_filename + '" '
else:
config_prefix = 'EPOCH_CONFIG="`pwd`/' + config_filename + '" '
print("Starting node with config prefix " + config_prefix)
p = os.popen("(cd ../.. && " + config_prefix + "make " + name + "-start;)","r")
while 1:
line = p.readline()
if not line: break
ext_api = external_api(name)
int_api = internal_api(name)
wait(lambda: node_online(ext_api, int_api), timeout_seconds=30, sleep_seconds=0.5)
def check_rule_presence(self, group, rule, present=True, timeout=0):
"""Verify step to check security group is present."""
group_id = group.id
def predicate():
rule_to_check = rule.copy()
cidr = rule_to_check.pop('cidr')
rule_to_check['ip_range'] = {'cidr': cidr}
try:
group = self._client.security_groups.get(group_id)
assert_that(group.rules, has_item(has_entries(rule_to_check)))
return present
except AssertionError:
return not present
wait(predicate, timeout_seconds=timeout)
def filter_users(self, query, check=True):
"""Step to filter users."""
page_users = self.page_users()
page_users.field_filter_users.value = query
page_users.button_filter_users.click()
pom.sleep(1, 'Wait table will be refreshed')
if check:
def check_rows():
for row in page_users.table_users.rows:
if not (row.is_present and
query in row.link_username.value):
return False
return True
wait(check_rows, timeout_seconds=10, sleep_seconds=0.1)
def wait(self, *args, **kwargs):
"""
Wrapping 'wait()' method of 'waiting' library with default parameter values.
WebDriverException is ignored in the expected exceptions by default.
"""
kwargs.setdefault('sleep_seconds', (1, None))
kwargs.setdefault('expected_exceptions', WebDriverException)
kwargs.setdefault('timeout_seconds', 30)
return wait(*args, **kwargs)
def ban_l3_agent(self, _ip, router_name, wait_for_migrate=True,
wait_for_die=True):
"""Ban L3 agent and wait until router rescheduling
Ban L3 agent on same node as router placed and wait until router
rescheduling
:param _ip: ip of server to to execute ban command
:param router_name: name of router to determine node with L3 agent
:param wait_for_migrate: wait until router migrate to new controller
:param wait_for_die: wait for l3 agent died
:returns: str -- name of banned node
"""
router = self.os_conn.neutron.list_routers(
name=router_name)['routers'][0]
node_with_l3 = self.os_conn.get_l3_agent_hosts(router['id'])[0]
# ban l3 agent on this node
with self.env.get_ssh_to_node(_ip) as remote:
remote.execute(
"pcs resource ban p_neutron-l3-agent {0}".format(node_with_l3))
logger.info("Ban L3 agent on node {0}".format(node_with_l3))
# wait for l3 agent died
if wait_for_die:
wait(
lambda: self.os_conn.get_l3_for_router(
router['id'])['agents'][0]['alive'] is False,
timeout_seconds=60 * 3, waiting_for="L3 agent is die",
sleep_seconds=(1, 60)
)
# Wait to migrate l3 agent on new controller
if wait_for_migrate:
waiting_for = "l3 agent migrate from {0}"
wait(lambda: not node_with_l3 == self.os_conn.get_l3_agent_hosts(
router['id'])[0], timeout_seconds=60 * 3,
waiting_for=waiting_for.format(node_with_l3),
sleep_seconds=(1, 60))
return node_with_l3
def disassociate_floating_ip(self, srv, floating_ip, use_neutron=False):
if use_neutron:
try:
self.neutron.update_floatingip(
floatingip=floating_ip['id'],
body={'floatingip': {}})
id = floating_ip['id']
wait(
lambda: self.neutron.show_floatingip(id)
['floatingip']['status'] == "DOWN",
timeout_seconds=60)
except NeutronClientException:
logger.info('The floatingip {} can not be disassociated.'
.format(floating_ip['id']))
else:
try:
self.nova.servers.remove_floating_ip(srv, floating_ip)
except NovaClientException:
logger.info('The floatingip {} can not be disassociated.'
.format(floating_ip))
def wait(self, **kwargs):
"""Waits for this pact to finish
"""
_logger.debug("Waiting for {0!r}", self)
try:
if 'timeout_seconds' not in kwargs and self._end_time is not None:
kwargs['timeout_seconds'] = max(0, self._end_time - flux.current_timeline.time())
waiting.wait(self.poll, waiting_for=self, **kwargs)
_logger.debug("Finish waiting for {0!r}", self)
except TimeoutExpired:
exc_info = sys.exc_info()
for timeout_callback in self._timeout_callbacks:
timeout_callback()
exc = self.get_timeout_exception(exc_info)
if exc is None:
reraise(*exc_info)
else:
raise exc # pylint: disable=raising-bad-type
except Exception:
_logger.debug("Exception was raised while waiting for {0!r}", self, exc_info=True)
raise
def sort_users(self, reverse=False, check=True):
"""Step to sort users."""
with self.page_users().table_users as table:
table.header.cell('name').click()
if reverse:
table.header.cell('name').click()
pom.sleep(1, 'Wait table will be refreshed')
if check:
def check_sort():
usernames = [row.link_username.value for row in table.rows]
expected_usernames = sorted(usernames)
if reverse:
expected_usernames = list(reversed(expected_usernames))
return usernames == expected_usernames
wait(check_sort, timeout_seconds=10, sleep_seconds=0.1)
