def check_vm_is_accessible_with_ssh(self, vm_ip, username=None, password=None, pkeys=None): """Check that instance is accessible with ssh via floating_ip. :param vm_ip: floating_ip of instance :param username: username to login to instance :param password: password to connect to instance :param pkeys: private keys to connect to instance """ error_msg = 'Instance with ip {0} is not accessible with ssh.'\ .format(vm_ip) def is_accessible(): try: with self.env.get_ssh_to_vm( vm_ip, username, password, pkeys) as vm_remote: vm_remote.execute("date") return True except ssh_exception.SSHException: return False except ssh_exception.NoValidConnectionsError: return False wait(is_accessible, sleep_seconds=10, timeout_seconds=60, waiting_for=error_msg)
def check_vm_is_available(self, vm, username=None, password=None, pkeys=None): """Check that instance is available for connect from controller. :param vm: instance to ping from it compute node :param username: username to login to instance :param password: password to connect to instance :param pkeys: private keys to connect to instance """ vm = self.os_conn.get_instance_detail(vm) srv_host = self.env.find_node_by_fqdn( self.os_conn.get_srv_hypervisor_name(vm)).data['ip'] vm_ip = self.os_conn.get_nova_instance_ips(vm)['floating'] with self.env.get_ssh_to_node(srv_host) as remote: cmd = "ping -c1 {0}".format(vm_ip) waiting_for_msg = ( 'Waiting for instance with ip {0} has ' 'connectivity from node with ip {1}.').format(vm_ip, srv_host) wait(lambda: remote.execute(cmd)['exit_code'] == 0, sleep_seconds=10, timeout_seconds=3 * 10, waiting_for=waiting_for_msg) return self.check_vm_is_accessible_with_ssh( vm_ip, username=username, password=password, pkeys=pkeys)
def reset_computes(self, hostnames, env_name): def get_hipervisors(): return [x for x in self.os_conn.nova.hypervisors.list() if x.hypervisor_hostname in hostnames] node_states = defaultdict(list) def is_nodes_started(): for hypervisor in get_hipervisors(): state = hypervisor.state prev_states = node_states[hypervisor.hypervisor_hostname] if len(prev_states) == 0 or state != prev_states[-1]: prev_states.append(state) return all(x[-2:] == ['down', 'up'] for x in node_states.values()) logger.info('Resetting computes {}'.format(hostnames)) for hostname in hostnames: node = self.env.find_node_by_fqdn(hostname) devops_node = DevopsClient.get_node_by_mac(env_name=env_name, mac=node.data['mac']) devops_node.reset() wait(is_nodes_started, timeout_seconds=10 * 60)
def assign_floating_ip(self, srv, use_neutron=False): if use_neutron: # Find external net id for tenant nets = self.neutron.list_networks()['networks'] err_msg = "Active external network not found in nets:{}" ext_net_ids = [ net['id'] for net in nets if net['router:external'] and net['status'] == "ACTIVE"] assert ext_net_ids, err_msg.format(nets) net_id = ext_net_ids[0] # Find instance port ports = self.neutron.list_ports(device_id=srv.id)['ports'] err_msg = "Not found active ports for instance:{}" assert ports, err_msg.format(srv.id) port = ports[0] # Create floating IP body = {'floatingip': {'floating_network_id': net_id, 'port_id': port['id']}} flip = self.neutron.create_floatingip(body) # Wait active state for port port_id = flip['floatingip']['port_id'] state = lambda: self.neutron.show_port(port_id)['port']['status'] wait(lambda: state() == "ACTIVE") return flip['floatingip'] fl_ips_pool = self.nova.floating_ip_pools.list() if fl_ips_pool: floating_ip = self.nova.floating_ips.create( pool=fl_ips_pool[0].name) self.nova.servers.add_floating_ip(srv, floating_ip) return floating_ip
def clear_l3_agent(self, _ip, router_name, node, wait_for_alive=False): """Clear L3 agent ban and wait until router moved to this node Clear previously banned L3 agent on node wait until router moved to this node :param _ip: ip of server to to execute clear command :param router_name: name of router to wait until it move to node :param node: name of node to clear :param wait_for_alive: """ router = self.os_conn.neutron.list_routers( name=router_name)['routers'][0] with self.env.get_ssh_to_node(_ip) as remote: remote.execute( "pcs resource clear p_neutron-l3-agent {0}".format(node)) logger.info("Clear L3 agent on node {0}".format(node)) # wait for l3 agent alive if wait_for_alive: wait( lambda: self.os_conn.get_l3_for_router( router['id'])['agents'][0]['alive'] is True, timeout_seconds=60 * 3, waiting_for="L3 agent is alive", sleep_seconds=(1, 60) )
def test_set_image_disk_and_ram_size(self, horizon, create_image): """Verify that image limits has influence to flavor choice.""" ram_size = 1024 disk_size = 4 image_name = next(generate_ids('image', length=20)) create_image(image_name, min_disk=disk_size, min_ram=ram_size) with horizon.page_images as page: page.table_images.row( name=image_name).dropdown_menu.item_default.click() with page.form_launch_instance as form: form.item_flavor.click() wait(lambda: form.tab_flavor.table_available_flavors.rows, timeout_seconds=30, sleep_seconds=0.1) for row in form.tab_flavor.table_available_flavors.rows: ram_cell = row.cell('ram') if get_size(ram_cell.value, to='mb') < ram_size: assert ram_cell.label_alert.is_present else: assert not ram_cell.label_alert.is_present disk_cell = row.cell('root_disk') if get_size(disk_cell.value, to='gb') < disk_size: assert disk_cell.label_alert.is_present else: assert not disk_cell.label_alert.is_present form.cancel()
def run_in_split(self, cmd, resize, background): tree = self.i3.get_tree() focused = tree.find_focused() ws = focused.workspace_of() ws.focus() ws.split_vertical() proc = subproc.popen(cmd, async_=True) def focus_changed(): new_focused = self.i3.get_tree().find_focused() return (new_focused != focused and new_focused != ws and new_focused.is_window) waiting.wait( focus_changed, timeout_seconds=2, sleep_seconds=0.1) new = self.i3.get_tree().find_focused() assert new != focused if resize: new.resize_set_ppt(height=resize) if background: focused.focus() status = proc.wait() print(status) sys.exit(status) return status
def test_predicate_sleeps_coelesce(timeline, do_sleep): timeline.predicate_sleep = 0.5 if do_sleep else 0 sleep_seconds = 2 expected_sleep = sleep_seconds - timeline.predicate_sleep timeline.satisfy_after_time = 10 waiting.wait(timeline.predicate, sleep_seconds=2) assert timeline.sleeps_performed == [expected_sleep for i in range(5)]
def wait_for_status(self, status, timeout=EVENT_TIMEOUT): """Wait status value after transit statuses.""" self.wait_for_presence() with self.cell('status') as cell: wait(lambda: cell.value not in self.transit_statuses, timeout_seconds=EVENT_TIMEOUT, sleep_seconds=0.1) assert cell.value == status
def go_to(self, item_names): """Go to page via navigate menu. Arguments: - item_names: list of items of navigate menu. """ container = self last_name = item_names[-1] for item_name in item_names: item = ui.UI( By.XPATH, './li/a[contains(., "{}")]'.format(item_name)) item.container = container if item_name == last_name: item.click() break sub_menu = ui.Block( By.XPATH, ('./li/ul[contains(@class, "collapse") and preceding-sibling' '::a[contains(., "{}")]]'.format(item_name))) sub_menu.container = container if not _is_expanded(sub_menu): item.click() wait(lambda: _is_expanded(sub_menu), timeout_seconds=10, sleep_seconds=0.1) container = sub_menu
def __enter__(self): try: waiting.wait(self._try_acquire) return self except: self.__exit__(*sys.exc_info()) raise
def _prepare_neutron_server_and_env(self, net_count): """Prepares neutron service network count on dhcp agent and prepares env. :param net_count: how many networks musth dhcp agent handle """ def _check_neutron_restart(): try: self.os_conn.list_networks()['networks'] except Exception as e: logger.debug(e) return False return True all_controllers = self.env.get_nodes_by_role('controller') for controller in all_controllers: with controller.ssh() as remote: res = self._apply_new_neutron_param_value(remote, net_count) error_msg = ( 'Neutron service restart with new value failed, ' 'exit code {exit_code},' 'stdout {stdout}, stderr {stderr}').format(**res) assert 0 == res['exit_code'], error_msg wait_msg = "Waiting for neutron is up" wait( lambda: _check_neutron_restart(), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for=wait_msg) self._prepare_openstack_state()
def test_wait_any_satisfied_on_start(predicates, forge, timeline): for index, p in enumerate(predicates): works = (index == 2) p().and_return(works) if works: break forge.replay() waiting.wait(waiting.ANY(predicates)) assert timeline.virtual_time == 0
def test__wait_any_satisfied_on_start(self): for index, p in enumerate(self.predicates): works = (index == 2) p().and_return(works) if works: break self.forge.replay() waiting.wait(waiting.ANY(self.predicates)) self.assertEquals(self.virtual_time, 0)
def test_nexted_stop_iteration_preserve_exception(): expected_exception = StopIteration('foo') def predicate(): raise expected_exception with pytest.raises(StopIteration) as ex: waiting.wait(predicate, timeout_seconds=0) assert ex.value is expected_exception
def warm_start_nodes(self, devops_nodes): for node in devops_nodes: logger.info('Starting node {}'.format(node.name)) node.create() wait(self.check_nodes_get_online_state, timeout_seconds=10 * 60) logger.info('wait until the nodes get online state') for node in self.get_all_nodes(): logger.info('online state of node {0} now is {1}' .format(node.data['name'], node.data['online']))
def wait_for_absence(self, timeout=None): """Wait for ui element absence.""" timeout = timeout or self.timeout try: wait(lambda: not self.is_present, timeout_seconds=timeout, sleep_seconds=0.1) except TimeoutExpired: raise Exception( "{!r} is still present after {} sec".format(self, timeout))
def test_nexted_stop_iteration_preserve_traceback(): expected_exception = StopIteration('foo') def predicate(): raise expected_exception with pytest.raises(StopIteration) as ex: waiting.wait(predicate, timeout_seconds=0) assert ' raise expected_exception' in ex.traceback[-1].statement.lines
def test__wait_all(self): for iteration in range(len(self.predicates) * 2, 0, -1): iteration -= 1 for index, p in reversed(list(enumerate(self.predicates))): if index > iteration: continue p().and_return(index == iteration) self.forge.replay() waiting.wait(waiting.ALL(self.predicates)) self.assertEquals(self.virtual_time, len(self.predicates) * 2 - 1)
def test_wait_all(predicates, forge, timeline): for iteration in range(len(predicates) * 2, 0, -1): iteration -= 1 for index, p in reversed(list(enumerate(predicates))): if index > iteration: continue p().and_return(index == iteration) forge.replay() waiting.wait(waiting.ALL(predicates)) assert timeline.virtual_time == ((len(predicates) * 2) - 1)
def wait_for_all_volumes(self): from time import sleep from waiting import wait def predicate(): try: _ = [partition.get_volume().get_volume_guid() for partition in self.get_partitions()] return True except: logger.exception("Exception in wait_for_all_volumes") return False wait(predicate, timeout_seconds=30)
def destroy_nodes(self, devops_nodes): logger.info('wait until the nodes get offline state') node_ips = [node.get_ip_address_by_network_name('admin') for node in devops_nodes] for node in devops_nodes: node.destroy() wait(lambda: self.check_nodes_get_offline_state(node_ips), timeout_seconds=10 * 60) for node in self.get_all_nodes(): logger.info('online state of node {0} now is {1}' .format(node.data['name'], node.data['online']))
def wait_for_controller(self, cluster, nodes): cluster.download_kubeconfig_no_ingress() self.update_oc_config(nodes, cluster) def check_status(): res = utils.get_assisted_controller_status(cluster.kubeconfig_path) return "Running" in str(res, "utf-8") waiting.wait( lambda: check_status(), timeout_seconds=900, sleep_seconds=30, waiting_for="controller to be running", )
def wait_for_event(self, event_to_find, reference_time, params_list=None, host_id="", timeout=10): logging.info(f"Searching for event: {event_to_find}") if params_list is None: params_list = list() try: waiting.wait( lambda: self._find_event(event_to_find, reference_time, params_list, host_id), timeout_seconds=timeout, sleep_seconds=2, waiting_for="Event: %s" % event_to_find, ) except waiting.exceptions.TimeoutExpired: logging.error(f"Event: {event_to_find} did't found") raise
def main(): deploy_options = handle_arguments() service_url = utils.get_service_url(SERVICE, deploy_options.target, deploy_options.domain, deploy_options.namespace) health_url = f'{service_url}/health' print(f'Wait for {health_url}') waiting.wait(lambda: wait_for_request(health_url), timeout_seconds=TIMEOUT, expected_exceptions=(requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout), sleep_seconds=SLEEP, waiting_for="assisted-service to be healthy")
def test_heartbeat(temp_workdir): with open("replicate.yaml", "w") as f: f.write("repository: file://.replicate/") experiment = replicate.init() heartbeat_path = f".replicate/metadata/heartbeats/{experiment.id}.json" wait(lambda: os.path.exists(heartbeat_path), timeout_seconds=1, sleep_seconds=0.01) assert json.load(open(heartbeat_path))["experiment_id"] == experiment.id experiment.stop() assert not os.path.exists(heartbeat_path) # check starting and stopping immediately doesn't do anything weird experiment = replicate.init() experiment.stop()
def wait_till_cluster_is_in_status(client, cluster_id, statuses, timeout=consts.NODES_REGISTERED_TIMEOUT, interval=30): log.info("Wait till cluster %s is in status %s", cluster_id, statuses) try: waiting.wait(lambda: client.cluster_get(cluster_id).status in statuses, timeout_seconds=timeout, sleep_seconds=interval, waiting_for="Cluster to be in status %s" % statuses) except: log.info("Cluster: %s", client.cluster_get(cluster_id)) raise
def wait_for_health(self, name, health_check=None, interval=1, timeout=60): """Container stopped context manager. :param str name: container name as it appears in the docker compose file. :param callable health_check: a callable used to determine if the service has recovered. :param int interval: interval (in seconds) between checks. :param int timeout: timeout (in seconds) for all checks to pass. """ log.debug("Waiting for %s container to be healthy", name) health_check = health_check if health_check else lambda: self.is_container_ready( name) waiting.wait(health_check, sleep_seconds=interval, timeout_seconds=timeout)
def _remote_ping(self, count): cmd = ' '.join(self._prepare_cmd(count)) output_file = tempfile.mktemp() pid = self.remote.background_call(cmd, stdout=output_file) result = PingResult() yield result if count: waiting.wait(lambda: not self.remote.execute('ps -o pid | grep {}'. format(pid)).is_ok, timeout_seconds=count * 10) self.remote.execute('kill -SIGINT {}'.format(pid)) result.stdout = self.remote.check_call( "cat {}".format(output_file)).stdout self.remote.execute('rm {}'.format(output_file))
def main(): deploy_options = handle_arguments() if not deploy_options.apply_manifest: return waiting.wait( lambda: is_assisted_service_ready( target=deploy_options.target, domain=deploy_options.domain, namespace=deploy_options.namespace, disable_tls=deploy_options.disable_tls), timeout_seconds=TIMEOUT, expected_exceptions=(requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout), sleep_seconds=SLEEP, waiting_for="assisted-service to be healthy")
def wait_for_controller_logs(client, cluster_id, timeout, interval=60): try: # if logs_info has any content, the conroller is alive and healthy waiting.wait( lambda: client.cluster_get(cluster_id).logs_info, timeout_seconds=timeout, sleep_seconds=interval, waiting_for="controller logs_info to be filled", ) except BaseException: log.exception( "Failed to wait on start of controller logs on cluster %s", cluster_id) return False
def run_install_flow(client, cluster_id, kubeconfig_path, pull_secret, tf=None): log.info("Verifying cluster exists") cluster = client.cluster_get(cluster_id) log.info("Verifying pull secret") verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret) log.info("Wait till cluster is ready") utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster_id, statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING], ) cluster = client.cluster_get(cluster_id) if cluster.status == consts.ClusterStatus.READY: log.info("Install cluster %s", cluster_id) _install_cluster(client=client, cluster=cluster) else: log.info( "Cluster is already in installing status, skipping install command" ) log.info("Download kubeconfig-noingress") client.download_kubeconfig_no_ingress(cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) wait_till_installed(client=client, cluster=cluster) log.info("Download kubeconfig") waiting.wait( lambda: client.download_kubeconfig( cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) is None, timeout_seconds=240, sleep_seconds=20, expected_exceptions=Exception, waiting_for="Kubeconfig", ) # set new vips if tf: cluster_info = client.cluster_get(cluster.id) if not cluster_info.api_vip: cluster_info.api_vip = helper_cluster.get_api_vip_from_cluster( client, cluster_info) tf.set_new_vip(cluster_info.api_vip)
def ban_dhcp_agent(self, node_to_ban, host, network_name=None, wait_for_die=True, wait_for_rescheduling=True): """Ban DHCP agent and wait until agents rescheduling. Ban dhcp agent on same node as network placed and wait until agents rescheduling. :param node_to_ban: dhcp-agent host to ban :param host: host or ip of controller onto execute ban command :param network_name: name of network to determine node with dhcp agents :param wait_for_die: wait until dhcp-agent die :param wait_for_rescheduling: wait new dhcp-agent starts :returns: str, name of banned node """ list_dhcp_agents = lambda: self.os_conn.list_all_neutron_agents( agent_type='dhcp', filter_attr='host') if network_name: network = self.os_conn.neutron.list_networks( name=network_name)['networks'][0] list_dhcp_agents = ( lambda: self.os_conn.get_node_with_dhcp_for_network( network['id'])) current_agents = list_dhcp_agents() # ban dhcp agent on provided node with self.env.get_ssh_to_node(host) as remote: remote.execute( "pcs resource ban p_neutron-dhcp-agent {0}".format( node_to_ban)) logger.info("Ban DHCP agent on node {0}".format(node_to_ban)) # Wait to die banned dhcp agent if wait_for_die: err_msg = "Awainting ban of DHCP agent: {0}" wait( lambda: (node_to_ban not in list_dhcp_agents()), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for=err_msg.format(node_to_ban)) # Wait to reschedule dhcp agent if wait_for_rescheduling: err_msg = "New DHCP agent wasn't rescheduled" wait( lambda: (set(list_dhcp_agents()) - set(current_agents)), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for=err_msg) return node_to_ban
def wait_for_networking( self, timeout=3 * consts.MINUTE, interval=consts.DEFAULT_CHECK_STATUSES_INTERVAL, ): log.info("Wait till %s nodes have MAC and IP address", len(self.nodes)) # Best effort with SuppressAndLog(waiting.TimeoutExpired): waiting.wait( lambda: self._are_nodes_network_prepared(), timeout_seconds=timeout, sleep_seconds=interval, waiting_for="nodes to have IP and MAC addresses", )
def download_logs_from_all_hosts(client, cluster_id): output_folder = f'build/{cluster_id}' utils.recreate_folder(output_folder) hosts = client.get_cluster_hosts(cluster_id=cluster_id) for host in hosts: output_file = os.path.join(output_folder, f'{host["id"]}_logs.tar.gz') waiting.wait( lambda: client.download_host_logs(cluster_id=cluster_id, host_id=host["id"], output_file=output_file) is None, timeout_seconds=240, sleep_seconds=20, expected_exceptions=Exception, waiting_for="Logs", )
def create_nodes_and_wait_till_registered(inventory_client, cluster, image_path, storage_path, master_count, nodes_details): nodes_count = master_count + nodes_details["worker_count"] create_nodes(image_path, storage_path=storage_path, master_count=master_count, nodes_details=nodes_details) utils.wait_till_nodes_are_ready(nodes_count=nodes_count, cluster_name=nodes_details["cluster_name"]) if not inventory_client: log.info("No inventory url, will not wait till nodes registration") return log.info("Wait till nodes will be registered") waiting.wait(lambda: len(inventory_client.get_cluster_hosts(cluster.id)) >= nodes_count, timeout_seconds=consts.NODES_REGISTERED_TIMEOUT, sleep_seconds=5, waiting_for="Nodes to be registered in inventory service") log.info("Registered nodes are:") pprint.pprint(inventory_client.get_cluster_hosts(cluster.id))
def send_tokens_to_unchanging_user(address, tokens, fee, external_api, internal_api): def get_balance(k): return get_account_balance(external_api, internal_api, k).balance bal0 = get_balance(address) spend_tx_obj = SpendTx(recipient_pubkey=address, amount=tokens, fee=fee, ttl=100, payload="sending tokens") internal_api.post_spend_tx(spend_tx_obj) wait(lambda: get_balance(address) == (bal0 + tokens), timeout_seconds=120, sleep_seconds=0.25)
def wait_for_health(health_check, interval=1, timeout=60): try: return waiting.wait(health_check, sleep_seconds=interval, timeout_seconds=timeout) except waiting.TimeoutExpired: return False
def start_server(port, app_module=None, log_file_name=None, procname='hammock-test'): args = app.command(port, app_module, procname) kwargs = {} log_file = None if log_file_name: log_file = open(log_file_name, 'w') args += ['--logger', 'file:{}'.format(log_file_name)] kwargs = {'stderr': log_file, 'stdout': log_file} server = subprocess.Popen(args, **kwargs) waiting.wait( functools.partial(testing.test_connection, ("localhost", port)), timeout_seconds=10, waiting_for="server to start listening", ) return server, log_file
def run(self, n_works): while n_works: try: waiting.wait(lambda: len(self.queue) > 0, timeout_seconds=240) except TimeoutExpired: break input = self.queue.popleft() result = self.process(input) if self.next_unit: self.next_unit.enqueue(result) else: self.result_queue.append(result) n_works = n_works - 1
def wait_done(self, name, build_number, output_progress=False): info = {'out': 0} def is_build_done(): build_info = self.get_build_info(name, build_number) if output_progress: output = self.get_build_console_output(name, build_number) rest = output[info['out']:] if rest.strip(): logging.info(rest) info['out'] += len(rest) if build_info['result']: return build_info return None init_build_info = self.get_build_info(name, build_number) logging.info('waiting for done: {}console'.format( init_build_info['url'])) try: return waiting.wait(is_build_done) except KeyboardInterrupt: c = raw_input('choose: [s]top build, else deattach').strip() if c == 's': self.stop_build(name, build_number) raise
def get_ip_for_single_node(cluster_deployment, is_ipv4, timeout=300): agents = cluster_deployment.list_agents() assert len(agents) == 1 agent = agents[0] def get_bmc_address(): interfaces = agent.status().get('inventory', {}).get('interfaces') if not interfaces: return ip_addresses = interfaces[0].get( 'ipV4Addresses' if is_ipv4 else 'ipV6Addresses') if not ip_addresses: return ip_addr = ip_addresses[0] ip_interface = IPv4Interface(ip_addr) if is_ipv4 else IPv6Interface( ip_addr) return str(ip_interface.ip) return waiting.wait( get_bmc_address, sleep_seconds=0.5, timeout_seconds=timeout, waiting_for=f'single node ip of agent {agent.ref}', )
def wait_for_state( self, required_state: str, timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT) -> None: required_state = required_state.lower() def _has_required_state() -> bool: state, _ = self.state(timeout=0.5) return state.lower() == required_state waiting.wait( _has_required_state, timeout_seconds=timeout, waiting_for=f'cluster {self.ref} state to be {required_state}', expected_exceptions=waiting.exceptions.TimeoutExpired)
def create_fraud_warning(api): charge = api.charges.create( valid_charge_req(card=fraud_warning_card_req())) # fmt: off wait( lambda: api.charges.get(charge["id"])["fraudDetails"]["status"] != "in_progress", timeout_seconds=30, expected_exceptions=KeyError, ) # fmt: on fraud_warnings = api.fraud_warnings.list({"limit": 100}) for fraud_warning in fraud_warnings["list"]: if fraud_warning.get("charge") == charge["id"]: return [fraud_warning, charge] raise AssertionError("Expected fraud_warning to be created")
def start_node(name, config_filename=None): if should_start_node(name): print("\nNode " + name + " starting") config_prefix = "" if config_filename != None: if config_filename[0] == "/": # absolute path config_prefix = 'ERL_FLAGS="-config ' + config_filename + '" ' else: config_prefix = 'ERL_FLAGS="-config `pwd`/' + config_filename + '" ' p = os.popen(config_prefix + "make " + name + "-start", "r") while 1: line = p.readline() if not line: break api = external_api(name) wait(lambda: node_online(api), timeout_seconds=10, sleep_seconds=0.5)
def wait_till_nodes_are_ready(nodes_count, cluster_name): log.info("Wait till %s nodes will be ready and have ips", nodes_count) cmd = "%s %s| grep %s | wc -l" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK, cluster_name) try: waiting.wait( lambda: int(run_command(cmd, shell=True).strip()) >= nodes_count, timeout_seconds=consts.NODES_REGISTERED_TIMEOUT * nodes_count, sleep_seconds=10, waiting_for="Nodes to have ips") log.info("All nodes have booted and got ips") except: cmd = "%s %s" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK) log.error("Not all nodes are ready. Current dhcp leases are %s", run_command(cmd, shell=False).strip()) raise
def _wait_started(self, job_in_queue_url): def get_queue_info(): js = self.jenkins_open(jenkins.Request(job_in_queue_url)) return json.loads(js) q_info = get_queue_info() def is_build_started(): progress = get_queue_info() logging.info('waiting for started: {}'.format( _get_queue_info_short(progress))) if ('executable' in progress and 'number' in progress['executable']): build_number = progress['executable']['number'] return build_number else: return None try: return waiting.wait(is_build_started) except KeyboardInterrupt: c = raw_input('choose: [s]top queue, else: deattach').strip() if c == 's': self.cancel_queue(q_info['id']) raise
def wait_till_nodes_are_ready(nodes_count, network_name): log.info("Wait till %s nodes will be ready and have ips", nodes_count) try: waiting.wait( lambda: len(get_network_leases(network_name)) >= nodes_count, timeout_seconds=consts.NODES_REGISTERED_TIMEOUT * nodes_count, sleep_seconds=10, waiting_for="Nodes to have ips", ) log.info("All nodes have booted and got ips") except: log.error( "Not all nodes are ready. Current dhcp leases are %s", get_network_leases(network_name), ) raise
def wait_for_all_volumes(self): from time import sleep from waiting import wait def predicate(): try: _ = [ partition.get_volume().get_volume_guid() for partition in self.get_partitions() ] return True except: logger.exception("Exception in wait_for_all_volumes") return False wait(predicate, timeout_seconds=30)
def _set_host_name_from_node(cls, nodes: List[Node], agent: Agent, is_ipv4: bool) -> None: """ Use the MAC address that is listed in the virt node object to find the interface entry in the host's inventory and take the host name from there The setting of the hostname is required for IPv6 only, because the nodes are booted with hostname equal to localhost which is neither unique not legal name for AI host """ if is_ipv4: return def find_matching_node_and_return_name(): inventory = agent.status().get("inventory", {}) for node in nodes: for mac_address in node.macs: for interface in inventory.get("interfaces", []): if interface["macAddress"].lower() == mac_address.lower(): return node.name hostname = waiting.wait( find_matching_node_and_return_name, timeout_seconds=60, sleep_seconds=1, waiting_for=f"agent={agent.ref} to find a hostname", ) log.info(f"patching agent {agent.ref} with hostname {hostname}") agent.patch(hostname=hostname)
def wait_for_output(self, text, timeout_seconds=10, regex=False, **kwargs): """ :return: line with text or raises TimeoutExpired exception """ seen = self.stdout_offset def finder(): nonlocal seen for line in self.output[seen:]: if regex: match = re.search(text, line) else: match = text in line if match: return line seen += 1 return "" waiting_for = 'Text "%s" is present inside output of "%s" cmd' % ( text, self.cmd) return wait(lambda: self.output and finder(), timeout_seconds, waiting_for=waiting_for, sleep_seconds=0.01, **kwargs)
def start_node(name, config_filename): if should_start_node(name): print("\nNode " + name + " starting") config_prefix = "" if config_filename[0] == "/": # absolute path config_prefix = 'EPOCH_CONFIG="' + config_filename + '" ' else: config_prefix = 'EPOCH_CONFIG="`pwd`/' + config_filename + '" ' print("Starting node with config prefix " + config_prefix) p = os.popen("(cd ../.. && " + config_prefix + "make " + name + "-start;)","r") while 1: line = p.readline() if not line: break ext_api = external_api(name) int_api = internal_api(name) wait(lambda: node_online(ext_api, int_api), timeout_seconds=30, sleep_seconds=0.5)
def check_rule_presence(self, group, rule, present=True, timeout=0): """Verify step to check security group is present.""" group_id = group.id def predicate(): rule_to_check = rule.copy() cidr = rule_to_check.pop('cidr') rule_to_check['ip_range'] = {'cidr': cidr} try: group = self._client.security_groups.get(group_id) assert_that(group.rules, has_item(has_entries(rule_to_check))) return present except AssertionError: return not present wait(predicate, timeout_seconds=timeout)
def filter_users(self, query, check=True): """Step to filter users.""" page_users = self.page_users() page_users.field_filter_users.value = query page_users.button_filter_users.click() pom.sleep(1, 'Wait table will be refreshed') if check: def check_rows(): for row in page_users.table_users.rows: if not (row.is_present and query in row.link_username.value): return False return True wait(check_rows, timeout_seconds=10, sleep_seconds=0.1)
def wait(self, *args, **kwargs): """ Wrapping 'wait()' method of 'waiting' library with default parameter values. WebDriverException is ignored in the expected exceptions by default. """ kwargs.setdefault('sleep_seconds', (1, None)) kwargs.setdefault('expected_exceptions', WebDriverException) kwargs.setdefault('timeout_seconds', 30) return wait(*args, **kwargs)
def ban_l3_agent(self, _ip, router_name, wait_for_migrate=True, wait_for_die=True): """Ban L3 agent and wait until router rescheduling Ban L3 agent on same node as router placed and wait until router rescheduling :param _ip: ip of server to to execute ban command :param router_name: name of router to determine node with L3 agent :param wait_for_migrate: wait until router migrate to new controller :param wait_for_die: wait for l3 agent died :returns: str -- name of banned node """ router = self.os_conn.neutron.list_routers( name=router_name)['routers'][0] node_with_l3 = self.os_conn.get_l3_agent_hosts(router['id'])[0] # ban l3 agent on this node with self.env.get_ssh_to_node(_ip) as remote: remote.execute( "pcs resource ban p_neutron-l3-agent {0}".format(node_with_l3)) logger.info("Ban L3 agent on node {0}".format(node_with_l3)) # wait for l3 agent died if wait_for_die: wait( lambda: self.os_conn.get_l3_for_router( router['id'])['agents'][0]['alive'] is False, timeout_seconds=60 * 3, waiting_for="L3 agent is die", sleep_seconds=(1, 60) ) # Wait to migrate l3 agent on new controller if wait_for_migrate: waiting_for = "l3 agent migrate from {0}" wait(lambda: not node_with_l3 == self.os_conn.get_l3_agent_hosts( router['id'])[0], timeout_seconds=60 * 3, waiting_for=waiting_for.format(node_with_l3), sleep_seconds=(1, 60)) return node_with_l3
def disassociate_floating_ip(self, srv, floating_ip, use_neutron=False): if use_neutron: try: self.neutron.update_floatingip( floatingip=floating_ip['id'], body={'floatingip': {}}) id = floating_ip['id'] wait( lambda: self.neutron.show_floatingip(id) ['floatingip']['status'] == "DOWN", timeout_seconds=60) except NeutronClientException: logger.info('The floatingip {} can not be disassociated.' .format(floating_ip['id'])) else: try: self.nova.servers.remove_floating_ip(srv, floating_ip) except NovaClientException: logger.info('The floatingip {} can not be disassociated.' .format(floating_ip))
def wait(self, **kwargs): """Waits for this pact to finish """ _logger.debug("Waiting for {0!r}", self) try: if 'timeout_seconds' not in kwargs and self._end_time is not None: kwargs['timeout_seconds'] = max(0, self._end_time - flux.current_timeline.time()) waiting.wait(self.poll, waiting_for=self, **kwargs) _logger.debug("Finish waiting for {0!r}", self) except TimeoutExpired: exc_info = sys.exc_info() for timeout_callback in self._timeout_callbacks: timeout_callback() exc = self.get_timeout_exception(exc_info) if exc is None: reraise(*exc_info) else: raise exc # pylint: disable=raising-bad-type except Exception: _logger.debug("Exception was raised while waiting for {0!r}", self, exc_info=True) raise
def sort_users(self, reverse=False, check=True): """Step to sort users.""" with self.page_users().table_users as table: table.header.cell('name').click() if reverse: table.header.cell('name').click() pom.sleep(1, 'Wait table will be refreshed') if check: def check_sort(): usernames = [row.link_username.value for row in table.rows] expected_usernames = sorted(usernames) if reverse: expected_usernames = list(reversed(expected_usernames)) return usernames == expected_usernames wait(check_sort, timeout_seconds=10, sleep_seconds=0.1)