def wait_for_phase(self, phase, timeout=300, sleep=5): """ Wait till phase of resource is the same as required one passed in the phase parameter. Args: phase (str): Desired phase of resource object timeout (int): Timeout in seconds to wait for desired phase sleep (int): Time in seconds to sleep between attempts Raises: ResourceInUnexpectedState: In case the resource is not in expected phase. NotSupportedFunctionError: If resource doesn't have phase! ResourceNameNotSpecifiedException: in case the name is not specified. """ self.check_function_supported(self._has_phase) self.check_name_is_specified() sampler = TimeoutSampler(timeout, sleep, self.check_phase, phase=phase) if not sampler.wait_for_func_status(True): raise ResourceInUnexpectedState( f"Resource: {self.resource_name} is not in expected phase: " f"{phase}")
def verify_uls_state(self, uls_name, is_available): check_type = "Delete" if is_available: check_type = "Create" sample = TimeoutSampler(timeout=180, sleep=15, func=self.verify_uls_exists, uls_name=uls_name) if sample.wait_for_func_status(result=is_available): logger.info( f"Underlying Storage {uls_name} {check_type.lower()}d successfully." ) else: if is_available: raise ResourceInUnexpectedState( f"{check_type[:-1]}ion of Underlying Storage {uls_name} timed out. " f"Unable to {check_type.lower()} {uls_name}") logger.warning( f"{uls_name} still found after 3 minutes, and might require manual removal." )
def wait_for_phase(self, phase, timeout=300, sleep=5): """ Wait till phase of CSV resource is the same as required one passed in the phase parameter. Args: phase (str): Desired phase of CSV object timeout (int): Timeout in seconds to wait for desired phase sleep (int): Time in seconds to sleep between attempts Raises: ResourceInUnexpectedState: In case the CSV is not in expected phase. """ self.check_name_is_specified() sampler = TimeoutSampler(timeout, sleep, self.check_phase, phase=phase) if not sampler.wait_for_func_status(True): raise ResourceInUnexpectedState( f"CSV: {self.resource_name} is not in expected phase: {phase}")
def wait_for_state(self, state, timeout=300, sleep=5): """ Wait till state of catalog source resource is the same as required one passed in the state parameter. Args: state (str): Desired state of catalog source object timeout (int): Timeout in seconds to wait for desired state sleep (int): Time in seconds to sleep between attempts Raises: ResourceInUnexpectedState: In case the catalog source is not in expected state. """ self.check_name_is_specified() sampler = TimeoutSampler(timeout, sleep, self.check_state, state=state) if not sampler.wait_for_func_status(True): raise ResourceInUnexpectedState( f"Catalog source: {self.resource_name} is not in expected " f"state: {state}")
def wait_for_resource( self, condition, resource_name="", column="STATUS", selector=None, resource_count=0, timeout=60, sleep=3, dont_allow_other_resources=False, error_condition=None, ): """ Wait for a resource to reach to a desired condition Args: condition (str): The desired state the resource that is sampled from 'oc get <kind> <resource_name>' command resource_name (str): The name of the resource to wait for (e.g.my-pv1) column (str): The name of the column to compare with selector (str): The resource selector to search with. Example: 'app=rook-ceph-mds' resource_count (int): How many resources expected to be timeout (int): Time in seconds to wait sleep (int): Sampling time in seconds dont_allow_other_resources (bool): If True it will not allow other resources in different state. For example you are waiting for 2 resources and there are currently 3 (2 in running state, 1 in ContainerCreating) the function will continue to next iteration to wait for only 2 resources in running state and no other exists. error_condition (str): State of the resource that is sampled from 'oc get <kind> <resource_name>' command, which makes this method to fail immediately without waiting for a timeout. This is optional and makes sense only when there is a well defined unrecoverable state of the resource(s) which is not expected to be part of a workflow under test, and at the same time, the timeout itself is large. Returns: bool: True in case all resources reached desired condition, False otherwise """ if condition == error_condition: # when this fails, this method is used in a wrong way raise ValueError( f"Condition '{condition}' we are waiting for must be different" f" from error condition '{error_condition}'" " which describes unexpected error state.") log.info((f"Waiting for a resource(s) of kind {self._kind}" f" identified by name '{resource_name}'" f" using selector {selector}" f" at column name {column}" f" to reach desired condition {condition}")) resource_name = resource_name if resource_name else self.resource_name selector = selector if selector else self.selector # actual status of the resource we are waiting for, setting it to None # now prevents UnboundLocalError raised when waiting timeouts actual_status = None try: for sample in TimeoutSampler(timeout, sleep, self.get, resource_name, True, selector): # Only 1 resource expected to be returned if resource_name: retry = int(timeout / sleep if sleep else timeout / 1) status = self.get_resource( resource_name, column, retry=retry, wait=sleep, ) if status == condition: log.info(f"status of {resource_name} at {column}" " reached condition!") return True log.info(( f"status of {resource_name} at column {column} was {status}," f" but we were waiting for {condition}")) actual_status = status if error_condition is not None and status == error_condition: raise ResourceInUnexpectedState( f"Status of '{resource_name}' at column {column}" f" is {status}.") # More than 1 resources returned elif sample.get("kind") == "List": in_condition = [] in_condition_len = 0 actual_status = [] sample = sample["items"] sample_len = len(sample) for item in sample: try: item_name = item.get("metadata").get("name") status = self.get_resource(item_name, column) actual_status.append(status) if status == condition: in_condition.append(item) in_condition_len = len(in_condition) if (error_condition is not None and status == error_condition): raise ResourceInUnexpectedState( f"Status of '{item_name}' " f" at column {column} is {status}.") except CommandFailed as ex: log.info( f"Failed to get status of resource: {item_name} at column {column}, " f"Error: {ex}") if resource_count: if in_condition_len == resource_count: log.info( f"{in_condition_len} resources already " f"reached condition!") if (dont_allow_other_resources and sample_len != in_condition_len): log.info( f"There are {sample_len} resources in " f"total. Continue to waiting as " f"you don't allow other resources!") continue return True elif len(sample) == len(in_condition): return True # preparing logging message with expected number of # resource items we are waiting for if resource_count > 0: exp_num_str = f"all {resource_count}" else: exp_num_str = "all" log.info(( f"status of {resource_name} at column {column} - item(s) were {actual_status}," f" but we were waiting" f" for {exp_num_str} of them to be {condition}")) except TimeoutExpiredError as ex: log.error(f"timeout expired: {ex}") log.error(( f"Wait for {self._kind} resource {resource_name} at column {column}" f" to reach desired condition {condition} failed," f" last actual status was {actual_status}")) # run `oc describe` on the resources we were waiting for to provide # evidence so that we can understand what was wrong output = self.describe(resource_name, selector=selector) log.warning( "Description of the resource(s) we were waiting for:\n%s", output) raise (ex) except ResourceInUnexpectedState: log.error( ("Waiting for %s resource %s at column %s" " to reach desired condition %s was aborted" " because at least one is in unexpected %s state."), self._kind, resource_name, column, condition, error_condition, ) output = self.describe(resource_name, selector=selector) log.warning( "Description of the resource(s) we were waiting for:\n%s", output) raise return False