def release_guest( self, logger: gluetool.log.ContextAdapter, guest_request: GuestRequest ) -> Result[bool, Failure]: """ Release guest and its resources back to the pool. :param Guest guest: a guest to be destroyed. :rtype: result.Result[bool, str] """ if BeakerPoolData.is_empty(guest_request): return Ok(True) r_job_cancel = self._dispatch_resource_cleanup( logger, job_id=BeakerPoolData.unserialize(guest_request).job_id, guest_request=guest_request ) if r_job_cancel.is_error: return Error(r_job_cancel.unwrap_error()) return Ok(True)
def test_driver_load_or_none( logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, mockpatch: MockPatcher ) -> None: mock_record = MagicMock( name='Pool()<M>', driver='dummy-driver', parameters={} ) mock_pool = MagicMock(name='pool<M>') mockpatch(tft.artemis.drivers.SafeQuery, 'one_or_none').return_value = Ok(mock_record) # type: ignore[attr-defined] mockpatch(tft.artemis.drivers.PoolDriver, '_instantiate').return_value = Ok(mock_pool) r_pool = tft.artemis.drivers.PoolDriver.load_or_none(logger, session, 'dummy-pool') assert r_pool.is_ok pool = r_pool.unwrap() cast(MagicMock, tft.artemis.drivers.PoolDriver._instantiate).assert_called_once_with( logger, 'dummy-driver', 'dummy-pool', mock_record.parameters ) assert pool is mock_pool
def policy_supports_spot_instances( logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: """ If guest request requires spot instance, disallow all pools that lack this capability. """ # If request does not insist on using spot or non-spot instance, we can easily move forward and use any # pool we've been given. if guest_request.environment.spot_instance is None: return Ok(PolicyRuling(allowed_pools=pools)) r_capabilities = collect_pool_capabilities(pools) if r_capabilities.is_error: return Error(r_capabilities.unwrap_error()) pool_capabilities = r_capabilities.unwrap() # Pick only pools whose spot instance support matches the request - a pool cannot support both kinds at the same # time. return Ok( PolicyRuling(allowed_pools=[ pool for pool, capabilities in pool_capabilities if capabilities.supports_spot_instances is guest_request.environment.spot_instance ]))
def policy_timeout_reached(logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: """ Cancel the guest request if it reached a certain age. The threshold is controlled by :py:data:`KNOB_ROUTE_REQUEST_MAX_TIME`. """ r_events = guest_request.fetch_events(session, eventname='created') if r_events.is_error: return Error(r_events.unwrap_error()) events = r_events.unwrap() if not events: return Ok(PolicyRuling(allowed_pools=pools)) r_time = KNOB_ROUTE_REQUEST_MAX_TIME.get_value(session=session) if r_time.is_error: return Error(r_time.unwrap_error()) validity = events[0].updated + datetime.timedelta(seconds=r_time.unwrap()) logger.info(f'event created {events[0].updated}, valid until {validity}') if datetime.datetime.utcnow() > validity: return Ok(PolicyRuling(cancel=True)) return Ok(PolicyRuling(allowed_pools=pools))
def policy_prefer_spot_instances( logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: """ Prefer pools capable of using spot instances to satisfy the request. If there are no such pools, all given pools are returned - *prefer*, not *allow only*. """ # If request does insist on using spot or non-spot instance, we should not mess with its request by # possibly removing the group it requests. For such environments, do nothing and let other policies # apply their magic. if guest_request.environment.spot_instance is not None: return Ok(PolicyRuling(allowed_pools=pools)) r_capabilities = collect_pool_capabilities(pools) if r_capabilities.is_error: return Error(r_capabilities.unwrap_error()) preferred_pools = [ pool for pool, capabilities in r_capabilities.unwrap() if capabilities.supports_spot_instances is True ] if not preferred_pools: return Ok(PolicyRuling(allowed_pools=pools)) return Ok(PolicyRuling(allowed_pools=preferred_pools))
def _update_guest_log_console_url( self, logger: gluetool.log.ContextAdapter, guest_request: GuestRequest, guest_log: GuestLog) -> Result[GuestLogUpdateProgress, Failure]: r_delay_update = KNOB_CONSOLE_BLOB_UPDATE_TICK.get_value( poolname=self.poolname) if r_delay_update.is_error: return Error(r_delay_update.unwrap_error()) delay_update = r_delay_update.unwrap() r_output = self._do_fetch_console(guest_request, 'url') if r_output.is_error: return Error(r_output.unwrap_error()) output = r_output.unwrap() if output is None: return Ok( GuestLogUpdateProgress(state=GuestLogState.IN_PROGRESS, delay_update=delay_update)) return Ok( GuestLogUpdateProgress( state=GuestLogState.COMPLETE, url=cast(Dict[str, str], output)['url'], expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=KNOB_CONSOLE_URL_EXPIRES.value)))
def test_collect_pool_capabilities_error( mock_inputs: MockInputs, monkeypatch: _pytest.monkeypatch.MonkeyPatch, mockpatch: MockPatcher ) -> None: mock_logger, mock_session, mock_pools, mock_guest_request = mock_inputs mockpatch( mock_pools[0], 'capabilities' ).return_value = Ok(MagicMock(name=f'{mock_pools[0].poolname}.capabilities<mock>')) mockpatch( mock_pools[1], 'capabilities' ).return_value = Error(MagicMock(name='failure<mock>')) mockpatch( mock_pools[2], 'capabilities' ).return_value = Ok(MagicMock(name=f'{mock_pools[2].poolname}.capabilities<mock>')) r = tft.artemis.routing_policies.collect_pool_capabilities(mock_pools) assert r.is_error failure = r.unwrap_error() assert isinstance(failure, tft.artemis.Failure) assert failure.caused_by == cast(MagicMock, mock_pools[1]).capabilities.return_value.unwrap_error()
def release_guest(self, logger: gluetool.log.ContextAdapter, guest_request: GuestRequest) -> Result[bool, Failure]: """ Release guest and its resources back to the pool. :param Guest guest: a guest to be destroyed. :rtype: result.Result[bool, str] """ if OpenStackPoolData.is_empty(guest_request): return Ok(True) if guest_request.poolname != self.poolname: return Error(Failure('guest is not owned by this pool')) r_cleanup = self._dispatch_resource_cleanup( logger, instance_id=OpenStackPoolData.unserialize( guest_request).instance_id, guest_request=guest_request) if r_cleanup.is_error: return Error(r_cleanup.unwrap_error()) return Ok(True)
def can_acquire(self, logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, guest_request: GuestRequest) -> Result[bool, Failure]: r_answer = super(OpenStackDriver, self).can_acquire(logger, session, guest_request) if r_answer.is_error: return Error(r_answer.unwrap_error()) if r_answer.unwrap() is False: return r_answer r_image = self.image_info_mapper.map_or_none(logger, guest_request) if r_image.is_error: return Error(r_image.unwrap_error()) if r_image.unwrap() is None: return Ok(False) r_flavor = self._env_to_flavor(logger, session, guest_request) if r_flavor.is_error: return Error(r_flavor.unwrap_error()) if r_flavor.unwrap() is None: return Ok(False) return Ok(True)
def _update_guest_log_console_blob( self, logger: gluetool.log.ContextAdapter, guest_request: GuestRequest, guest_log: GuestLog) -> Result[GuestLogUpdateProgress, Failure]: r_delay_update = KNOB_CONSOLE_BLOB_UPDATE_TICK.get_value( poolname=self.poolname) if r_delay_update.is_error: return Error(r_delay_update.unwrap_error()) delay_update = r_delay_update.unwrap() r_output = self._do_fetch_console(guest_request, 'log', json_format=False) if r_output.is_error: return Error(r_output.unwrap_error()) output = r_output.unwrap() if output is None: return Ok( GuestLogUpdateProgress(state=GuestLogState.IN_PROGRESS, delay_update=delay_update)) return Ok( GuestLogUpdateProgress( state=GuestLogState.IN_PROGRESS, # TODO logs: well, this *is* overwriting what we already downloaded... Do something. blob=cast(str, output), delay_update=delay_update))
def policy_least_crowded(logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: """ Pick the least crowded pools, i.e. pools with the lowest absolute usage. """ if len(pools) <= 1: return Ok(PolicyRuling(allowed_pools=pools)) r_pool_metrics = collect_pool_metrics(pools) if r_pool_metrics.is_error: return Error(r_pool_metrics.unwrap_error()) pool_metrics = r_pool_metrics.unwrap() log_dict(logger.debug, 'pool metrics', pool_metrics) min_usage = min( [metrics.current_guest_request_count for _, metrics in pool_metrics]) return Ok( PolicyRuling(allowed_pools=[ pool for pool, metrics in pool_metrics if metrics.current_guest_request_count == min_usage ]))
def test_unwrap(): o = Ok('foo') n = Error('foo') assert o.unwrap() == 'foo' with pytest.raises(gluetool.GlueError): n.unwrap()
def test_expect_error(): o = Ok('foo') n = Error('foo') with pytest.raises(gluetool.GlueError): o.expect_error('failure') assert n.expect_error('failure') == 'foo'
def test_eq(): assert Ok(1) == Ok(1) assert Error(1) == Error(1) assert Ok(1) != Error(1) assert Ok(1) != Ok(2) assert not (Ok(1) != Ok(1)) assert Ok(1) != 'foo' assert Ok('0') != Ok(0)
def test_unwrap_error(): o = Ok('foo') n = Error('foo') with pytest.raises(gluetool.GlueError): o.unwrap_error() assert n.unwrap_error() == 'foo'
def _fetch_from_env(self, envvar: str) -> Result[Optional[T], 'Failure']: if envvar not in os.environ: return Ok(None) assert self.knob.cast_from_str is not None return Ok( self.knob.cast_from_str(os.environ[envvar]) )
def map_compose_to_imagename_by_pattern_map( logger: gluetool.log.ContextAdapter, pool: PoolDriver, compose_id: str, mapping_filename: Optional[str] = None, mapping_filepath: Optional[str] = None ) -> Result[Optional[str], Failure]: """ Using a given pattern mapping file, try to map a compose to its corresponding image name. Pattern mapping files are described `here <https://gluetool.readthedocs.io/en/latest/gluetool.utils.html#gluetool.utils.PatternMap>`_. :param compose_id: compose ID to translate. :param mapping_filename: if set, pattern mapping file of this name is searched in Artemis' configuration directory. :param mapping_filepath: if set, this pattern mapping file is searched. :returns: either a image name, or :py:class:`tft.artemis.Failure` if the mapping was unsuccessfull. """ if mapping_filepath: pass elif mapping_filename: mapping_filepath = os.path.join(KNOB_CONFIG_DIRPATH.value, mapping_filename) else: return Error( Failure('no compose/image mapping file specified', compose=compose_id)) logger.debug(f'using pattern map {mapping_filepath}') r_cache_enabled = KNOB_CACHE_PATTERN_MAPS.get_value(poolname=pool.poolname) if r_cache_enabled.is_error: return Error(r_cache_enabled.unwrap_error()) r_pattern_map = get_pattern_map(logger, mapping_filepath, use_cache=r_cache_enabled.unwrap()) if r_pattern_map.is_error: return Error(r_pattern_map.unwrap_error().update(compose=compose_id)) pattern_map = r_pattern_map.unwrap() try: imagename = pattern_map.match(compose_id) except gluetool.glue.GlueError: return Ok(None) return Ok(imagename[0] if isinstance(imagename, list) else imagename)
def policy(logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: preferred_pools: List[PoolDriver] = [ pool for pool in pools if isinstance(pool, preferred_drivers) ] if not preferred_pools: return Ok(PolicyRuling(allowed_pools=pools)) return Ok(PolicyRuling(allowed_pools=preferred_pools))
def do_test_policy_supports_snapshots( mock_inputs: MockInputs, require_snapshots: bool, provide_snapshots: bool, monkeypatch: _pytest.monkeypatch.MonkeyPatch ) -> None: mock_logger, mock_session, mock_pools, mock_guest_request = mock_inputs for mock_pool in mock_pools: monkeypatch.setattr( mock_pool, 'capabilities', lambda: Ok(tft.artemis.drivers.PoolCapabilities(supports_snapshots=False)) ) if provide_snapshots: monkeypatch.setattr( mock_pools[0], 'capabilities', lambda: Ok(tft.artemis.drivers.PoolCapabilities(supports_snapshots=True)) ) mock_guest_request.environment = tft.artemis.environment.Environment( hw=tft.artemis.environment.HWRequirements(arch='x86_64'), os=tft.artemis.environment.OsRequirements(compose='dummy-compose'), snapshots=require_snapshots ) r_ruling = tft.artemis.routing_policies.policy_supports_snapshots( mock_logger, mock_session, mock_pools, mock_guest_request ) assert r_ruling.is_ok ruling = r_ruling.unwrap() assert isinstance(ruling, tft.artemis.routing_policies.PolicyRuling) assert ruling.cancel is False if require_snapshots and provide_snapshots: assert ruling.allowed_pools == [mock_pools[0]] elif require_snapshots and not provide_snapshots: assert ruling.allowed_pools == [] elif not require_snapshots: assert ruling.allowed_pools == mock_pools else: assert False, 'unreachable'
def get_pattern_map( logger: gluetool.log.ContextAdapter, filepath: str, use_cache: bool = True) -> Result[gluetool.utils.PatternMap, Failure]: if not use_cache: try: return Ok( gluetool.utils.PatternMap(filepath, allow_variables=True, logger=logger)) except Exception as exc: return Error( Failure.from_exc('cannot load mapping file', exc, filepath=filepath)) def _refresh_cache() -> Result[gluetool.utils.PatternMap, Failure]: try: stat = os.stat(filepath) pattern_map = gluetool.utils.PatternMap(filepath, allow_variables=True, logger=logger) except Exception as exc: return Error( Failure.from_exc('cannot load mapping file', exc, filepath=filepath)) logger.info(f'pattern-map-cache: {filepath} - refreshing') _PATTERN_MAP_CACHE[filepath] = (stat.st_mtime, pattern_map) return Ok(pattern_map) with _PATTERN_MAP_CACHE_LOCK: if filepath not in _PATTERN_MAP_CACHE: logger.debug(f'pattern-map-cache: {filepath} - not in cache') return _refresh_cache() stamp, pattern_map = _PATTERN_MAP_CACHE[filepath] stat = os.stat(filepath) if stat.st_mtime > stamp: logger.warning(f'pattern-map-cache: {filepath} - outdated') return _refresh_cache() logger.debug(f'pattern-map-cache: {filepath} - using cached') return Ok(pattern_map)
def fixture_mock_pool( mockpatch: MockPatcher ) -> Tuple[PoolDriver, Callable[ [gluetool.log.ContextAdapter, sqlalchemy.orm.session.Session, str], Result[ tft.artemis.drivers.PoolDriver, Failure]]]: mock_pool = MagicMock(name='PoolDriver (mock)') mock_pool.release_pool_resources = MagicMock( name='PoolDriver.release_pool_driver (mock)', return_value=Ok(None)) mock_get_pool = mockpatch(tft.artemis.drivers.PoolDriver, 'load') mock_get_pool.return_value = Ok(mock_pool) return mock_pool, mock_get_pool
def _run_os( self, options: List[str], json_format: bool = True, commandname: Optional[str] = None ) -> Result[Union[JSONType, str], Failure]: """ Run os command with additional options and return output in json format :param List(str) options: options for the command :param bool json_format: returns json format if true :param commandname: if specified, driver will increase "CLI calls" metrics for this ``commandname``. :rtype: result.Result[str, Failure] :returns: :py:class:`result.Result` with output, or specification of error. """ # Copy the command base, we don't want to spoil it for others. os_base = self._os_cmd_base[:] # -f(format) option must be placed after a command if json_format: options += ['-f', 'json'] r_run = run_cli_tool(self.logger, os_base + options, json_output=json_format, command_scrubber=lambda cmd: (['openstack'] + options), poolname=self.poolname, commandname=commandname) if r_run.is_error: failure = r_run.unwrap_error() # Detect "instance does not exist" - this error is clearly irrecoverable. No matter how often we would # run this method, we would never evenr made it remove instance that doesn't exist. if test_cli_error(failure, MISSING_INSTANCE_ERROR_PATTERN): failure.recoverable = False PoolMetrics.inc_error(self.poolname, 'missing-instance') return Error(failure) cli_output = r_run.unwrap() if json_format: return Ok(cli_output.json) return Ok(cli_output.stdout)
def policy_use_only_when_addressed( logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: GuestRequest) -> PolicyReturnType: """ Disallow pools that are marked as to be used only when requested by name. """ if guest_request.environment.pool is not None: return Ok(PolicyRuling(allowed_pools=pools)) return Ok( PolicyRuling(allowed_pools=[ pool for pool in pools if pool.use_only_when_addressed is False ]))
def _parse_job_status( self, logger: gluetool.log.ContextAdapter, job_results: bs4.BeautifulSoup ) -> Result[Tuple[str, str], Failure]: """ Parse job results and return its result and status. :param bs4.BeautifulSoup job_results: Job results in xml format. :rtype: result.Result[Tuple[str, str], Failure] :returns: a tuple with two items, job result and status, or specification of error. """ if not job_results.find('job') or len(job_results.find_all('job')) != 1: return Error(Failure( 'job results XML has unknown structure', job_results=job_results.prettify() )) job = job_results.find('job') if not job['result']: return Error(Failure( 'job results XML does not contain result attribute', job_results=job_results.prettify() )) if not job['status']: return Error(Failure( 'job results XML does not contain status attribute', job_results=job_results.prettify() )) return Ok((job['result'].lower(), job['status'].lower()))
def _get_job_results( self, logger: gluetool.log.ContextAdapter, job_id: str ) -> Result[bs4.BeautifulSoup, Failure]: """ Run 'bkr job-results' comand and return job results. :param str job_id: Job id that will be rescheduled. :rtype: result.Result[str, Failure] :returns: :py:class:`result.Result` with job results, or specification of error. """ r_results = self._run_bkr(logger, ['job-results', job_id], commandname='bkr.job-results') if r_results.is_error: return Error(r_results.unwrap_error()) bkr_output = r_results.unwrap() try: return Ok(bs4.BeautifulSoup(bkr_output.stdout, 'xml')) except Exception as exc: return Error(Failure.from_exc( 'failed to parse job results XML', exc, command_output=bkr_output.process_output ))
def _run_bkr( self, logger: gluetool.log.ContextAdapter, options: List[str], commandname: Optional[str] = None ) -> Result[CLIOutput, Failure]: """ Run bkr command with additional options :param gluetool.log.ContextAdapter logger: logger to use for logging. :param List(str) options: options for the command :returns: either a valid result, :py:class:`CLIOutput` instance, or an error with a :py:class:`Failure` describing the problem. """ r_run = run_cli_tool( logger, self._bkr_command + options, json_output=False, poolname=self.poolname, commandname=commandname ) if r_run.is_error: return Error(r_run.unwrap_error()) return Ok(r_run.unwrap())
def refresh_avoid_groups_hostnames(self, logger: ContextAdapter) -> Result[None, Failure]: groups: List[AvoidGroupHostnames] = [] r_avoid_groups = self.avoid_groups if r_avoid_groups.is_error: return Error(r_avoid_groups.unwrap_error()) for groupname in r_avoid_groups.unwrap(): r_list = self._fetch_avoid_group_hostnames(logger, groupname) if r_list.is_error: return Error(r_list.unwrap_error()) groups.append(AvoidGroupHostnames( groupname=groupname, hostnames=r_list.unwrap() )) r_refresh = refresh_cached_set( CACHE.get(), self.avoid_groups_hostnames_cache_key, { h.groupname: h for h in groups } ) if r_refresh.is_error: return Error(r_refresh.unwrap_error()) return Ok(None)
def test_boilerplate(mock_inputs: MockInputs) -> None: mock_return_value: tft.artemis.routing_policies.PolicyReturnType = Ok(MagicMock(name='policy_ruling<mock>')) mock_logger, mock_session, mock_pools, mock_guest_request = mock_inputs @tft.artemis.routing_policies.policy_boilerplate def policy_dummy_whatever( logger: gluetool.log.ContextAdapter, session: sqlalchemy.orm.session.Session, pools: List[PoolDriver], guest_request: tft.artemis.db.GuestRequest ) -> tft.artemis.routing_policies.PolicyReturnType: assert isinstance(logger, tft.artemis.routing_policies.PolicyLogger) assert logger._contexts == {'policy_name': (50, 'dummy-whatever')} assert session is mock_session assert pools is mock_pools assert guest_request is mock_guest_request return mock_return_value r_ruling = policy_dummy_whatever(mock_logger, mock_session, mock_pools, mock_guest_request) assert r_ruling is mock_return_value assert r_ruling.unwrap() is mock_return_value.unwrap()
def test_collect_pool_capabilities( mock_inputs: MockInputs, mockpatch: MockPatcher ) -> None: mock_logger, mock_session, mock_pools, mock_guest_request = mock_inputs mock_capabilities = [ MagicMock(name=f'{pool.poolname}.capabilities<mock>') for pool in mock_pools ] for pool, capabilities in zip(mock_pools, mock_capabilities): mockpatch(pool, 'capabilities').return_value = Ok(capabilities) r = tft.artemis.routing_policies.collect_pool_capabilities(mock_pools) assert r.is_ok collected = r.unwrap() for i in range(0, len(mock_pools)): expected_pool = mock_pools[i] expected_capabilities = mock_capabilities[i] actual_pool, actual_capabilities = collected[i] assert actual_pool is expected_pool assert actual_capabilities is expected_capabilities
def test_sanity(log): # type: (Any) -> None return_values = [ Error('failed first time'), Error('failed second time'), Ok('finally passed') ] # type: List[Result[str, str]] def _check(): # type: () -> Result[str, str] return return_values.pop(0) wait('dummy check', _check, timeout=10, tick=2) assert len(log.records) == 9 # todo: check decreasing remaining time # pylint: disable=line-too-long assert re.match( r"waiting for condition 'dummy check', timeout \d seconds, check every 2 seconds", log.records[0].message) is not None # Ignore PEP8Bear assert log.records[1].message == 'calling callback function' assert log.records[ 2].message == 'check failed with \'failed first time\', assuming failure' assert re.match(r'\d seconds left, sleeping for 2 seconds$', log.records[3].message) is not None assert log.records[4].message == 'calling callback function' assert log.records[ 5].message == 'check failed with \'failed second time\', assuming failure' assert re.match(r'\d seconds left, sleeping for 2 seconds$', log.records[6].message) is not None assert log.records[7].message == 'calling callback function' assert log.records[8].message == 'check passed, assuming success'