def update_response_map(self, request, response, append=False): """Record a response for a request. The response_map only contains requests that were either satisfied, or that ran into an exception. Often this translates to reserving hosts against a request. If the rdb hit an exception processing a request, the exception gets recorded in the map for the client to reraise. @param response: A response for the request. @param request: The request that has reserved these hosts. @param append: Boolean, whether to append new hosts in |response| for existing request. Will not append if existing response is a list of exceptions. @raises RDBException: If an empty values is added to the map. """ if not response: raise rdb_utils.RDBException( 'response_map dict can only contain ' 'valid responses. Request %s, response %s is invalid.' % (request, response)) exist_response = self.response_map.setdefault(request, []) if exist_response and not append: raise rdb_utils.RDBException('Request %s already has response %s ' 'the rdb cannot return multiple ' 'responses for the same request.' % (request, response)) if exist_response and append and not isinstance( exist_response[0], rdb_hosts.RDBHost): # Do not append if existing response contains exception. return exist_response.extend(response)
def _check_line(self, line, key): """Sanity check a cache line. This method assumes that a cache line is made up of RDBHost objects, and checks to see if they all match each other/the key passed in. Checking is done in terms of host labels and acls, note that the hosts in the line can have different deps/acls, as long as they all have the deps required by the key, and at least one matching acl of the key. @param line: The cache line value. @param key: The key the line will be stored under. @raises rdb_utils.RDBException: If one of the hosts in the cache line is already leased. The cache already has a different line under the given key. The given key doesn't match the hosts in the line. """ # Note that this doesn't mean that all hosts in the cache are unleased. if any(host.leased for host in line): raise rdb_utils.RDBException('Cannot cache leased hosts %s' % line) # Confirm that the given line can be used to service the key by checking # that all hosts have the deps mentioned in the key, and at least one # matching acl. h_keys = set([self.get_key(host.labels, host.acls) for host in line]) for h_key in h_keys: if (not h_key.deps.issuperset(key.deps) or not key.acls.intersection(h_key.acls)): raise rdb_utils.RDBException( 'Given key: %s does not match key ' 'computed from hosts in line: %s' % (key, h_keys)) if self._cache_backend.has_key(key): raise rdb_utils.RDBException( 'Cannot override a cache line. It ' 'must be cleared before setting. Key: %s, hosts %s' % (key, line))
def __init__(self, **kwargs): for key,value in kwargs.iteritems(): try: hash(value) except TypeError as e: raise rdb_utils.RDBException('All fields of a %s must be. ' 'hashable %s: %s, %s failed this test.' % (self.__class__, key, type(value), value)) try: self._request = self.template(**kwargs) except TypeError: raise rdb_utils.RDBException('Creating %s requires args %s got %s' % (self.__class__, self.template._fields, kwargs.keys()))
def set_line(self, key, hosts): """Cache a list of similar hosts. set_line will no-op if: The hosts aren't all unleased. The hosts don't have deps/acls matching the key. A cache line under the same key already exists. The first 2 cases will lead to a cache miss in the corresponding get. @param hosts: A list of unleased hosts with the same deps/acls. @raises RDBException: If hosts is None, since None is reserved for key expiration. """ if hosts is None: raise rdb_utils.RDBException('Cannot set None in the cache.') # An empty list means no hosts matching the request are available. # This can happen if a previous request leased all matching hosts. if not hosts or not self.use_cache: self._cache_backend.set(key, []) return try: self._check_line(hosts, key) except rdb_utils.RDBException as e: logging.error(e) else: self._cache_backend.set(key, set(hosts))
def testResponseMapChecking(self): """Test response map sanity check. Test that adding the same RDBHostServerWrapper for 2 requests will raise an exception. """ # Assign the same host to 2 requests and check for exceptions. self.get_hosts_manager.add_request(host_id=1) self.get_hosts_manager.add_request(host_id=2) request_1 = self.get_hosts_manager.request_queue[0] request_2 = self.get_hosts_manager.request_queue[1] response = [rdb_testing_utils.FakeHost(hostname='host', host_id=1)] self.handler.update_response_map(request_1, response) self.handler.update_response_map(request_2, response) self.assertRaises(rdb_utils.RDBException, self.handler.get_response) # Assign the same exception to 2 requests and make sure there isn't a # an exception, then check that the response returned is the # exception_string and not the exception itself. self.handler.response_map = {} exception_string = 'This is an exception' response = [rdb_utils.RDBException(exception_string)] self.handler.update_response_map(request_1, response) self.handler.update_response_map(request_2, response) for response in self.handler.get_response().values(): self.assertTrue(response[0] == exception_string)
def valid_host_assignment(cls, request, host): """Check if a host, request pairing is valid. @param request: The request to match against the host. @param host: An RDBServerHostWrapper instance. @return: True if the host, request assignment is valid. @raises RDBException: If the request already has another host_ids associated with it. """ if request.host_id and request.host_id != host.id: raise rdb_utils.RDBException( 'Cannot assign a different host for request: %s, it ' 'already has one: %s ' % (request, host.id)) # Getting all labels and acls might result in large queries, so # bail early if the host is already leased. if host.leased: return False # If a host is invalid it must be a one time host added to the # afe specifically for this purpose, so it doesn't require acl checking. acl_match = (request.acls.intersection(host.acls) or host.invalid) label_match = (request.deps.intersection(host.labels) == request.deps) return acl_match and label_match
def __init__(self, **kwargs): try: kwargs['payload'] = HashableDict(kwargs['payload']) except (KeyError, TypeError) as e: raise rdb_utils.RDBException('Creating %s requires args %s got %s' % (self.__class__, self.template._fields, kwargs.keys())) super(UpdateHostRequest, self).__init__(**kwargs)
def schedule_host_job(cls, host, queue_entry): """Schedule a job on a host. Scheduling a job involves: 1. Setting the active bit on the queue_entry. 2. Scheduling a special task on behalf of the queue_entry. Performing these actions will lead the job scheduler through a chain of events, culminating in running the test and collecting results from the host. @param host: The host against which to schedule the job. @param queue_entry: The queue_entry to schedule. """ if queue_entry.host_id is None: queue_entry.set_host(host) elif host.id != queue_entry.host_id: raise rdb_utils.RDBException( 'The rdb returned host: %s ' 'but the job:%s was already assigned a host: %s. ' % (host.hostname, queue_entry.job_id, queue_entry.host.hostname)) queue_entry.update_field('active', True) # TODO: crbug.com/373936. The host scheduler should only be assigning # jobs to hosts, but the criterion we use to release hosts depends # on it not being used by an active hqe. Since we're activating the # hqe here, we also need to schedule its first prejob task. OTOH, # we could converge to having the host scheduler manager all special # tasks, since their only use today is to verify/cleanup/reset a host. logging.info('Scheduling pre job tasks for entry: %s', queue_entry) queue_entry.schedule_pre_job_tasks()
def _record_exceptions(self, request, exceptions): """Record a list of exceptions for a request. @param request: The request for which the exceptions were hit. @param exceptions: The exceptions hit while processing the request. """ rdb_exceptions = [rdb_utils.RDBException(ex) for ex in exceptions] self.update_response_map(request, rdb_exceptions)
def batch_validate_hosts(self, requests): """Validate requests with hosts. Reserve all hosts, check each one for validity and discard invalid request-host pairings. Lease the remaining hsots. @param requests: A list of requests to validate. @raises RDBException: If multiple hosts or the wrong host is returned for a response. """ # The following cases are possible for frontend requests: # 1. Multiple requests for 1 host, with different acls/deps/priority: # These form distinct requests because they hash differently. # The response map will contain entries like: {r1: h1, r2: h1} # after the batch_get_hosts call. There are 2 sub-cases: # a. Same deps/acls, different priority: # Since we sort the requests based on priority, the # higher priority request r1, will lease h1. The # validation of r2, h1 will fail because of the r1 lease. # b. Different deps/acls, only one of which matches the host: # The matching request will lease h1. The other host # pairing will get dropped from the response map. # 2. Multiple requests with the same acls/deps/priority and 1 host: # These all have the same request hash, so the response map will # contain: {r: h}, regardless of the number of r's. If this is not # a valid host assignment it will get dropped from the response. self.batch_get_hosts(set(requests)) for request in sorted(self.response_map.keys(), key=lambda request: request.priority, reverse=True): hosts = self.response_map[request] if len(hosts) > 1: raise rdb_utils.RDBException( 'Got multiple hosts for a single ' 'request. Hosts: %s, request %s.' % (hosts, request)) # Job-shard is 1:1 mapping. Because a job can only belongs # to one shard, or belongs to master, we disallow frontend job # that spans hosts on and off shards or across multiple shards, # which would otherwise break the 1:1 mapping. # As such, on master, if a request asks for multiple hosts and # if any host is found on shard, we assume other requested hosts # would also be on the same shard. We can safely drop this request. ignore_request = _is_master and any( [host.shard_id for host in hosts]) if (not ignore_request and (self.valid_host_assignment(request, hosts[0]) and self.lease_hosts(hosts))): continue del self.response_map[request] logging.warning('Request %s was not able to lease host %s', request, hosts[0])
def __init__(self, **kwargs): try: kwargs['deps'] = frozenset(kwargs['deps']) kwargs['preferred_deps'] = frozenset(kwargs['preferred_deps']) kwargs['acls'] = frozenset(kwargs['acls']) # parent_job_id defaults to NULL but always serializing it as an int # fits the rdb's type assumptions. Note that job ids are 1 based. if kwargs['parent_job_id'] is None: kwargs['parent_job_id'] = 0 except (KeyError, TypeError) as e: raise rdb_utils.RDBException('Creating %s requires args %s got %s' % (self.__class__, self.template._fields, kwargs.keys())) super(AcquireHostRequest, self).__init__(**kwargs)
def lease(self): """Set the leased bit on the host object, and in the database. @raises RDBException: If the host is already leased. """ self.refresh(fields=['leased']) if self.leased: raise rdb_utils.RDBException('Host %s is already leased' % self.hostname) self.leased = True # TODO: Avoid leaking django out of rdb.QueryManagers. This is still # preferable to calling save() on the host object because we're only # updating/refreshing a single indexed attribute, the leased bit. afe_models.Host.objects.filter(id=self.id).update(leased=self.leased)
def get_required_fields_from_host(cls, host): """Returns all required attributes of the host parsed into a dict. Required attributes are defined as the attributes required to create an RDBHost, and mirror the columns of the host table. @param host: A host object containing all required fields as attributes. """ required_fields_map = {} try: for field in cls.required_fields: required_fields_map[field] = getattr(host, field) except AttributeError as e: raise rdb_utils.RDBException('Required %s' % e) required_fields_map['id'] = host.id return required_fields_map
def _check_response_map(self): """Verify that we never give the same host to different requests. @raises RDBException: If the same host is assigned to multiple requests. """ unique_hosts = set([]) for request, response in self.response_map.iteritems(): # Each value in the response map can only either be a list of # RDBHosts or a list of RDBExceptions, not a mix of both. if isinstance(response[0], rdb_hosts.RDBHost): if any([host in unique_hosts for host in response]): raise rdb_utils.RDBException( 'Assigning the same host to multiple requests. New ' 'hosts %s, request %s, response_map: %s' % (response, request, self.response_map)) else: unique_hosts = unique_hosts.union(response)
def response(self): """Execute the api call and return a response for each request. The order of responses is the same as the order of requests added to the queue. @yield: A response for each request added to the queue after the last invocation of response. """ if not self.request_queue: raise rdb_utils.RDBException('No requests. Call add_requests ' 'with the appropriate kwargs, before calling response.') result = self.api_call(self.request_queue) requests = self.request_queue self.request_queue = [] for request in requests: yield result.get(request) if result else None
def _update(self, payload): """Send an update to rdb, save the attributes of the payload locally. @param: A dictionary representing 'key':value of the update required. @raises RDBException: If the update fails. """ logging.info('Host %s in %s updating %s through rdb on behalf of: %s ', self.hostname, self.status, payload, self.dbg_str) self.update_request_manager.add_request(host_id=self.id, payload=payload) for response in self.update_request_manager.response(): if response: raise rdb_utils.RDBException( 'Host %s unable to perform update ' '%s through rdb on behalf of %s: %s', self.hostname, payload, self.dbg_str, response) super(RDBClientHostWrapper, self)._update_attributes(payload)
def refresh(self, fields=None): """Refresh the attributes on this instance. @param fields: A list of fieldnames to refresh. If None all the required fields of the host are refreshed. @raises RDBException: If refreshing a field fails. """ # TODO: This is mainly required for cache correctness. If it turns # into a bottleneck, cache host_ids instead of rdbhosts and rebuild # the hosts once before leasing them out. The important part is to not # trust the leased bit on a cached host. fields = self.required_fields if not fields else fields try: refreshed_fields = afe_models.Host.objects.filter( id=self.id).values(*fields)[0] except django_exceptions.FieldError as e: raise rdb_utils.RDBException( 'Couldn\'t refresh fields %s: %s' % fields, e) self._update_attributes(refreshed_fields)
def __init__(self, **kwargs): if self.required_fields - set(kwargs.keys()): raise rdb_utils.RDBException( 'Creating %s requires %s, got %s ' % (self.__class__, self.required_fields, kwargs.keys())) self._update_attributes(kwargs)