def update_response_map(self, request, response, append=False):
"""Record a response for a request.
The response_map only contains requests that were either satisfied, or
that ran into an exception. Often this translates to reserving hosts
against a request. If the rdb hit an exception processing a request, the
exception gets recorded in the map for the client to reraise.
@param response: A response for the request.
@param request: The request that has reserved these hosts.
@param append: Boolean, whether to append new hosts in
|response| for existing request.
Will not append if existing response is
a list of exceptions.
@raises RDBException: If an empty values is added to the map.
"""
if not response:
raise rdb_utils.RDBException(
'response_map dict can only contain '
'valid responses. Request %s, response %s is invalid.' %
(request, response))
exist_response = self.response_map.setdefault(request, [])
if exist_response and not append:
raise rdb_utils.RDBException('Request %s already has response %s '
'the rdb cannot return multiple '
'responses for the same request.' %
(request, response))
if exist_response and append and not isinstance(
exist_response[0], rdb_hosts.RDBHost):
# Do not append if existing response contains exception.
return
exist_response.extend(response)
def _check_line(self, line, key):
"""Sanity check a cache line.
This method assumes that a cache line is made up of RDBHost objects,
and checks to see if they all match each other/the key passed in.
Checking is done in terms of host labels and acls, note that the hosts
in the line can have different deps/acls, as long as they all have the
deps required by the key, and at least one matching acl of the key.
@param line: The cache line value.
@param key: The key the line will be stored under.
@raises rdb_utils.RDBException:
If one of the hosts in the cache line is already leased.
The cache already has a different line under the given key.
The given key doesn't match the hosts in the line.
"""
# Note that this doesn't mean that all hosts in the cache are unleased.
if any(host.leased for host in line):
raise rdb_utils.RDBException('Cannot cache leased hosts %s' % line)
# Confirm that the given line can be used to service the key by checking
# that all hosts have the deps mentioned in the key, and at least one
# matching acl.
h_keys = set([self.get_key(host.labels, host.acls) for host in line])
for h_key in h_keys:
if (not h_key.deps.issuperset(key.deps)
or not key.acls.intersection(h_key.acls)):
raise rdb_utils.RDBException(
'Given key: %s does not match key '
'computed from hosts in line: %s' % (key, h_keys))
if self._cache_backend.has_key(key):
raise rdb_utils.RDBException(
'Cannot override a cache line. It '
'must be cleared before setting. Key: %s, hosts %s' %
(key, line))
def __init__(self, **kwargs):
for key,value in kwargs.iteritems():
try:
hash(value)
except TypeError as e:
raise rdb_utils.RDBException('All fields of a %s must be. '
'hashable %s: %s, %s failed this test.' %
(self.__class__, key, type(value), value))
try:
self._request = self.template(**kwargs)
except TypeError:
raise rdb_utils.RDBException('Creating %s requires args %s got %s' %
(self.__class__, self.template._fields, kwargs.keys()))
def set_line(self, key, hosts):
"""Cache a list of similar hosts.
set_line will no-op if:
The hosts aren't all unleased.
The hosts don't have deps/acls matching the key.
A cache line under the same key already exists.
The first 2 cases will lead to a cache miss in the corresponding get.
@param hosts: A list of unleased hosts with the same deps/acls.
@raises RDBException: If hosts is None, since None is reserved for
key expiration.
"""
if hosts is None:
raise rdb_utils.RDBException('Cannot set None in the cache.')
# An empty list means no hosts matching the request are available.
# This can happen if a previous request leased all matching hosts.
if not hosts or not self.use_cache:
self._cache_backend.set(key, [])
return
try:
self._check_line(hosts, key)
except rdb_utils.RDBException as e:
logging.error(e)
else:
self._cache_backend.set(key, set(hosts))
def testResponseMapChecking(self):
"""Test response map sanity check.
Test that adding the same RDBHostServerWrapper for 2 requests will
raise an exception.
"""
# Assign the same host to 2 requests and check for exceptions.
self.get_hosts_manager.add_request(host_id=1)
self.get_hosts_manager.add_request(host_id=2)
request_1 = self.get_hosts_manager.request_queue[0]
request_2 = self.get_hosts_manager.request_queue[1]
response = [rdb_testing_utils.FakeHost(hostname='host', host_id=1)]
self.handler.update_response_map(request_1, response)
self.handler.update_response_map(request_2, response)
self.assertRaises(rdb_utils.RDBException, self.handler.get_response)
# Assign the same exception to 2 requests and make sure there isn't a
# an exception, then check that the response returned is the
# exception_string and not the exception itself.
self.handler.response_map = {}
exception_string = 'This is an exception'
response = [rdb_utils.RDBException(exception_string)]
self.handler.update_response_map(request_1, response)
self.handler.update_response_map(request_2, response)
for response in self.handler.get_response().values():
self.assertTrue(response[0] == exception_string)
def valid_host_assignment(cls, request, host):
"""Check if a host, request pairing is valid.
@param request: The request to match against the host.
@param host: An RDBServerHostWrapper instance.
@return: True if the host, request assignment is valid.
@raises RDBException: If the request already has another host_ids
associated with it.
"""
if request.host_id and request.host_id != host.id:
raise rdb_utils.RDBException(
'Cannot assign a different host for request: %s, it '
'already has one: %s ' % (request, host.id))
# Getting all labels and acls might result in large queries, so
# bail early if the host is already leased.
if host.leased:
return False
# If a host is invalid it must be a one time host added to the
# afe specifically for this purpose, so it doesn't require acl checking.
acl_match = (request.acls.intersection(host.acls) or host.invalid)
label_match = (request.deps.intersection(host.labels) == request.deps)
return acl_match and label_match
def __init__(self, **kwargs):
try:
kwargs['payload'] = HashableDict(kwargs['payload'])
except (KeyError, TypeError) as e:
raise rdb_utils.RDBException('Creating %s requires args %s got %s' %
(self.__class__, self.template._fields, kwargs.keys()))
super(UpdateHostRequest, self).__init__(**kwargs)
def schedule_host_job(cls, host, queue_entry):
"""Schedule a job on a host.
Scheduling a job involves:
1. Setting the active bit on the queue_entry.
2. Scheduling a special task on behalf of the queue_entry.
Performing these actions will lead the job scheduler through a chain of
events, culminating in running the test and collecting results from
the host.
@param host: The host against which to schedule the job.
@param queue_entry: The queue_entry to schedule.
"""
if queue_entry.host_id is None:
queue_entry.set_host(host)
elif host.id != queue_entry.host_id:
raise rdb_utils.RDBException(
'The rdb returned host: %s '
'but the job:%s was already assigned a host: %s. ' %
(host.hostname, queue_entry.job_id, queue_entry.host.hostname))
queue_entry.update_field('active', True)
# TODO: crbug.com/373936. The host scheduler should only be assigning
# jobs to hosts, but the criterion we use to release hosts depends
# on it not being used by an active hqe. Since we're activating the
# hqe here, we also need to schedule its first prejob task. OTOH,
# we could converge to having the host scheduler manager all special
# tasks, since their only use today is to verify/cleanup/reset a host.
logging.info('Scheduling pre job tasks for entry: %s', queue_entry)
queue_entry.schedule_pre_job_tasks()
def _record_exceptions(self, request, exceptions):
"""Record a list of exceptions for a request.
@param request: The request for which the exceptions were hit.
@param exceptions: The exceptions hit while processing the request.
"""
rdb_exceptions = [rdb_utils.RDBException(ex) for ex in exceptions]
self.update_response_map(request, rdb_exceptions)
def batch_validate_hosts(self, requests):
"""Validate requests with hosts.
Reserve all hosts, check each one for validity and discard invalid
request-host pairings. Lease the remaining hsots.
@param requests: A list of requests to validate.
@raises RDBException: If multiple hosts or the wrong host is returned
for a response.
"""
# The following cases are possible for frontend requests:
# 1. Multiple requests for 1 host, with different acls/deps/priority:
# These form distinct requests because they hash differently.
# The response map will contain entries like: {r1: h1, r2: h1}
# after the batch_get_hosts call. There are 2 sub-cases:
# a. Same deps/acls, different priority:
# Since we sort the requests based on priority, the
# higher priority request r1, will lease h1. The
# validation of r2, h1 will fail because of the r1 lease.
# b. Different deps/acls, only one of which matches the host:
# The matching request will lease h1. The other host
# pairing will get dropped from the response map.
# 2. Multiple requests with the same acls/deps/priority and 1 host:
# These all have the same request hash, so the response map will
# contain: {r: h}, regardless of the number of r's. If this is not
# a valid host assignment it will get dropped from the response.
self.batch_get_hosts(set(requests))
for request in sorted(self.response_map.keys(),
key=lambda request: request.priority,
reverse=True):
hosts = self.response_map[request]
if len(hosts) > 1:
raise rdb_utils.RDBException(
'Got multiple hosts for a single '
'request. Hosts: %s, request %s.' % (hosts, request))
# Job-shard is 1:1 mapping. Because a job can only belongs
# to one shard, or belongs to master, we disallow frontend job
# that spans hosts on and off shards or across multiple shards,
# which would otherwise break the 1:1 mapping.
# As such, on master, if a request asks for multiple hosts and
# if any host is found on shard, we assume other requested hosts
# would also be on the same shard. We can safely drop this request.
ignore_request = _is_master and any(
[host.shard_id for host in hosts])
if (not ignore_request
and (self.valid_host_assignment(request, hosts[0])
and self.lease_hosts(hosts))):
continue
del self.response_map[request]
logging.warning('Request %s was not able to lease host %s',
request, hosts[0])
def __init__(self, **kwargs):
try:
kwargs['deps'] = frozenset(kwargs['deps'])
kwargs['preferred_deps'] = frozenset(kwargs['preferred_deps'])
kwargs['acls'] = frozenset(kwargs['acls'])
# parent_job_id defaults to NULL but always serializing it as an int
# fits the rdb's type assumptions. Note that job ids are 1 based.
if kwargs['parent_job_id'] is None:
kwargs['parent_job_id'] = 0
except (KeyError, TypeError) as e:
raise rdb_utils.RDBException('Creating %s requires args %s got %s' %
(self.__class__, self.template._fields, kwargs.keys()))
super(AcquireHostRequest, self).__init__(**kwargs)
def lease(self):
"""Set the leased bit on the host object, and in the database.
@raises RDBException: If the host is already leased.
"""
self.refresh(fields=['leased'])
if self.leased:
raise rdb_utils.RDBException('Host %s is already leased' %
self.hostname)
self.leased = True
# TODO: Avoid leaking django out of rdb.QueryManagers. This is still
# preferable to calling save() on the host object because we're only
# updating/refreshing a single indexed attribute, the leased bit.
afe_models.Host.objects.filter(id=self.id).update(leased=self.leased)
def get_required_fields_from_host(cls, host):
"""Returns all required attributes of the host parsed into a dict.
Required attributes are defined as the attributes required to
create an RDBHost, and mirror the columns of the host table.
@param host: A host object containing all required fields as attributes.
"""
required_fields_map = {}
try:
for field in cls.required_fields:
required_fields_map[field] = getattr(host, field)
except AttributeError as e:
raise rdb_utils.RDBException('Required %s' % e)
required_fields_map['id'] = host.id
return required_fields_map
def _check_response_map(self):
"""Verify that we never give the same host to different requests.
@raises RDBException: If the same host is assigned to multiple requests.
"""
unique_hosts = set([])
for request, response in self.response_map.iteritems():
# Each value in the response map can only either be a list of
# RDBHosts or a list of RDBExceptions, not a mix of both.
if isinstance(response[0], rdb_hosts.RDBHost):
if any([host in unique_hosts for host in response]):
raise rdb_utils.RDBException(
'Assigning the same host to multiple requests. New '
'hosts %s, request %s, response_map: %s' %
(response, request, self.response_map))
else:
unique_hosts = unique_hosts.union(response)
def response(self):
"""Execute the api call and return a response for each request.
The order of responses is the same as the order of requests added
to the queue.
@yield: A response for each request added to the queue after the
last invocation of response.
"""
if not self.request_queue:
raise rdb_utils.RDBException('No requests. Call add_requests '
'with the appropriate kwargs, before calling response.')
result = self.api_call(self.request_queue)
requests = self.request_queue
self.request_queue = []
for request in requests:
yield result.get(request) if result else None
def _update(self, payload):
"""Send an update to rdb, save the attributes of the payload locally.
@param: A dictionary representing 'key':value of the update required.
@raises RDBException: If the update fails.
"""
logging.info('Host %s in %s updating %s through rdb on behalf of: %s ',
self.hostname, self.status, payload, self.dbg_str)
self.update_request_manager.add_request(host_id=self.id,
payload=payload)
for response in self.update_request_manager.response():
if response:
raise rdb_utils.RDBException(
'Host %s unable to perform update '
'%s through rdb on behalf of %s: %s', self.hostname,
payload, self.dbg_str, response)
super(RDBClientHostWrapper, self)._update_attributes(payload)
def refresh(self, fields=None):
"""Refresh the attributes on this instance.
@param fields: A list of fieldnames to refresh. If None
all the required fields of the host are refreshed.
@raises RDBException: If refreshing a field fails.
"""
# TODO: This is mainly required for cache correctness. If it turns
# into a bottleneck, cache host_ids instead of rdbhosts and rebuild
# the hosts once before leasing them out. The important part is to not
# trust the leased bit on a cached host.
fields = self.required_fields if not fields else fields
try:
refreshed_fields = afe_models.Host.objects.filter(
id=self.id).values(*fields)[0]
except django_exceptions.FieldError as e:
raise rdb_utils.RDBException(
'Couldn\'t refresh fields %s: %s' % fields, e)
self._update_attributes(refreshed_fields)
def __init__(self, **kwargs):
if self.required_fields - set(kwargs.keys()):
raise rdb_utils.RDBException(
'Creating %s requires %s, got %s ' %
(self.__class__, self.required_fields, kwargs.keys()))
self._update_attributes(kwargs)
