def test_commit_updates_attributes(self):
"""Tests that existing attributes are updated correctly on commit."""
self.mock_afe.get_hosts.return_value = [
self._create_mock_host([], {'attrib1': 'val1'})
]
info = host_info.HostInfo([], {'attrib1': 'val1_updated'})
self.store._commit_impl(info)
self.assertEqual(self.mock_afe.set_host_attribute.call_count, 1)
self.mock_afe.set_host_attribute.assert_called_once_with(
'attrib1', 'val1_updated', hostname=self.hostname)
def test_commit_refresh_separate_stores(self):
"""Refresh-commit cycle from separate stores restores HostInfo."""
info = host_info.HostInfo(labels=['labels'],
attributes={'attrib': 'value'})
store = file_store.FileStore(self._store_file)
store.commit(info)
read_store = file_store.FileStore(self._store_file)
got = read_store.get()
self.assertEqual(info, got)
def test_commit_labels_raises(self):
"""Test that exception while committing is translated properly."""
self.mock_afe.get_hosts.return_value = [
self._create_mock_host(['label1'], {})
]
self.mock_afe.run.side_effect = rpc_proxy.JSONRPCException(
'some error')
info = host_info.HostInfo(['label2'], {})
with self.assertRaises(host_info.StoreError):
self.store._commit_impl(info)
def test_remove_pool_label(self):
"""Tests that pool labels are not removed on commit."""
self.mock_afe.get_hosts.return_value = [
self._create_mock_host(['pool:XXX'], {})
]
new_info = host_info.HostInfo(['label2'], {})
old_info = self.store._refresh_impl()
labels_to_remove, labels_to_add = self.store._adjust_pool(
old_info, new_info)
self.assertEqual((labels_to_remove, labels_to_add), ([], ['label2']))
def create_host(hostname,
board,
model,
servo_hostname,
servo_port,
servo_serial=None,
logs_dir=None):
"""Yield a server.hosts.CrosHost object to use for DUT preparation.
This object contains just enough inventory data to be able to prepare the
DUT for lab deployment. It does not contain any reference to AFE / Skylab so
that DUT preparation is guaranteed to be isolated from the scheduling
infrastructure.
@param hostname: FQDN of the host to prepare.
@param board: The autotest board label for the DUT.
@param model: The autotest model label for the DUT.
@param servo_hostname: FQDN of the servo host controlling the DUT.
@param servo_port: Servo host port used for the controlling servo.
@param servo_serial: (Optional) Serial number of the controlling servo.
@param logs_dir: (Optional) Directory to save logs obtained from the
host.
@yield a server.hosts.Host object.
"""
labels = [
'board:%s' % board,
'model:%s' % model,
]
attributes = {
servo_host.SERVO_HOST_ATTR: servo_hostname,
servo_host.SERVO_PORT_ATTR: servo_port,
}
if servo_serial is not None:
attributes[servo_host.SERVO_SERIAL_ATTR] = servo_serial
store = host_info.InMemoryHostInfoStore(info=host_info.HostInfo(
labels=labels,
attributes=attributes,
))
machine_dict = {
'hostname': hostname,
'host_info_store': store,
'afe_host': server_utils.EmptyAFEHost(),
}
host = hosts.create_host(machine_dict)
servohost = servo_host.ServoHost(
**servo_host.get_servo_args_for_host(host))
_prepare_servo(servohost)
host.set_servo_host(servohost)
host.servo.uart_logs_dir = logs_dir
try:
yield host
finally:
host.close()
def test_serialize_empty(self):
"""Serializing empty HostInfo results in the expected json."""
info = host_info.HostInfo()
file_obj = cStringIO.StringIO()
host_info.json_serialize(info, file_obj)
file_obj.seek(0)
expected_dict = {
'serializer_version': self.CURRENT_SERIALIZATION_VERSION,
'attributes': {},
'labels': [],
}
self.assertEqual(json.load(file_obj), expected_dict)
def test_refresh_from_unreadable_path_raises(self):
"""Refresh from an unreadable backing file raises StoreError."""
# file_lock_timeout of 0 forces no retries (speeds up the test)
store = file_store.FileStore(self._store_file,
file_lock_timeout_seconds=0)
store.commit(host_info.HostInfo())
old_mode = os.stat(self._store_file).st_mode
os.chmod(self._store_file, old_mode & ~stat.S_IRUSR)
self.addCleanup(os.chmod, self._store_file, old_mode)
with self.assertRaises(host_info.StoreError):
store.get(force_refresh=True)
def test_commit_blocks_for_locked_file(self):
"""Commit blocks when the backing file is locked.
This is a greybox test. We artificially lock the backing file.
This test intentionally uses a real locking.FileLock to ensure that
locking API is used correctly.
"""
# file_lock_timeout of 0 forces no retries (speeds up the test)
store = file_store.FileStore(self._store_file,
file_lock_timeout_seconds=0)
file_lock = locking.FileLock(store._lock_path, locktype=locking.FLOCK)
with file_lock.lock(), self.assertRaises(host_info.StoreError):
store.commit(host_info.HostInfo())
def test_commit_labels(self):
"""Tests that labels are updated correctly on commit."""
self.mock_afe.get_hosts.return_value = [
self._create_mock_host(['label1'], {})
]
info = host_info.HostInfo(['label2'], {})
self.store._commit_impl(info)
self.assertEqual(self.mock_afe.run.call_count, 2)
expected_run_calls = [
mock.call('host_remove_labels', id='some-host', labels={'label1'}),
mock.call('host_add_labels', id='some-host', labels={'label2'}),
]
self.mock_afe.run.assert_has_calls(expected_run_calls, any_order=True)
def test_failed_commit_cleans_cache(self):
"""Check that a failed commit cleanes cache."""
# Let's initialize the store without errors.
self.store.refresh_raises = False
self.store.get(force_refresh=True)
self.store.refresh_raises = True
with self.assertRaises(host_info.StoreError):
self.store.commit(host_info.HostInfo())
# Since |commit| hit an error, a subsequent get should again hit the
# store.
with self.assertRaises(host_info.StoreError):
self.store.get()
def test_deserialize_malformed_host_info_raises(self):
"""Deserializing a malformed host_info raises."""
info = host_info.HostInfo()
serialized_fp = cStringIO.StringIO()
host_info.json_serialize(info, serialized_fp)
serialized_fp.seek(0)
serialized_dict = json.load(serialized_fp)
del serialized_dict['labels']
serialized_no_version_str = json.dumps(serialized_dict)
with self.assertRaises(host_info.DeserializationError):
host_info.json_deserialize(
cStringIO.StringIO(serialized_no_version_str))
def test_refresh_fixes_mismatch_in_stores(self):
"""On finding a mismatch, the difference is fixed by the store"""
callback = mock.MagicMock()
p_info = host_info.HostInfo('primary')
primary = _FakeRaisingStore(p_info)
shadow = _FakeRaisingStore()
store = shadowing_store.ShadowingStore(primary,
shadow,
mismatch_callback=callback)
# ShadowingStore will update shadow on initialization, so we modify it
# after creating store.
s_info = host_info.HostInfo('shadow')
shadow.commit(s_info)
got = store.get(force_refresh=True)
self.assertEqual(got, p_info)
callback.assert_called_once_with(p_info, s_info)
self.assertEqual(got, shadow.get())
got = store.get(force_refresh=True)
self.assertEqual(got, p_info)
# No extra calls, just the one we already saw above.
callback.assert_called_once_with(p_info, s_info)
def _gen_machine_dict(hostname='localhost', labels=[], attributes={}):
"""Generate a machine dictionary with the specified parameters.
@param hostname: hostname of machine
@param labels: list of host labels
@param attributes: dict of host attributes
@return: machine dict with mocked AFE Host object and fake AfeStore.
"""
afe_host = base_label_unittest.MockAFEHost(labels, attributes)
store = host_info.InMemoryHostInfoStore()
store.commit(host_info.HostInfo(labels, attributes))
return {'hostname': hostname,
'afe_host': afe_host,
'host_info_store': store}
def test_serialize_non_empty(self):
"""Serializing a populated HostInfo results in expected json."""
info = host_info.HostInfo(labels=['label1'],
attributes={'attrib': 'val'})
file_obj = cStringIO.StringIO()
host_info.json_serialize(info, file_obj)
file_obj.seek(0)
expected_dict = {
'serializer_version': self.CURRENT_SERIALIZATION_VERSION,
'attributes': {
'attrib': 'val'
},
'labels': ['label1'],
}
self.assertEqual(json.load(file_obj), expected_dict)
def test_serialize_pretty_print(self):
"""Serializing a host_info dumps the json in human-friendly format"""
info = host_info.HostInfo(labels=['label1'],
attributes={'attrib': 'val'})
serialized_fp = cStringIO.StringIO()
host_info.json_serialize(info, serialized_fp)
expected = """{
"attributes": {
"attrib": "val"
},
"labels": [
"label1"
],
"serializer_version": %d
}""" % self.CURRENT_SERIALIZATION_VERSION
self.assertEqual(serialized_fp.getvalue(), inspect.cleandoc(expected))
def test_commit_with_negative_timeout_clips(self, mock_file_lock_class):
"""Commit request with negative timeout is same as 0 timeout.
@param mock_file_lock_class: A patched version of the locking.FileLock
class.
"""
mock_file_lock = mock_file_lock_class.return_value
mock_file_lock.__enter__.return_value = mock_file_lock
mock_file_lock.write_lock.side_effect = (
locking.LockNotAcquiredError('Testing error'))
store = file_store.FileStore(self._store_file,
file_lock_timeout_seconds=-1)
with self.assertRaises(host_info.StoreError):
store.commit(host_info.HostInfo())
self.assertEqual(1, mock_file_lock.write_lock.call_count)
def _refresh_impl(self):
"""Obtains HostInfo directly from the AFE."""
try:
hosts = self._afe.get_hosts(hostname=self._hostname)
except rpc_proxy.JSONRPCException as e:
raise host_info.StoreError(e)
if not hosts:
raise host_info.StoreError('No hosts founds with hostname: %s' %
self._hostname)
if len(hosts) > 1:
logging.warning(
'Found %d hosts with the name %s. Picking the first one.',
len(hosts), self._hostname)
host = hosts[0]
return host_info.HostInfo(host.labels, host.attributes)
def test_commit_succeeds_after_lock_retry(self, mock_file_lock_class):
"""Tests that commit succeeds when locking requires retries.
@param mock_file_lock_class: A patched version of the locking.FileLock
class.
"""
mock_file_lock = mock_file_lock_class.return_value
mock_file_lock.__enter__.return_value = mock_file_lock
mock_file_lock.write_lock.side_effect = [
locking.LockNotAcquiredError('Testing error'),
True,
]
store = file_store.FileStore(self._store_file,
file_lock_timeout_seconds=0.1)
store.commit(host_info.HostInfo())
self.assertEqual(2, mock_file_lock.write_lock.call_count)
def test_get_returns_deepcopy(self):
"""The cached object is protected from |get| caller modifications."""
self.store.info = host_info.HostInfo(['label1'],
{'attrib1': {
'key1': 'data1'
}})
got = self.store.get()
self._verify_host_info_data(got, ['label1'],
{'attrib1': {
'key1': 'data1'
}})
got.labels.append('label2')
got.attributes['attrib1']['key1'] = 'data2'
got = self.store.get()
self._verify_host_info_data(got, ['label1'],
{'attrib1': {
'key1': 'data1'
}})
def _detect_host_info(self, host):
"""Detect platform and labels from the host.
@param host: hostname
@return: HostInfo object
"""
# Mock an afe_host object so that the host is constructed as if the
# data was already in afe
data = {'attributes': self.attributes, 'labels': self.labels}
afe_host = frontend.Host(None, data)
store = host_info.InMemoryHostInfoStore(
host_info.HostInfo(labels=self.labels, attributes=self.attributes))
machine = {
'hostname': host,
'afe_host': afe_host,
'host_info_store': store
}
try:
if bin_utils.ping(host, tries=1, deadline=1) == 0:
serials = self.attributes.get('serials', '').split(',')
if serials and len(serials) > 1:
host_dut = hosts.create_testbed(machine,
adb_serials=serials)
else:
adb_serial = self.attributes.get('serials')
host_dut = hosts.create_host(machine,
adb_serial=adb_serial)
info = HostInfo(host, host_dut.get_platform(),
host_dut.get_labels())
# Clean host to make sure nothing left after calling it,
# e.g. tunnels.
if hasattr(host_dut, 'close'):
host_dut.close()
else:
# Can't ping the host, use default information.
info = HostInfo(host, None, [])
except (socket.gaierror, error.AutoservRunError,
error.AutoservSSHTimeout):
# We may be adding a host that does not exist yet or we can't
# reach due to hostname/address issues or if the host is down.
info = HostInfo(host, None, [])
return info
def test_update_labels(self):
"""Check that we add/remove the expected labels in update_labels()."""
label_to_add = 'label_to_add'
label_to_remove = 'prefix:label_to_remove'
store = host_info.InMemoryHostInfoStore(info=host_info.HostInfo(
labels=[label_to_remove, TestBaseLabel._NAME], ), )
mockhost = MockHost(store=store)
retriever = base_label.LabelRetriever(
[TestStringPrefixLabel(label=label_to_add),
TestBaseLabel()])
retriever.update_labels(mockhost)
self.assertEqual(
set(store.get().labels),
{
'%s:%s' % (TestStringPrefixLabel._NAME, label_to_add),
TestBaseLabel._NAME
},
)
def update_labels(self, host, keep_pool=False):
"""
Retrieve the labels from the host and update if needed.
@param host: The host to update the labels for.
"""
# If we haven't yet grabbed our list of known labels, do so now.
if not self.label_full_names and not self.label_prefix_names:
self._populate_known_labels(self._labels)
# Label detection hits the DUT so it can be slow. Do it before reading
# old labels from HostInfoStore to minimize the time between read and
# commit of the HostInfo.
new_labels = self.get_labels(host)
old_info = host.host_info_store.get()
self._carry_over_unknown_labels(old_info.labels, new_labels)
new_info = host_info.HostInfo(
labels=new_labels,
attributes=old_info.attributes,
)
if old_info != new_info:
self._commit_info(host, new_info, keep_pool)
def __init__(self, labels, *args):
self._afe_host = MockAFEHost(labels)
self.mock_cmds = {c.cmd: c for c in args}
info = host_info.HostInfo(labels=labels)
self.host_info_store = host_info.InMemoryHostInfoStore(info)
def _refresh_impl(self):
if self.refresh_raises:
raise host_info.StoreError('no can do')
return host_info.HostInfo()
def test_str(self):
"""Sanity checks the __str__ implementation."""
info = host_info.HostInfo(labels=['a'], attributes={'b': 2})
self.assertEqual(str(info),
"HostInfo[Labels: ['a'], Attributes: {'b': 2}]")
def setUp(self):
self.info = host_info.HostInfo()
def test_first_get_refreshes_cache(self):
"""Test that the first call to get gets the data from store."""
self.store.info = host_info.HostInfo(['label1'], {'attrib1': 'val1'})
got = self.store.get()
self._verify_host_info_data(got, ['label1'], {'attrib1': 'val1'})
def test_empty_infos_are_equal(self):
"""Tests that empty HostInfo objects are considered equal."""
self.assertEqual(host_info.HostInfo(), host_info.HostInfo())
# equality and non-equality are unrelated by the data model.
self.assertFalse(host_info.HostInfo() != host_info.HostInfo())