def test_execute(self):
"""Test task queue execute."""
drone1 = self.create_remote_drone('fakehostname1')
drone2 = self.create_remote_drone('fakehostname2')
drone3 = self.create_remote_drone('fakehostname3')
# Check task queue exception conditions.
task_queue = thread_lib.ThreadedTaskQueue()
task_queue.results_queue.put(1)
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.execute, [])
task_queue.results_queue.get()
task_queue.drone_threads[drone1] = None
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.execute, [])
task_queue.drone_threads = {}
# Queue 2 calls against each drone, and confirm that the host's
# run method is called 3 times. Then check the threads created,
# and finally compare results returned by the task queue against
# the mock results.
drones = [drone1, drone2, drone3]
for drone in drones:
drone.queue_call('foo')
drone.queue_call('bar')
mock_result = utils.CmdResult(
stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
task_queue.execute(drones, wait=False)
self.assertTrue(set(task_queue.drone_threads.keys()) == set(drones))
for drone, thread in task_queue.drone_threads.iteritems():
self.assertTrue(drone.hostname in thread.getName())
self.assertTrue(thread.isDaemon())
self.assertRaises(RuntimeError, thread.start)
results = task_queue.get_results()
for drone, result in results.iteritems():
self.assertTrue(result == self.mock_return['results'])
# Test synchronous execute
drone1.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone1.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
self.assertTrue(
task_queue.execute(drones, wait=True)[drone1] ==
self.mock_return['results'])
self.god.check_playback()
def test_is_eth1(self):
"""Test when public network interface is eth1."""
run_func = self.god.create_mock_function('run_func')
self.god.stub_with(utils, 'run', run_func)
run_func.expect_call('readlink -f /sys/class/net/eth0').and_return(
utils.CmdResult(exit_status=0, stdout='is usb'))
run_func.expect_call('readlink -f /sys/class/net/eth1').and_return(
utils.CmdResult(exit_status=0, stdout='not_u_s_b'))
eth = utils.get_built_in_ethernet_nic_name()
self.assertEqual('eth1', eth)
self.god.check_playback()
def testInitializeFail(self):
"""Ensure we fail after too many _Clone() retries."""
self.mox.StubOutWithMock(self.mv, '_Clone')
self.mox.StubOutWithMock(time, 'sleep')
for i in range(0, self.mv._CLONE_MAX_RETRIES):
self.mv._Clone().AndRaise(
error.CmdError('retried git clone failure', utils.CmdResult()))
time.sleep(self.mv._CLONE_RETRY_SECONDS)
self.mv._Clone().AndRaise(
error.CmdError('final git clone failure', utils.CmdResult()))
self.mox.ReplayAll()
self.assertRaises(manifest_versions.CloneException, self.mv.Initialize)
def test_execute(self):
"""Test execute method."""
drone1 = self.create_remote_drone('fakehostname1')
drone2 = self.create_remote_drone('fakehostname2')
drone3 = self.create_remote_drone('fakehostname3')
# Check task queue exception conditions.
task_queue = drone_task_queue.DroneTaskQueue()
task_queue.results[drone1] = None
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.execute, [])
task_queue.results.clear()
# Queue 2 calls against each drone, and confirm that the host's
# run method is called 3 times. Then compare results returned by
# the task queue against the mock results.
drones = [drone1, drone2, drone3]
for drone in drones:
drone.queue_call('foo')
drone.queue_call('bar')
mock_result = utils.CmdResult(
stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
task_queue.execute(drones, wait=False)
self.assertTrue(set(task_queue.results.keys()) == set(drones))
results = task_queue.get_results()
self.assertTrue(len(task_queue.results) == 0)
for drone, result in results.iteritems():
self.assertTrue(result == self.mock_return['results'])
# Test execute and get_results
drone1.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone1.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
self.assertTrue(
task_queue.execute(drones, wait=True)[drone1] ==
self.mock_return['results'])
self.assertTrue(len(task_queue.results) == 0)
self.assertTrue(len(task_queue.get_results()) == 0)
self.god.check_playback()
def _setup_test_check_diskspace(self, command, command_result,
command_exit_status, directory,
directory_exists):
self.god.stub_function(os.path, 'isdir')
self.god.stub_function(base_classes.Host, 'run')
host = base_classes.Host()
host.hostname = 'unittest-host'
host.run.expect_call(
'test -e "%s"' % directory, ignore_status=True).and_return(
utils.CmdResult(exit_status=0 if directory_exists else 1))
if directory_exists:
fake_cmd_status = utils.CmdResult(exit_status=command_exit_status,
stdout=command_result)
host.run.expect_call(command).and_return(fake_cmd_status)
return host
def test_trigger_refresh(self):
"""Test drone manager trigger refresh."""
self.god.stub_with(self._DRONE_CLASS, '_drone_utility_path',
self.drone_utility_path)
mock_drone = self.create_drone('fakedrone1', 'fakehost1')
self.manager._drones[mock_drone.hostname] = mock_drone
# Create some fake pidfiles and confirm that a refresh call is
# executed on each drone host, with the same pidfile paths. Then
# check that each drone gets a key in the returned results dictionary.
for i in range(0, 1):
pidfile_info = self.create_fake_pidfile_info(
'tag%s' % i, 'name%s' % i)
pidfile_paths = [pidfile.path for pidfile in pidfile_info.keys()]
refresh_call = drone_utility.call('refresh', pidfile_paths)
expected_results = {}
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
for drone in self.manager.get_drones():
drone._host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps([refresh_call]),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
expected_results[drone] = self.mock_return['results']
self.manager.trigger_refresh()
self.assertTrue(
self.manager._refresh_task_queue.get_results() == expected_results)
self.god.check_playback()
def test_install(self):
host = self.god.create_mock_class(hosts.RemoteHost, "host")
host.bootloader = self.god.create_mock_class(bootloader.Bootloader,
"bootloader")
self.god.stub_function(self.kernel, "get_image_name")
self.god.stub_function(self.kernel, "get_vmlinux_name")
rpm = self.kernel.source_material
rpm_package = "package.rpm"
# record
remote_tmpdir = 'remote_tmp_dir'
host.get_tmp_dir.expect_call().and_return(remote_tmpdir)
remote_rpm = os.path.join(remote_tmpdir, os.path.basename(rpm))
result = common_utils.CmdResult()
result.exit_status = 0
result.stdout = rpm_package
utils.run.expect_call('/usr/bin/rpm -q -p %s' % rpm).and_return(result)
self.kernel.get_image_name.expect_call().and_return("vmlinuz")
host.send_file.expect_call(rpm, remote_rpm)
host.run.expect_call('rpm -e ' + rpm_package, ignore_status=True)
host.run.expect_call('rpm --force -i ' + remote_rpm)
self.kernel.get_vmlinux_name.expect_call().and_return("/boot/vmlinux")
host.run.expect_call('cd /;rpm2cpio %s | cpio -imuv ./boot/vmlinux' %
remote_rpm)
host.run.expect_call('ls /boot/vmlinux')
host.bootloader.remove_kernel.expect_call('autotest')
host.bootloader.add_kernel.expect_call("vmlinuz",
'autotest',
args='',
default=False)
host.bootloader.boot_once.expect_call('autotest')
# run and test
self.kernel.install(host)
self.god.check_playback()
def test_install(self):
self.common_code()
# record
self.host.run.expect_call('dpkg -i "%s"' %
(utils.sh_escape(self.remote_filename)))
result = common_utils.CmdResult()
result.stdout = "1"
utils.run.expect_call(
'dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
utils.run.expect_call(
'dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
self.host.run.expect_call('mkinitramfs -o "/boot/initrd.img-1" "1"')
utils.run.expect_call(
'dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
utils.run.expect_call(
'dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
self.host.bootloader.add_kernel.expect_call(
'/boot/vmlinuz-1', initrd='/boot/initrd.img-1')
# run and check
self.kernel.install(self.host)
self.god.check_playback()
def test_readlin_fails(self):
"""Test when readlink does not work."""
run_func = self.god.create_mock_function('run_func')
self.god.stub_with(utils, 'run', run_func)
run_func.expect_call('readlink -f /sys/class/net/eth0').and_return(
utils.CmdResult(exit_status=-1, stdout='blah'))
eth = utils.get_built_in_ethernet_nic_name()
self.assertEqual('eth0', eth)
self.god.check_playback()
def testInitializeRetry(self):
"""Ensure we retry _Clone() the correct number of times."""
self.mox.StubOutWithMock(self.mv, '_Clone')
self.mox.StubOutWithMock(time, 'sleep')
for i in range(0, self.mv._CLONE_MAX_RETRIES):
self.mv._Clone().AndRaise(
error.CmdError('retried git clone failure', utils.CmdResult()))
time.sleep(self.mv._CLONE_RETRY_SECONDS)
self.mv._Clone()
self.mox.ReplayAll()
self.mv.Initialize()
def test_get_initrd_name(self):
# record
result = common_utils.CmdResult()
result.exit_status = 0
utils.run.expect_call('rpm -q -l -p %s | grep /boot/initrd' %
"source.rpm",
ignore_status=True).and_return(result)
# run and test
self.kernel.get_initrd_name()
self.god.check_playback()
def test_get_vmlinux_name(self):
# record
result = common_utils.CmdResult()
result.exit_status = 0
result.stdout = "vmlinuz"
utils.run.expect_call(
'rpm -q -l -p source.rpm | grep /boot/vmlinux').and_return(result)
# run and test
self.assertEquals(self.kernel.get_vmlinux_name(), result.stdout)
self.god.check_playback()
def test_get_version(self):
# record
result = common_utils.CmdResult()
result.exit_status = 0
result.stdout = "image"
cmd = ('rpm -qpi %s | grep Version | awk \'{print($3);}\'' %
(utils.sh_escape("source.rpm")))
utils.run.expect_call(cmd).and_return(result)
# run and test
self.assertEquals(self.kernel.get_version(), result.stdout)
self.god.check_playback()
def _init_transport(self):
for path, key in self.keys.iteritems():
try:
logging.debug("Connecting with %s", path)
transport = self._connect_transport(key)
transport.set_keepalive(self.KEEPALIVE_TIMEOUT_SECONDS)
self.transport = transport
self.pid = os.getpid()
return
except paramiko.AuthenticationException:
logging.debug("Authentication failure")
else:
raise error.AutoservSshPermissionDeniedError(
"Permission denied using all keys available to ParamikoHost",
utils.CmdResult())
def test_check_diskspace(self):
self.god.stub_function(base_classes.Host, 'run')
host = base_classes.Host()
host.hostname = 'unittest-host'
test_df_tail = ('/dev/sda1 1061 939'
' 123 89% /')
fake_cmd_status = utils.CmdResult(exit_status=0, stdout=test_df_tail)
host.run.expect_call('df -PB 1000000 /foo | tail -1').and_return(
fake_cmd_status)
self.assertRaises(error.AutoservDiskFullHostError,
host.check_diskspace, '/foo', 0.2)
host.run.expect_call('df -PB 1000000 /foo | tail -1').and_return(
fake_cmd_status)
host.check_diskspace('/foo', 0.1)
self.god.check_playback()
def test_execute_calls_impl(self):
self.god.stub_with(drones._RemoteDrone, '_drone_utility_path',
self.drone_utility_path)
drones.drone_utility.create_host.expect_call('fakehost').and_return(
self._mock_host)
self._mock_host.is_up.expect_call().and_return(True)
mock_calls = ('foo', )
mock_result = utils.CmdResult(stdout=cPickle.dumps('mock return'))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(mock_calls),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
drone = drones._RemoteDrone('fakehost', timestamp_remote_calls=False)
self.assertEqual('mock return', drone._execute_calls_impl(mock_calls))
self.god.check_playback()
def testEnableNotificationUsingCredentialsInBucketFail(self):
config_mock = self.mox.CreateMockAnything()
moblab_rpc_interface._CONFIG = config_mock
config_mock.get_config_value(
'CROS', 'image_storage_server').AndReturn('gs://bucket1/')
self.mox.StubOutWithMock(moblab_rpc_interface.GsUtil, 'get_gsutil_cmd')
moblab_rpc_interface.GsUtil.get_gsutil_cmd().AndReturn(
'/path/to/gsutil')
self.mox.StubOutWithMock(common_lib_utils, 'run')
common_lib_utils.run('/path/to/gsutil',
args=('cp', 'gs://bucket1/pubsub-key-do-not-delete.json',
'/tmp')).AndRaise(
error.CmdError("fakecommand", common_lib_utils.CmdResult(), ""))
self.mox.ReplayAll()
moblab_rpc_interface._enable_notification_using_credentials_in_bucket()
def _mock_cmd_runner(cmd, **kwargs):
"""
Mock running a DUT command, returning a CmdResult.
We handle a few mock functions here:
* cros_config $path $property: $path determines error or success.
$property is not used here.
* echo $text: Returns $text with a trailing newline.
Additionally, if the kwarg `ignore_status` is passed in as True,
then when cros_config would raise an error, it instead returns a
CmdResult with an exit_status of 1.
@param cmd: A command, as would be run on the DUT
@param **kwargs: Kwargs that might be passed into, say, utils.run()
@return: A mock response from the DUT
@type cmd: string
@rtype: client.common_lib.utils.CmdResult
@raise error.CmdError if cros_config should raise an exception.
@raise NotImplementedError if cros_config has an unexpected path
"""
result = utils.CmdResult(cmd)
if cmd.startswith('cros_config '):
_, path, _ = cmd.split()
if path == CC_PATHS[SUCCEEDS]:
result.stdout = CROS_CONFIG_SUCCESS_RESPONSE
elif path == CC_PATHS[FAILS]:
result.exit_status = 1
if not kwargs.get('ignore_status'):
raise error.CmdError(cmd, result)
else:
raise NotImplementedError('Bad cros_config path: %s' % path)
elif cmd.startswith('echo '):
result.stdout = cmd.lstrip('echo ') + '\n'
else:
result.exit_status = 2
if not kwargs.get('ignore_status'):
raise error.CmdError(cmd, result)
return result
def test_execute_queued_calls(self):
self.god.stub_with(drones._RemoteDrone, '_drone_utility_path',
self.drone_utility_path)
drones.drone_utility.create_host.expect_call('fakehost').and_return(
self._mock_host)
self._mock_host.is_up.expect_call().and_return(True)
drone = drones._RemoteDrone('fakehost', timestamp_remote_calls=False)
mock_return = {}
mock_return['results'] = ['mock return']
mock_return['warnings'] = []
drone.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
self.assertEqual(mock_return['results'], drone.execute_queued_calls())
self.god.check_playback()
def testRunBucketPerformanceTestFail(self):
self.mox.StubOutWithMock(moblab_rpc_interface.GsUtil, 'get_gsutil_cmd')
moblab_rpc_interface.GsUtil.get_gsutil_cmd().AndReturn(
'/path/to/gsutil')
self.mox.StubOutWithMock(common_lib_utils, 'run')
common_lib_utils.run(
'/path/to/gsutil',
args=('-o', 'Credentials:gs_access_key_id=key', '-o',
'Credentials:gs_secret_access_key=secret', 'perfdiag', '-s',
'1K', '-o', 'testoutput', '-n', '10',
'gs://bucket1')).AndRaise(
error.CmdError("fakecommand",
common_lib_utils.CmdResult(),
"xxxxxx<Error>yyyyyyyyyy</Error>"))
self.mox.ReplayAll()
self.assertRaisesRegexp(
moblab_rpc_interface.BucketPerformanceTestException,
'<Error>yyyyyyyyyy',
moblab_rpc_interface._run_bucket_performance_test, 'key', 'secret',
'gs://bucket1', '1K', '10', 'testoutput')
self.mox.VerifyAll()
def test_worker(self):
"""Test the worker method of a ThreadedTaskQueue."""
# Invoke the worker method with a drone that has a queued call and check
# that the drones host.run method is invoked for the call, and the
# results queue contains the expected results.
drone = self.create_remote_drone('fakehostname')
task_queue = thread_lib.ThreadedTaskQueue()
drone.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
task_queue.worker(drone, task_queue.results_queue)
result = task_queue.results_queue.get()
self.assertTrue(task_queue.results_queue.empty()
and result.drone == drone
and result.results == self.mock_return['results'])
self.god.check_playback()
timed_out = True
break
stdin = self.__send_stdin(channel, stdin)
time.sleep(1)
if timed_out:
exit_status = -signal.SIGTERM
else:
exit_status = channel.recv_exit_status()
channel.settimeout(10)
self._exhaust_stream(stdout, raw_stdout, channel.recv)
self._exhaust_stream(stderr, raw_stderr, channel.recv_stderr)
channel.close()
duration = time.time() - start_time
# create the appropriate results
stdout = "".join(raw_stdout)
stderr = "".join(raw_stderr)
result = utils.CmdResult(command, stdout, stderr, exit_status,
duration)
if exit_status == -signal.SIGHUP:
msg = "ssh connection unexpectedly terminated"
raise error.AutoservRunError(msg, result)
if timed_out:
logging.warn('Paramiko command timed out after %s sec: %s',
timeout, command)
raise error.AutoservRunError("command timed out", result)
if not ignore_status and exit_status:
raise error.AutoservRunError(command, result)
return result
