def testDownloadSymbolsFromGS(self):
"""Basic symbols download from Google Storage."""
self._DownloadSymbolsHelper(
downloader.GoogleStorageDownloader(
self._work_dir, self.archive_url,
downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL(
self.archive_url)))
def testArtifactFactory(self):
"""Tests that BuildArtifact works for both named and file artifacts."""
name_artifact = 'test_suites' # This file is in every real GS dir.
file_artifact = 'metadata.json' # This file is in every real GS dir.
factory = build_artifact.ChromeOSArtifactFactory(
self.work_dir, [name_artifact], [file_artifact], _VERSION)
artifacts = factory.RequiredArtifacts()
self.assertEqual(len(artifacts), 2)
expected_installed_files_0 = [
os.path.join(self.work_dir, filename)
for filename in ([build_artifact.TEST_SUITES_FILE] +
_TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT)
]
expected_installed_files_1 = [
os.path.join(self.work_dir, file_artifact)
]
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifacts[0].Process(dl, False)
artifacts[1].Process(dl, False)
self.assertEqual(artifacts[0].installed_files,
expected_installed_files_0)
self.assertEqual(artifacts[1].installed_files,
expected_installed_files_1)
# Test suites directory exists.
self.assertExists(
os.path.join(self.work_dir, 'autotest', 'test_suites'))
# File artifact was staged.
self.assertExists(os.path.join(self.work_dir, file_artifact))
self._CheckMarker(artifacts[0].marker_name,
artifacts[0].installed_files)
self._CheckMarker(artifacts[1].marker_name,
artifacts[1].installed_files)
def testSimpleDownloadOfTestSuitesFromGS(self):
"""Basic test_suites test.
Verifies that if we request the test_suites from Google Storage, it gets
downloaded and the autotest tarball is attempted in the background.
"""
self._SimpleDownloadOfTestSuites(
downloader.GoogleStorageDownloader(
self._work_dir, self.archive_url,
downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL(
self.archive_url)))
def testProcessBuildArtifact(self):
"""Processes a real tarball from GSUtil and stages it."""
artifact = build_artifact.Artifact(build_artifact.TEST_SUITES_FILE,
self.work_dir, _VERSION)
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
self.assertEqual(
artifact.installed_files,
[os.path.join(self.work_dir, build_artifact.TEST_SUITES_FILE)])
self.assertExists(
os.path.join(self.work_dir, build_artifact.TEST_SUITES_FILE))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def _DownloadFullPayload(self):
"""Full payload download."""
downloader_instance = downloader.GoogleStorageDownloader(
self._work_dir, self.archive_url,
downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL(
self.archive_url))
factory = build_artifact.ChromeOSArtifactFactory(
downloader_instance.GetBuildDir(), ['full_payload'],
None, downloader_instance.GetBuild())
full_dev = os.path.join(self.dest_dir, self.payload)
self.downloaded = [full_dev, full_dev + '.json']
return downloader_instance, factory
def testProcessBuildArtifactWithException(self):
"""Test processing a non-existing artifact from GSUtil."""
artifact = build_artifact.Artifact(build_artifact.TEST_SUITES_FILE,
self.work_dir, _VERSION)
try:
dl = downloader.GoogleStorageDownloader(
self.work_dir, _TEST_NON_EXISTING_GOLO_ARCHIVE,
_TEST_NON_EXISTING_GOLO_BUILD_ID)
artifact.Process(dl, False)
except Exception as e:
expected_exception = e
saved_exception = artifact.GetException()
# saved_exception has traceback info - strip it.
self.assertEqual(str(expected_exception),
str(saved_exception).split('\n')[0])
def testAUTestPayloadBuildArtifact(self):
"""Downloads a real tarball and treats it like an AU payload."""
artifact = build_artifact.AUTestPayload(
build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION)
expected_installed_files = [
os.path.join(self.work_dir, devserver_constants.UPDATE_FILE)
]
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
self.assertEqual(artifact.installed_files, expected_installed_files)
self.assertExists(
os.path.join(self.work_dir, devserver_constants.UPDATE_FILE))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testDeltaPayloadsArtifact(self):
"""Downloads delta paylaods from test bucket."""
nton = build_artifact.DeltaPayloadNtoN(self.work_dir, _VERSION)
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
nton.Process(dl, False)
nton_dir = os.path.join(self.work_dir, 'au', '%s_nton' % _VERSION)
# TODO(ahassani, achuith): update.gz appears twice.
delta_installed_files = ('update.gz', 'update.gz.json', 'update.gz',
'stateful.tgz')
self.assertEqual(nton.installed_files, [
os.path.join(nton_dir, filename)
for filename in delta_installed_files
])
self.assertExists(os.path.join(nton_dir, 'update.gz'))
self._CheckMarker(nton.marker_name, nton.installed_files)
def testProcessTarball(self):
"""Downloads a real tarball and untars it."""
artifact = build_artifact.BundledArtifact(
build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION)
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
expected_installed_files = [
os.path.join(self.work_dir, filename)
for filename in ([build_artifact.TEST_SUITES_FILE] +
_TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT)
]
self.assertEqual(artifact.installed_files, expected_installed_files)
self.assertTrue(
os.path.isdir(
os.path.join(self.work_dir, 'autotest', 'test_suites')))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def _Download(self, gs_url, artifacts, build_id):
"""Download the artifacts from the given gs_url.
Raises:
build_artifact.ArtifactDownloadError: If we failed to download the
artifact.
"""
with XBuddy._staging_thread_count_lock:
XBuddy._staging_thread_count += 1
try:
_Log('Downloading %s from %s', artifacts, gs_url)
dl = downloader.GoogleStorageDownloader(self.static_dir, gs_url, build_id)
factory = build_artifact.ChromeOSArtifactFactory(
dl.GetBuildDir(), artifacts, [], dl.GetBuild())
dl.Download(factory)
finally:
with XBuddy._staging_thread_count_lock:
XBuddy._staging_thread_count -= 1
def testImageUnzipWithExcludes(self):
"""Downloads and stages a zip file while excluding all large files."""
artifact = build_artifact.BundledArtifact(build_artifact.IMAGE_FILE,
self.work_dir,
_VERSION,
exclude=['*.bin'])
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
expected_installed_files = [
os.path.join(self.work_dir, filename)
for filename in _TEST_GOLO_ARCHIVE_IMAGE_ZIPFILE_CONTENT
]
self.assertEqual(expected_installed_files, artifact.installed_files)
self.assertNotExists(
os.path.join(self.work_dir, 'chromiumos_test_image.bin'))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testImageUnzip(self):
"""Downloads and stages a zip file and extracts a test image."""
files_to_extract = ['chromiumos_test_image.bin']
artifact = build_artifact.BundledArtifact(
build_artifact.IMAGE_FILE,
self.work_dir,
_VERSION,
files_to_extract=files_to_extract)
expected_installed_files = [
os.path.join(self.work_dir, filename)
for filename in [build_artifact.IMAGE_FILE] + files_to_extract
]
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
self.assertEqual(expected_installed_files, artifact.installed_files)
self.assertExists(
os.path.join(self.work_dir, 'chromiumos_test_image.bin'))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testProcessTarballWithFile(self):
"""Downloads a real tarball and only untars one file from it."""
file_to_download = 'autotest/test_suites/control.provision'
artifact = build_artifact.BundledArtifact(
build_artifact.TEST_SUITES_FILE,
self.work_dir,
_VERSION,
files_to_extract=[file_to_download])
expected_installed_files = [
os.path.join(self.work_dir, filename)
for filename in [build_artifact.TEST_SUITES_FILE] +
[file_to_download]
]
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
self.assertEqual(artifact.installed_files, expected_installed_files)
self.assertExists(os.path.join(self.work_dir, file_to_download))
self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testArtifactStaged(self):
"""Tests the artifact staging verification logic."""
artifact = build_artifact.BundledArtifact(
build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION)
expected_installed_files = [
os.path.join(self.work_dir, filename)
for filename in ([build_artifact.TEST_SUITES_FILE] +
_TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT)
]
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
artifact.Process(dl, False)
# Check that it works when all files are there.
self.assertTrue(artifact.ArtifactStaged())
# Remove an arbitrary file among the ones staged, ensure the check fails
# and that the marker files is removed.
os.remove(random.choice(expected_installed_files))
self.assertExists(os.path.join(self.work_dir, artifact.marker_name))
self.assertFalse(artifact.ArtifactStaged())
self.assertNotExists(os.path.join(self.work_dir, artifact.marker_name))
def testDownloadAutotest(self, wait_mock, fetch_mock, update_name_mock,
extract_mock):
"""Downloads a real autotest tarball for test."""
artifact = build_artifact.AutotestTarball(
build_artifact.AUTOTEST_FILE,
self.work_dir,
_VERSION,
files_to_extract=None,
exclude=['autotest/test_suites'])
install_dir = self.work_dir
artifact.staging_dir = install_dir
dl = downloader.GoogleStorageDownloader(self.work_dir,
_TEST_GOLO_ARCHIVE,
_TEST_GOLO_BUILD_ID)
rc = self.StartPatcher(cros_test_lib.RunCommandMock())
rc.SetDefaultCmdResult()
artifact.Process(dl, True)
self.assertFalse(artifact.installed_files)
self.assertTrue(
os.path.isdir(os.path.join(self.work_dir, 'autotest', 'packages')))
self.assertFalse(
os.path.getsize(os.path.join(self.work_dir, artifact.marker_name)))
wait_mock.assert_called_with('autotest.tar', False, 1)
fetch_mock.assert_called_with('autotest.tar', install_dir + '/')
update_name_mock.assert_called()
extract_mock.assert_called()
rc.assertCommandContains([
'autotest/utils/packager.py', '--action=upload', '--repository',
os.path.join(install_dir, 'autotest/packages'), '--all'
],
cwd=install_dir)