def testDownloadSymbolsFromGS(self): """Basic symbols download from Google Storage.""" self._DownloadSymbolsHelper( downloader.GoogleStorageDownloader( self._work_dir, self.archive_url, downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL( self.archive_url)))
def testArtifactFactory(self): """Tests that BuildArtifact works for both named and file artifacts.""" name_artifact = 'test_suites' # This file is in every real GS dir. file_artifact = 'metadata.json' # This file is in every real GS dir. factory = build_artifact.ChromeOSArtifactFactory( self.work_dir, [name_artifact], [file_artifact], _VERSION) artifacts = factory.RequiredArtifacts() self.assertEqual(len(artifacts), 2) expected_installed_files_0 = [ os.path.join(self.work_dir, filename) for filename in ([build_artifact.TEST_SUITES_FILE] + _TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT) ] expected_installed_files_1 = [ os.path.join(self.work_dir, file_artifact) ] dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifacts[0].Process(dl, False) artifacts[1].Process(dl, False) self.assertEqual(artifacts[0].installed_files, expected_installed_files_0) self.assertEqual(artifacts[1].installed_files, expected_installed_files_1) # Test suites directory exists. self.assertExists( os.path.join(self.work_dir, 'autotest', 'test_suites')) # File artifact was staged. self.assertExists(os.path.join(self.work_dir, file_artifact)) self._CheckMarker(artifacts[0].marker_name, artifacts[0].installed_files) self._CheckMarker(artifacts[1].marker_name, artifacts[1].installed_files)
def testSimpleDownloadOfTestSuitesFromGS(self): """Basic test_suites test. Verifies that if we request the test_suites from Google Storage, it gets downloaded and the autotest tarball is attempted in the background. """ self._SimpleDownloadOfTestSuites( downloader.GoogleStorageDownloader( self._work_dir, self.archive_url, downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL( self.archive_url)))
def testProcessBuildArtifact(self): """Processes a real tarball from GSUtil and stages it.""" artifact = build_artifact.Artifact(build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION) dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) self.assertEqual( artifact.installed_files, [os.path.join(self.work_dir, build_artifact.TEST_SUITES_FILE)]) self.assertExists( os.path.join(self.work_dir, build_artifact.TEST_SUITES_FILE)) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def _DownloadFullPayload(self): """Full payload download.""" downloader_instance = downloader.GoogleStorageDownloader( self._work_dir, self.archive_url, downloader.GoogleStorageDownloader.GetBuildIdFromArchiveURL( self.archive_url)) factory = build_artifact.ChromeOSArtifactFactory( downloader_instance.GetBuildDir(), ['full_payload'], None, downloader_instance.GetBuild()) full_dev = os.path.join(self.dest_dir, self.payload) self.downloaded = [full_dev, full_dev + '.json'] return downloader_instance, factory
def testProcessBuildArtifactWithException(self): """Test processing a non-existing artifact from GSUtil.""" artifact = build_artifact.Artifact(build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION) try: dl = downloader.GoogleStorageDownloader( self.work_dir, _TEST_NON_EXISTING_GOLO_ARCHIVE, _TEST_NON_EXISTING_GOLO_BUILD_ID) artifact.Process(dl, False) except Exception as e: expected_exception = e saved_exception = artifact.GetException() # saved_exception has traceback info - strip it. self.assertEqual(str(expected_exception), str(saved_exception).split('\n')[0])
def testAUTestPayloadBuildArtifact(self): """Downloads a real tarball and treats it like an AU payload.""" artifact = build_artifact.AUTestPayload( build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION) expected_installed_files = [ os.path.join(self.work_dir, devserver_constants.UPDATE_FILE) ] dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) self.assertEqual(artifact.installed_files, expected_installed_files) self.assertExists( os.path.join(self.work_dir, devserver_constants.UPDATE_FILE)) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testDeltaPayloadsArtifact(self): """Downloads delta paylaods from test bucket.""" nton = build_artifact.DeltaPayloadNtoN(self.work_dir, _VERSION) dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) nton.Process(dl, False) nton_dir = os.path.join(self.work_dir, 'au', '%s_nton' % _VERSION) # TODO(ahassani, achuith): update.gz appears twice. delta_installed_files = ('update.gz', 'update.gz.json', 'update.gz', 'stateful.tgz') self.assertEqual(nton.installed_files, [ os.path.join(nton_dir, filename) for filename in delta_installed_files ]) self.assertExists(os.path.join(nton_dir, 'update.gz')) self._CheckMarker(nton.marker_name, nton.installed_files)
def testProcessTarball(self): """Downloads a real tarball and untars it.""" artifact = build_artifact.BundledArtifact( build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION) dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) expected_installed_files = [ os.path.join(self.work_dir, filename) for filename in ([build_artifact.TEST_SUITES_FILE] + _TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT) ] self.assertEqual(artifact.installed_files, expected_installed_files) self.assertTrue( os.path.isdir( os.path.join(self.work_dir, 'autotest', 'test_suites'))) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def _Download(self, gs_url, artifacts, build_id): """Download the artifacts from the given gs_url. Raises: build_artifact.ArtifactDownloadError: If we failed to download the artifact. """ with XBuddy._staging_thread_count_lock: XBuddy._staging_thread_count += 1 try: _Log('Downloading %s from %s', artifacts, gs_url) dl = downloader.GoogleStorageDownloader(self.static_dir, gs_url, build_id) factory = build_artifact.ChromeOSArtifactFactory( dl.GetBuildDir(), artifacts, [], dl.GetBuild()) dl.Download(factory) finally: with XBuddy._staging_thread_count_lock: XBuddy._staging_thread_count -= 1
def testImageUnzipWithExcludes(self): """Downloads and stages a zip file while excluding all large files.""" artifact = build_artifact.BundledArtifact(build_artifact.IMAGE_FILE, self.work_dir, _VERSION, exclude=['*.bin']) dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) expected_installed_files = [ os.path.join(self.work_dir, filename) for filename in _TEST_GOLO_ARCHIVE_IMAGE_ZIPFILE_CONTENT ] self.assertEqual(expected_installed_files, artifact.installed_files) self.assertNotExists( os.path.join(self.work_dir, 'chromiumos_test_image.bin')) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testImageUnzip(self): """Downloads and stages a zip file and extracts a test image.""" files_to_extract = ['chromiumos_test_image.bin'] artifact = build_artifact.BundledArtifact( build_artifact.IMAGE_FILE, self.work_dir, _VERSION, files_to_extract=files_to_extract) expected_installed_files = [ os.path.join(self.work_dir, filename) for filename in [build_artifact.IMAGE_FILE] + files_to_extract ] dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) self.assertEqual(expected_installed_files, artifact.installed_files) self.assertExists( os.path.join(self.work_dir, 'chromiumos_test_image.bin')) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testProcessTarballWithFile(self): """Downloads a real tarball and only untars one file from it.""" file_to_download = 'autotest/test_suites/control.provision' artifact = build_artifact.BundledArtifact( build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION, files_to_extract=[file_to_download]) expected_installed_files = [ os.path.join(self.work_dir, filename) for filename in [build_artifact.TEST_SUITES_FILE] + [file_to_download] ] dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) self.assertEqual(artifact.installed_files, expected_installed_files) self.assertExists(os.path.join(self.work_dir, file_to_download)) self._CheckMarker(artifact.marker_name, artifact.installed_files)
def testArtifactStaged(self): """Tests the artifact staging verification logic.""" artifact = build_artifact.BundledArtifact( build_artifact.TEST_SUITES_FILE, self.work_dir, _VERSION) expected_installed_files = [ os.path.join(self.work_dir, filename) for filename in ([build_artifact.TEST_SUITES_FILE] + _TEST_GOLO_ARCHIVE_TEST_TARBALL_CONTENT) ] dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) artifact.Process(dl, False) # Check that it works when all files are there. self.assertTrue(artifact.ArtifactStaged()) # Remove an arbitrary file among the ones staged, ensure the check fails # and that the marker files is removed. os.remove(random.choice(expected_installed_files)) self.assertExists(os.path.join(self.work_dir, artifact.marker_name)) self.assertFalse(artifact.ArtifactStaged()) self.assertNotExists(os.path.join(self.work_dir, artifact.marker_name))
def testDownloadAutotest(self, wait_mock, fetch_mock, update_name_mock, extract_mock): """Downloads a real autotest tarball for test.""" artifact = build_artifact.AutotestTarball( build_artifact.AUTOTEST_FILE, self.work_dir, _VERSION, files_to_extract=None, exclude=['autotest/test_suites']) install_dir = self.work_dir artifact.staging_dir = install_dir dl = downloader.GoogleStorageDownloader(self.work_dir, _TEST_GOLO_ARCHIVE, _TEST_GOLO_BUILD_ID) rc = self.StartPatcher(cros_test_lib.RunCommandMock()) rc.SetDefaultCmdResult() artifact.Process(dl, True) self.assertFalse(artifact.installed_files) self.assertTrue( os.path.isdir(os.path.join(self.work_dir, 'autotest', 'packages'))) self.assertFalse( os.path.getsize(os.path.join(self.work_dir, artifact.marker_name))) wait_mock.assert_called_with('autotest.tar', False, 1) fetch_mock.assert_called_with('autotest.tar', install_dir + '/') update_name_mock.assert_called() extract_mock.assert_called() rc.assertCommandContains([ 'autotest/utils/packager.py', '--action=upload', '--repository', os.path.join(install_dir, 'autotest/packages'), '--all' ], cwd=install_dir)