def testParseArchiveLists(self): STATIC_ARCHIVE = 'static_archive.zip' ARCHIVE_1 = 'archive_1.zip' ARCHIVE_2 = 'archive_2.zip' ARCHIVENAMES = [STATIC_ARCHIVE, ARCHIVE_1, ARCHIVE_2] files_cfg = CreateTestFilesCfg(self.temp_dir) arch = '64bit' buildtype = 'official' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) archives = fparser.ParseArchiveLists() self.assertEqual(archives.keys(), [STATIC_ARCHIVE]) self.assertItemsEqual([x['filename'] for x in archives[STATIC_ARCHIVE]], ['archive_allany.txt', 'subdirectory/archive_allany.txt']) # 32bit dev has additional files under the same archive name. arch = '32bit' buildtype = 'dev' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) archives = fparser.ParseArchiveLists() self.assertEqual(archives.keys(), ARCHIVENAMES) self.assertItemsEqual([x['filename'] for x in archives[STATIC_ARCHIVE]], ['archive_allany.txt', 'subdirectory/archive_allany.txt', 'subdirectory/archive_dev32.txt']) self.assertItemsEqual([x['filename'] for x in archives[ARCHIVE_1]], ['multiple_archive.txt', 'archive_1.txt']) self.assertItemsEqual([x['filename'] for x in archives[ARCHIVE_2]], ['multiple_archive.txt', 'archive_2.txt'])
def testWinParseSymbols(self): files_cfg = options.src_base + RealFilesCfgTest.WIN_PATH # There should be some official build symbols. fparser = archive_utils.FilesCfgParser(files_cfg, 'official', '32bit') official_list = fparser.ParseGroup('symsrc') self.assertTrue(official_list) # Windows symbols should be the same regardless of arch. fparser = archive_utils.FilesCfgParser(files_cfg, 'official', '64bit') official64_list = fparser.ParseGroup('symsrc') self.assertEqual(official64_list, official_list)
def testOptionalFiles(self): files_cfg = CreateTestFilesCfg(self.temp_dir) optional_fn = 'allany_dev_optional.txt' arch = '64bit' buildtype = 'dev' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) self.assertTrue(fparser.IsOptional(optional_fn)) # It's only optional for 'dev' builds. buildtype = 'official' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) self.assertFalse(fparser.IsOptional(optional_fn))
def ParseFilesCfg(self, cfg_path): if cfg_path.startswith('svn://'): # Store the svn file so it will be automatically cleaned up in tearDown(). self.files_cfg = FetchSvn(cfg_path, self.svn) cfg_path = self.files_cfg # There should always be some 32bit, official and dev files (otherwise # there's nothing to archive). arch = '32bit' buildtype = 'official' fparser = archive_utils.FilesCfgParser(cfg_path, buildtype, arch) files_list = fparser.ParseLegacyList() self.assertTrue(files_list) fparser.buildtype = 'dev' files_list = fparser.ParseLegacyList() self.assertTrue(files_list) # Arbitrary buildtype shouldn't return anything. fparser.buildtype = 'bogus' files_list = fparser.ParseLegacyList() self.assertFalse(files_list) # Check for incomplete/incorrect settings. # buildtype must exist and be in ['dev', 'official'] self.assertFalse([f for f in fparser._files_cfg # pylint: disable=W0212 if not f['buildtype'] or set(f['buildtype']) - set(['dev', 'official'])])
def testCreateZipExtArchive(self): files_cfg = CreateTestFilesCfg(self.tool_dir) CreateFileSetInDir(self.build_dir, [i['filename'] for i in TEST_FILES_CFG]) archive_name = 'test_with_ext.zip' arch = '64bit' buildtype = 'official' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) files_list = fparser.ParseLegacyList() zip_dir, zip_file_path = archive_utils.CreateArchive( self.build_dir, self.temp_dir, files_list, archive_name) self.assertTrue(zip_dir) self.assertTrue(zip_file_path) self.assertTrue(os.path.exists(zip_file_path)) self.assertEqual(os.path.basename(zip_file_path), archive_name) self.verifyZipFile(zip_dir, zip_file_path, os.path.basename(zip_dir), files_list) # Creating the archive twice is wasteful, but shouldn't fail (e.g. due to # conflicts with existing zip_dir or zip_file_path). This also tests the # condition on the bots where they don't clean up their staging directory # between runs. zip_dir, zip_file_path = archive_utils.CreateArchive( self.build_dir, self.temp_dir, files_list, archive_name) self.assertTrue(zip_dir) self.assertTrue(zip_file_path) self.assertTrue(os.path.exists(zip_file_path)) self.verifyZipFile(zip_dir, zip_file_path, os.path.basename(zip_dir), files_list)
def testParserChange(self): """Changing parser criteria should be the same as creating a new one.""" files_cfg = CreateTestFilesCfg(self.temp_dir) arch = '64bit' buildtype = 'dev' oldfparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) old_dev_list = oldfparser.ParseLegacyList() buildtype = 'official' oldfparser.buildtype = buildtype old_official_list = oldfparser.ParseLegacyList() # The changed parser should return different ParseLegacyList. self.assertNotEqual(sorted(old_dev_list), sorted(old_official_list)) newfparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) new_official_list = newfparser.ParseLegacyList() # The new parser and changed parser should return the same data. self.assertEqual(sorted(old_official_list), sorted(new_official_list)) old_allany_list = oldfparser.ParseGroup('allany') new_allany_list = oldfparser.ParseGroup('allany') self.assertEqual(sorted(old_allany_list), sorted(new_allany_list))
def testDirectArchive(self): files_cfg = CreateTestFilesCfg(self.temp_dir) arch = '64bit' buildtype = 'dev' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) archives = fparser.ParseArchiveLists() self.assertTrue(fparser.IsDirectArchive( archives['renamed_direct_archive.txt'])) self.assertTrue(fparser.IsDirectArchive( archives['dev64_implied_direct_archive.txt'])) self.assertFalse(fparser.IsDirectArchive(archives['static_archive.zip']))
def testCreateEmptyArchive(self): files_cfg = CreateTestFilesCfg(self.tool_dir) archive_name = 'test_empty' arch = '64bit' buildtype = 'nosuchtype' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) files_list = fparser.ParseLegacyList() zip_dir, zip_file_path = archive_utils.CreateArchive( self.build_dir, self.temp_dir, files_list, archive_name) self.assertFalse(zip_dir) self.assertFalse(zip_file_path) self.assertFalse(os.path.exists(zip_file_path))
def testParseLegacyList(self): files_cfg = CreateTestFilesCfg(self.temp_dir) arch = '64bit' buildtype = 'official' fparser = archive_utils.FilesCfgParser(files_cfg, buildtype, arch) files_list = fparser.ParseLegacyList() # Verify FILES.cfg was parsed correctly. for i in TEST_FILES_CFG: if arch in i['arch'] and buildtype in i['buildtype']: # 'archive' flagged files shouldn't be included in the default parse. if i.get('archive'): self.assertFalse(i['filename'] in files_list) else: self.assertTrue(i['filename'] in files_list) files_list.remove(i['filename']) # No duplicate files. self.assertEqual(files_list.count(i['filename']), 0) # No unexpected files. self.assertEqual(len(files_list), 0)
def ArchiveBuild(self): """Zips build files and uploads them, their symbols, and a change log.""" result = 0 if self._build_revision is None: raise archive_utils.StagingError('No build revision was provided') print 'Staging in %s' % self._staging_dir fparser = archive_utils.FilesCfgParser(self._files_file, self.options.mode, self.options.arch) files_list = fparser.ParseLegacyList() self._archive_files = archive_utils.ExpandWildcards( self._build_dir, files_list) archives_list = fparser.ParseArchiveLists() # Check files and revision numbers. all_files_list = self._archive_files + [ item['filename'] for sublist in archives_list.values() for item in sublist ] all_files_list.append(self._version_file) not_found = archive_utils.VerifyFiles(all_files_list, self._build_dir, self.options.ignore) not_found_optional = [] for bad_fn in not_found[:]: if fparser.IsOptional(bad_fn): not_found_optional.append(bad_fn) not_found.remove(bad_fn) # Remove it from all file lists so we don't try to process it. if bad_fn in self._archive_files: self._archive_files.remove(bad_fn) for archive_list in archives_list.values(): archive_list[:] = [ x for x in archive_list if bad_fn != x['filename'] ] # TODO(mmoss): Now that we can declare files optional in FILES.cfg, should # we only allow not_found_optional, and fail on any leftover not_found # files? print 'last change: %s' % self._build_revision previous_revision = self.GetLastBuildRevision() # TODO(agable): This conditional only works for svn because git can't easily # compare revisions. if (slave_utils.GitOrSubversion(self._src_dir) == 'svn' and self._build_revision <= previous_revision): # If there have been no changes, report it but don't raise an exception. # Someone might have pushed the "force build" button. print 'No changes since last build (r%s <= r%s)' % ( self._build_revision, previous_revision) return 0 print 'build name: %s' % self._build_name archive_name = 'chrome-%s.zip' % self.TargetPlatformName() archive_file = self.CreateArchiveFile(archive_name, self._archive_files)[1] # Handle any custom archives. # TODO(mmoss): Largely copied from stage_build.py. Maybe refactor more of # this into archive_utils.py. archive_files = [archive_file] for archive_name in archives_list: # The list might be empty if it was all 'not_found' optional files. if not archives_list[archive_name]: continue if fparser.IsDirectArchive(archives_list[archive_name]): fileobj = archives_list[archive_name][0] # Copy the file to the path specified in archive_name, which might be # different than the dirname or basename in 'filename' (allowed by # 'direct_archive'). stage_subdir = os.path.dirname(archive_name) stage_fn = os.path.basename(archive_name) chromium_utils.MaybeMakeDirectory( os.path.join(self._staging_dir, stage_subdir)) print 'chromium_utils.CopyFileToDir(%s, %s, dest_fn=%s)' % ( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), stage_fn) if not self.options.dry_run: chromium_utils.CopyFileToDir( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), dest_fn=stage_fn) archive_files.append( os.path.join(self._staging_dir, archive_name)) else: custom_archive = self.CreateArchiveFile( archive_name, [f['filename'] for f in archives_list[archive_name]])[1] print 'Adding %s to be archived.' % (custom_archive) archive_files.append(custom_archive) # Generate a revisions file which contains the Chromium/WebKit/V8's # revision information. self.GenerateRevisionFile() www_dir = os.path.join(self._www_dir_base, self._build_path_component) gs_bucket = self.options.factory_properties.get('gs_bucket', None) gs_acl = self.options.factory_properties.get('gs_acl', None) gs_base = None if gs_bucket: gs_base = '/'.join( [gs_bucket, self._build_name, self._build_path_component]) self._UploadBuild(www_dir, self.revisions_path, archive_files, gs_base, gs_acl) # Archive Linux packages (if any -- only created for Chrome builds). if chromium_utils.IsLinux(): linux_packages = (glob.glob( os.path.join(self._build_dir, '*-r%s_*.deb' % self._chromium_revision))) linux_packages.extend( glob.glob( os.path.join(self._build_dir, '*-%s.*.rpm' % self._chromium_revision))) for package_file in linux_packages: print 'SshCopyFiles(%s, %s, %s)' % ( package_file, self.options.archive_host, www_dir) if not self.options.dry_run: print 'SshMakeDirectory(%s, %s)' % (self.options.archive_host, www_dir) self.MySshMakeDirectory(self.options.archive_host, www_dir, gs_base) for package_file in linux_packages: self.MyMakeWorldReadable(package_file, gs_base) self.MySshCopyFiles(package_file, self.options.archive_host, www_dir, gs_base, gs_acl=gs_acl) # Cleanup archived packages, otherwise they keep accumlating since # they have different filenames with each build. os.unlink(package_file) self.UploadTests(www_dir, gs_base, gs_acl) if not self.options.dry_run: # Save the current build revision locally so we can compute a changelog # next time self.SaveBuildRevisionToSpecifiedFile(self.last_change_file) # Record the latest revision in the developer archive directory. latest_file_path = os.path.join(self._www_dir_base, 'LATEST') if chromium_utils.IsWindows(): print 'Saving revision to %s' % latest_file_path if gs_base: self.CopyFileToGS(self.last_change_file, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) if not gs_base or self._dual_upload: self.SaveBuildRevisionToSpecifiedFile(latest_file_path) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): # Files are created umask 077 by default, so make it world-readable # before pushing to web server. self.MyMakeWorldReadable(self.last_change_file, gs_base) print 'Saving revision to %s:%s' % (self.options.archive_host, latest_file_path) self.MySshCopyFiles(self.last_change_file, self.options.archive_host, latest_file_path, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) if len(not_found_optional): sys.stderr.write('\n\nINFO: Optional File(s) not found: %s\n' % ', '.join(not_found_optional)) if len(not_found): sys.stderr.write('\n\nWARNING: File(s) not found: %s\n' % ', '.join(not_found)) return result
def DiffFilesCfg(cfg_path, svn): """Parse local FILES.cfg and show changes so they can be manually verified.""" print '\nDiff parsing "%s" ...' % cfg_path d = difflib.Differ() def CompareLists(svnlist, newlist, msg): diffs = [] for x in d.compare(svnlist, newlist): if x.startswith('- '): diffs.append(' DELETION: %s' % x[2:]) elif x.startswith('+ '): diffs.append(' ADDITION: %s' % x[2:]) if diffs: print msg print '\n'.join(diffs) svn_cfg = FetchSvn(RealFilesCfgTest.SVNBASE + cfg_path, svn) svnparser = archive_utils.FilesCfgParser(svn_cfg, None, None) os.unlink(svn_cfg) newparser = archive_utils.FilesCfgParser(options.src_base + cfg_path, None, None) # Determine the "parsable values" in the two versions. archs = [] buildtypes = [] groups = [] # pylint: disable=W0212 for item in newparser._files_cfg + svnparser._files_cfg: # pylint: enable=W0212 if item.get('arch'): archs.extend(item['arch']) if item.get('buildtype'): buildtypes.extend(item['buildtype']) if item.get('filegroup'): groups.extend(item['filegroup']) archs = set(archs) buildtypes = set(buildtypes) groups = set(groups) # Legacy list handling (i.e. default filegroup). print '\nChecking ParseLegacyList() ...' for arch, buildtype in itertools.product(archs, buildtypes): msg = '%s:%s' % (arch, buildtype) newparser.arch = svnparser.arch = arch newparser.buildtype = svnparser.buildtype = buildtype svn_legacy_list = svnparser.ParseLegacyList() new_legacy_list = newparser.ParseLegacyList() CompareLists(svn_legacy_list, new_legacy_list, msg) print '\nChecking ParseGroup() ...' for group, arch, buildtype in itertools.product(groups, archs, buildtypes): msg = '%s:%s:%s' % (group, arch, buildtype) newparser.arch = svnparser.arch = arch newparser.buildtype = svnparser.buildtype = buildtype svn_group_list = svnparser.ParseGroup(group) new_group_list = newparser.ParseGroup(group) CompareLists(svn_group_list, new_group_list, msg) print '\nChecking Archives() ...' for arch, buildtype in itertools.product(archs, buildtypes): newparser.arch = svnparser.arch = arch newparser.buildtype = svnparser.buildtype = buildtype svn_archive_lists = svnparser.ParseArchiveLists() new_archive_lists = newparser.ParseArchiveLists() archives = set(svn_archive_lists.keys() + new_archive_lists.keys()) for archive in archives: msg = '%s:%s:%s' % (archive, arch, buildtype) CompareLists([x['filename'] for x in svn_archive_lists.get(archive, [])], [x['filename'] for x in new_archive_lists.get(archive, [])], msg)