def __init__(self, options): """Overwrite _build_revision for archiving build by Chromium revision.""" StagerBase.__init__(self, options, None) self._build_revision = self._chromium_revision self._build_path_component = chromium_utils.GetSortableUploadPathForSortKey( *chromium_utils.GetBuildSortKey(options))
def GetBuildSortKey(options, primary_project): """Returns: (str) the build sort key for the specified project. Attempts to identify the build sort key for a given project. If 'primary_project' is None or if there is no sort key for the specified primary project, the checkout-wide sort key will be used. Raises: chromium_utils.NoIdentifiedRevision: if the checkout-wide sort key could not be resolved. """ if primary_project: try: return chromium_utils.GetBuildSortKey(options, project=primary_project) except chromium_utils.NoIdentifiedRevision: pass return chromium_utils.GetBuildSortKey(options)
def __init__(self, options, build_revision): """Sets a number of file and directory paths for convenient use.""" self.options = options self._src_dir = os.path.abspath(options.src_dir) self._chrome_dir = os.path.join(self._src_dir, 'chrome') build_dir = build_directory.GetBuildOutputDirectory() self._build_dir = os.path.join(build_dir, options.target) if chromium_utils.IsWindows(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'win') elif chromium_utils.IsLinux(): # On Linux, we might have built for chromeos. Archive the same. if (options.factory_properties.get('chromeos', None) or slave_utils.GypFlagIsOn(options, 'chromeos')): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'chromeos') # Or, we might have built for Android. elif options.factory_properties.get('target_os') == 'android': self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'android') else: self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'linux') elif chromium_utils.IsMac(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'mac') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) self._staging_dir = (options.staging_dir or slave_utils.GetStagingDir(self._src_dir)) if not os.path.exists(self._staging_dir): os.makedirs(self._staging_dir) self._symbol_dir_base = options.dirs['symbol_dir_base'] self._www_dir_base = options.dirs['www_dir_base'] if options.build_name: self._build_name = options.build_name else: self._build_name = slave_utils.SlaveBuildName(self._src_dir) self._symbol_dir_base = os.path.join(self._symbol_dir_base, self._build_name) self._www_dir_base = os.path.join(self._www_dir_base, self._build_name) self._version_file = os.path.join(self._chrome_dir, 'VERSION') self._chromium_revision = chromium_utils.GetBuildSortKey(options)[1] self._v8_revision = chromium_utils.GetBuildSortKey(options, project='v8')[1] self._v8_revision_git = chromium_utils.GetGitCommit(options, project='v8') self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE') # The REVISIONS file will record the revisions information of the main # components Chromium/WebKit/V8. self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS') self._build_revision = build_revision self._build_path_component = str(self._build_revision) # Will be initialized in GetLastBuildRevision. self.last_chromium_revision = None self.last_v8_revision = None self._files_file = os.path.join(self._tool_dir, archive_utils.FILES_FILENAME) self._test_files = self.BuildOldFilesList(TEST_FILE_NAME) self._dual_upload = options.factory_properties.get( 'dual_upload', False) self._archive_files = None
def main(): option_parser = optparse.OptionParser() chromium_utils.AddPropertiesOptions(option_parser) options, _ = option_parser.parse_args() if not os.path.exists('src'): raise Exception('ERROR: no src directory to package, exiting') try: revision_upload_path = chromium_utils.GetSortableUploadPathForSortKey( *chromium_utils.GetBuildSortKey(options)) except chromium_utils.NoIdentifiedRevision: revision_upload_path = 'NONE' completed_filename = '%s-%s.%s' % (options.factory_properties.get( 'package_filename', FILENAME), revision_upload_path, EXT) partial_filename = '%s.partial' % completed_filename chromium_utils.RunCommand(['rm', '-f', partial_filename]) if os.path.exists(partial_filename): raise Exception('ERROR: %s cannot be removed, exiting' % partial_filename) print '%s: Index generation...' % time.strftime('%X') indexing_successful = GenerateIndex() print '%s: Creating tar file...' % time.strftime('%X') packaging_successful = True find_command = [ 'find', 'src/', 'tools/', '/usr/include/', '-type', 'f', # The only files under src/out we want to package up # are index files.... '(', '-regex', '^src/out/.*\.index$', '-o', '(', # ... and generated sources... '-regex', '^src/out/.*/gen/.*', '-a', '(', '-name', '*.h', '-o', '-name', '*.cc', '-o', '-name', '*.cpp', '-o', '-name', '*.js', ')', '-a', # ... but none of the NaCL stuff. '!', '-regex', '^src/out/[^/]*/gen/lib[^/]*/.*', '-a', '!', '-regex', '^src/out/[^/]*/gen/sdk/.*', '-a', '!', '-regex', '^src/out/[^/]*/gen/tc_.*', ')', '-o', '!', '-regex', '^src/out/.*', ')', '-a', # Exclude all .svn directories, the native client toolchain # and the llvm build directory, and perf/data files. '!', '-regex', r'.*/\.svn/.*', '-a', '!', '-regex', r'.*/\.git/.*', '-a', '!', '-regex', '^src/native_client/toolchain/.*', '-a', '!', '-regex', '^src/native_client/.*/testdata/.*', '-a', '!', '-regex', '^src/third_party/llvm-build/.*', '-a', '!', '-regex', '^src/.*/\.cvsignore', '-a', '!', '-regex', '^src/chrome/tools/test/reference_build/.*', '-a', '!', '-regex', '^tools/perf/data/.*' ] try: if chromium_utils.RunCommand( find_command, pipes=[['tar', '-T-', '-cjvf', partial_filename] ]) != 0: raise Exception('ERROR: failed to create %s, exiting' % partial_filename) print '%s: Cleaning up google storage...' % time.strftime('%X') DeleteIfExists(completed_filename) DeleteIfExists(partial_filename) print '%s: Uploading...' % time.strftime('%X') status = slave_utils.GSUtilCopyFile(partial_filename, GSBASE, gs_acl=GSACL) if status != 0: raise Exception('ERROR: GSUtilCopyFile error %d. "%s" -> "%s"' % (status, partial_filename, GSBASE)) print '%s: Finalizing google storage...' % time.strftime('%X') status = slave_utils.GSUtilMoveFile( '%s/%s' % (GSBASE, partial_filename), '%s/%s' % (GSBASE, completed_filename), gs_acl=GSACL) if status != 0: raise Exception('ERROR: GSUtilMoveFile error %d. "%s" -> "%s"' % (status, '%s/%s' % (GSBASE, partial_filename), '%s/%s' % (GSBASE, completed_filename))) (status, output) = slave_utils.GSUtilListBucket(GSBASE, ['-l']) if status != 0: raise Exception('ERROR: failed to get list of GSBASE, exiting' % GSBASE) regex = re.compile('\s*\d+\s+([-:\w]+)\s+%s/%s\n' % (GSBASE, completed_filename)) match_data = regex.search(output) modified_time = None if match_data: modified_time = match_data.group(1) if not modified_time: raise Exception('ERROR: could not get modified_time, exiting') print 'Last modified time: %s' % modified_time print '%s: Deleting old archives on google storage...' % time.strftime( '%X') regex = re.compile('\s*\d+\s+([-:\w]+)\s+(%s/.*%s.*)\n' % (GSBASE, EXT)) last_week = int(time.time()) - 7 * 24 * 60 * 60 for match_data in regex.finditer(output): timestamp = int( time.strftime( '%s', time.strptime(match_data.group(1), '%Y-%m-%dT%H:%M:%S'))) if timestamp < last_week: print 'Deleting %s...' % match_data.group(2) status = slave_utils.GSUtilDeleteFile(match_data.group(2)) if status != 0: raise Exception('ERROR: GSUtilDeleteFile error %d. "%s"' % (status, match_data.group(2))) except Exception, e: print str(e) packaging_successful = False