def __init__(self, options):
"""Overwrite _build_revision for archiving build by Chromium revision."""
StagerBase.__init__(self, options, None)
self._build_revision = self._chromium_revision
self._build_path_component = chromium_utils.GetSortableUploadPathForSortKey(
*chromium_utils.GetBuildSortKey(options))
def GetBuildSortKey(options, primary_project):
"""Returns: (str) the build sort key for the specified project.
Attempts to identify the build sort key for a given project. If
'primary_project' is None or if there is no sort key for the specified
primary project, the checkout-wide sort key will be used.
Raises:
chromium_utils.NoIdentifiedRevision: if the checkout-wide sort key could not
be resolved.
"""
if primary_project:
try:
return chromium_utils.GetBuildSortKey(options,
project=primary_project)
except chromium_utils.NoIdentifiedRevision:
pass
return chromium_utils.GetBuildSortKey(options)
def __init__(self, options, build_revision):
"""Sets a number of file and directory paths for convenient use."""
self.options = options
self._src_dir = os.path.abspath(options.src_dir)
self._chrome_dir = os.path.join(self._src_dir, 'chrome')
build_dir = build_directory.GetBuildOutputDirectory()
self._build_dir = os.path.join(build_dir, options.target)
if chromium_utils.IsWindows():
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'win')
elif chromium_utils.IsLinux():
# On Linux, we might have built for chromeos. Archive the same.
if (options.factory_properties.get('chromeos', None)
or slave_utils.GypFlagIsOn(options, 'chromeos')):
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'chromeos')
# Or, we might have built for Android.
elif options.factory_properties.get('target_os') == 'android':
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'android')
else:
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'linux')
elif chromium_utils.IsMac():
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'mac')
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
self._staging_dir = (options.staging_dir
or slave_utils.GetStagingDir(self._src_dir))
if not os.path.exists(self._staging_dir):
os.makedirs(self._staging_dir)
self._symbol_dir_base = options.dirs['symbol_dir_base']
self._www_dir_base = options.dirs['www_dir_base']
if options.build_name:
self._build_name = options.build_name
else:
self._build_name = slave_utils.SlaveBuildName(self._src_dir)
self._symbol_dir_base = os.path.join(self._symbol_dir_base,
self._build_name)
self._www_dir_base = os.path.join(self._www_dir_base, self._build_name)
self._version_file = os.path.join(self._chrome_dir, 'VERSION')
self._chromium_revision = chromium_utils.GetBuildSortKey(options)[1]
self._v8_revision = chromium_utils.GetBuildSortKey(options,
project='v8')[1]
self._v8_revision_git = chromium_utils.GetGitCommit(options,
project='v8')
self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE')
# The REVISIONS file will record the revisions information of the main
# components Chromium/WebKit/V8.
self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS')
self._build_revision = build_revision
self._build_path_component = str(self._build_revision)
# Will be initialized in GetLastBuildRevision.
self.last_chromium_revision = None
self.last_v8_revision = None
self._files_file = os.path.join(self._tool_dir,
archive_utils.FILES_FILENAME)
self._test_files = self.BuildOldFilesList(TEST_FILE_NAME)
self._dual_upload = options.factory_properties.get(
'dual_upload', False)
self._archive_files = None
def main():
option_parser = optparse.OptionParser()
chromium_utils.AddPropertiesOptions(option_parser)
options, _ = option_parser.parse_args()
if not os.path.exists('src'):
raise Exception('ERROR: no src directory to package, exiting')
try:
revision_upload_path = chromium_utils.GetSortableUploadPathForSortKey(
*chromium_utils.GetBuildSortKey(options))
except chromium_utils.NoIdentifiedRevision:
revision_upload_path = 'NONE'
completed_filename = '%s-%s.%s' % (options.factory_properties.get(
'package_filename', FILENAME), revision_upload_path, EXT)
partial_filename = '%s.partial' % completed_filename
chromium_utils.RunCommand(['rm', '-f', partial_filename])
if os.path.exists(partial_filename):
raise Exception('ERROR: %s cannot be removed, exiting' %
partial_filename)
print '%s: Index generation...' % time.strftime('%X')
indexing_successful = GenerateIndex()
print '%s: Creating tar file...' % time.strftime('%X')
packaging_successful = True
find_command = [
'find',
'src/',
'tools/',
'/usr/include/',
'-type',
'f',
# The only files under src/out we want to package up
# are index files....
'(',
'-regex',
'^src/out/.*\.index$',
'-o',
'(',
# ... and generated sources...
'-regex',
'^src/out/.*/gen/.*',
'-a',
'(',
'-name',
'*.h',
'-o',
'-name',
'*.cc',
'-o',
'-name',
'*.cpp',
'-o',
'-name',
'*.js',
')',
'-a',
# ... but none of the NaCL stuff.
'!',
'-regex',
'^src/out/[^/]*/gen/lib[^/]*/.*',
'-a',
'!',
'-regex',
'^src/out/[^/]*/gen/sdk/.*',
'-a',
'!',
'-regex',
'^src/out/[^/]*/gen/tc_.*',
')',
'-o',
'!',
'-regex',
'^src/out/.*',
')',
'-a',
# Exclude all .svn directories, the native client toolchain
# and the llvm build directory, and perf/data files.
'!',
'-regex',
r'.*/\.svn/.*',
'-a',
'!',
'-regex',
r'.*/\.git/.*',
'-a',
'!',
'-regex',
'^src/native_client/toolchain/.*',
'-a',
'!',
'-regex',
'^src/native_client/.*/testdata/.*',
'-a',
'!',
'-regex',
'^src/third_party/llvm-build/.*',
'-a',
'!',
'-regex',
'^src/.*/\.cvsignore',
'-a',
'!',
'-regex',
'^src/chrome/tools/test/reference_build/.*',
'-a',
'!',
'-regex',
'^tools/perf/data/.*'
]
try:
if chromium_utils.RunCommand(
find_command, pipes=[['tar', '-T-', '-cjvf', partial_filename]
]) != 0:
raise Exception('ERROR: failed to create %s, exiting' %
partial_filename)
print '%s: Cleaning up google storage...' % time.strftime('%X')
DeleteIfExists(completed_filename)
DeleteIfExists(partial_filename)
print '%s: Uploading...' % time.strftime('%X')
status = slave_utils.GSUtilCopyFile(partial_filename,
GSBASE,
gs_acl=GSACL)
if status != 0:
raise Exception('ERROR: GSUtilCopyFile error %d. "%s" -> "%s"' %
(status, partial_filename, GSBASE))
print '%s: Finalizing google storage...' % time.strftime('%X')
status = slave_utils.GSUtilMoveFile(
'%s/%s' % (GSBASE, partial_filename),
'%s/%s' % (GSBASE, completed_filename),
gs_acl=GSACL)
if status != 0:
raise Exception('ERROR: GSUtilMoveFile error %d. "%s" -> "%s"' %
(status, '%s/%s' %
(GSBASE, partial_filename), '%s/%s' %
(GSBASE, completed_filename)))
(status, output) = slave_utils.GSUtilListBucket(GSBASE, ['-l'])
if status != 0:
raise Exception('ERROR: failed to get list of GSBASE, exiting' %
GSBASE)
regex = re.compile('\s*\d+\s+([-:\w]+)\s+%s/%s\n' %
(GSBASE, completed_filename))
match_data = regex.search(output)
modified_time = None
if match_data:
modified_time = match_data.group(1)
if not modified_time:
raise Exception('ERROR: could not get modified_time, exiting')
print 'Last modified time: %s' % modified_time
print '%s: Deleting old archives on google storage...' % time.strftime(
'%X')
regex = re.compile('\s*\d+\s+([-:\w]+)\s+(%s/.*%s.*)\n' %
(GSBASE, EXT))
last_week = int(time.time()) - 7 * 24 * 60 * 60
for match_data in regex.finditer(output):
timestamp = int(
time.strftime(
'%s',
time.strptime(match_data.group(1), '%Y-%m-%dT%H:%M:%S')))
if timestamp < last_week:
print 'Deleting %s...' % match_data.group(2)
status = slave_utils.GSUtilDeleteFile(match_data.group(2))
if status != 0:
raise Exception('ERROR: GSUtilDeleteFile error %d. "%s"' %
(status, match_data.group(2)))
except Exception, e:
print str(e)
packaging_successful = False