def testGSUtilCopyDirWithQuietFlag(
self, # pylint: disable=no-self-use
run_command_mock):
slave_utils.GSUtilCopyDir('foo', 'bar', add_quiet_flag=True)
run_command_mock.assert_called_with([
'/mock/gsutil',
'-m',
'-q',
'cp',
'-R',
'foo',
'bar',
])
def testGSUtilCopyCacheControl(
self, # pylint: disable=R0201
run_command_mock,
gs_util_setup_mock):
slave_utils.GSUtilCopyFile('foo', 'bar', cache_control='mock_cache')
run_command_mock.assert_called_with([
'/mock/gsutil', '-h', 'Cache-Control:mock_cache', 'cp',
'file://foo', 'file://bar/foo'
])
slave_utils.GSUtilCopyDir('foo', 'bar', cache_control='mock_cache')
run_command_mock.assert_called_with([
'/mock/gsutil', '-m', '-h', 'Cache-Control:mock_cache', 'cp', '-R',
'foo', 'bar'
])
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Ensure the right .boto file is used by gsutil.
if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file(
timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
gs_base=self._dest_gsbase,
gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME,
SKPICTURES_DIR_NAME)) == "0":
raise Exception(
'Missing .boto file or .boto does not have the right credentials.'
'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q'
'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit '
'(may have to request access). The .boto file will need to be placed '
'under third_party/chromium_buildbot/site_config/')
# Download the credentials file if it was not previously downloaded.
if self._skip_all_gs_access:
print """\n\nPlease create a %s file that contains:
{
"google": {
"username": "******",
"password": "******"
},
"facebook": {
"username": "******",
"password": "******"
}
}\n\n""" % CREDENTIALS_FILE_PATH
raw_input("Please press a key when you are ready to proceed...")
elif not os.path.isfile(CREDENTIALS_FILE_PATH):
# Download the credentials.json file from Google Storage.
slave_utils.GSUtilDownloadFile(
src=CREDENTIALS_GS_LOCATION, dst=CREDENTIALS_FILE_PATH)
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0] + '.json'
wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
if self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(TELEMETRY_BINARIES_DIR, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
page_set
)
for _ in range(RETRY_RECORD_WPR_COUNT):
output = shell_utils.run(' '.join(record_wpr_cmd), shell=True)
if RECORD_FAILURE_MSG in output:
print output
else:
# Break out of the retry loop since there were no errors.
break
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' % page_set)
else:
if not self._skip_all_gs_access:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file, page_set_basename)
run_measurement_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
'timeout', '300',
os.path.join(TELEMETRY_BINARIES_DIR, 'run_measurement'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK,
page_set,
'-o',
'/tmp/test.skp',
'--skp-outdir=%s' % TMP_SKP_DIR
)
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_measurement_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
if self._record:
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_basename),
self._local_record_webpages_archive_dir)
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
# If we get here then run_measurement did not succeed and thus did not
# break out of the loop.
raise Exception('run_measurement failed for page_set: %s' % page_set)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'),
'-r', self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'),
self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code, output) = shell_utils.log_process_after_completion(proc,
echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
os.path.join(self._local_skp_dir, '*')))
print '\n\n'
if not self._skip_all_gs_access and self._upload_to_gs:
print '\n\n=======Uploading to Google Storage=======\n\n'
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
gs_status = slave_utils.GSUtilCopyDir(
src_dir=LOCAL_PLAYBACK_ROOT_DIR,
gs_base=self._dest_gsbase,
dest_dir=dest_dir_name,
gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
if gs_status != 0:
raise Exception(
'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % (
gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase,
ROOT_PLAYBACK_DIR_NAME))
self._SetGoogleReadACLs(
posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
# Add a timestamp file to the SKP directory in Google Storage so we can
# use directory level rsync like functionality.
old_gs_utils.write_timestamp_file(
timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
timestamp_value=time.time(),
gs_base=self._dest_gsbase,
gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME),
gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
local_dir=LOCAL_PLAYBACK_ROOT_DIR)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to Google Storage=======\n\n'
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
def archive_layout(options, args):
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s:%(lineno)-3d'
' %(levelname)s %(message)s',
datefmt='%y%m%d %H:%M:%S')
chrome_dir = os.path.abspath(options.build_dir)
results_dir_basename = os.path.basename(options.results_dir)
if options.results_dir is not None:
options.results_dir = os.path.abspath(os.path.join(options.build_dir,
options.results_dir))
else:
options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
print 'Archiving results from %s' % options.results_dir
staging_dir = options.staging_dir or slave_utils.GetStagingDir(chrome_dir)
print 'Staging in %s' % staging_dir
if not os.path.exists(staging_dir):
os.makedirs(staging_dir)
(actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir)
zip_file = chromium_utils.MakeZip(staging_dir,
results_dir_basename,
actual_file_list,
options.results_dir)[1]
# TODO(ojan): Stop separately uploading full_results.json once garden-o-matic
# switches to using failing_results.json.
full_results_json = os.path.join(options.results_dir, 'full_results.json')
failing_results_json = os.path.join(options.results_dir,
'failing_results.json')
# Extract the build name of this slave (e.g., 'chrome-release') from its
# configuration file if not provided as a param.
build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir)
build_name = re.sub('[ .()]', '_', build_name)
wc_dir = os.path.dirname(chrome_dir)
last_change = slave_utils.GetHashOrRevision(wc_dir)
# TODO(dpranke): Is it safe to assume build_number is not blank? Should we
# assert() this ?
build_number = str(options.build_number)
print 'last change: %s' % last_change
print 'build name: %s' % build_name
print 'build number: %s' % build_number
print 'host name: %s' % socket.gethostname()
if options.gs_bucket:
# Create a file containing last_change revision. This file will be uploaded
# after all layout test results are uploaded so the client can check this
# file to see if the upload for the revision is complete.
# See crbug.com/574272 for more details.
last_change_file = os.path.join(staging_dir, 'LAST_CHANGE')
with open(last_change_file, 'w') as f:
f.write(last_change)
# Copy the results to a directory archived by build number.
gs_base = '/'.join([options.gs_bucket, build_name, build_number])
gs_acl = options.gs_acl
# These files never change, cache for a year.
cache_control = "public, max-age=31556926"
slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
# TODO(dpranke): Remove these two lines once clients are fetching the
# files from the layout-test-results dir.
slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyFile(failing_results_json, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyFile(last_change_file,
gs_base + '/' + results_dir_basename, gs_acl=gs_acl,
cache_control=cache_control)
# And also to the 'results' directory to provide the 'latest' results
# and make sure they are not cached at all (Cloud Storage defaults to
# caching w/ a max-age=3600).
gs_base = '/'.join([options.gs_bucket, build_name, 'results'])
cache_control = 'no-cache'
slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyFile(last_change_file,
gs_base + '/' + results_dir_basename, gs_acl=gs_acl,
cache_control=cache_control)
else:
# Where to save layout test results.
dest_parent_dir = os.path.join(archive_utils.Config.www_dir_base,
results_dir_basename.replace('-', '_'), build_name)
dest_dir = os.path.join(dest_parent_dir, last_change)
_MaybeMakeDirectoryOnArchiveHost(dest_dir)
_CopyFileToArchiveHost(zip_file, dest_dir)
_CopyFileToArchiveHost(full_results_json, dest_dir)
_CopyFileToArchiveHost(failing_results_json, dest_dir)
# Not supported on Google Storage yet.
_ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list,
options)
return 0