def _get_skp_version():
"""Find an unused SKP version."""
current_skp_version = None
with open(SKP_VERSION_FILE) as f:
current_skp_version = int(f.read().rstrip())
# Find the first SKP version which has no uploaded SKPs.
new_version = current_skp_version + 1
while True:
gs_path = 'playback_%d/skps' % new_version
if not gs_utils.GSUtils().does_storage_object_exist(
'chromium-skia-gm', gs_path):
return new_version
new_version += 1
def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name):
"""Downloads the webpages archive and its required page set from GS."""
wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
wpr_data_file)
page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME,
'webpages_archive',
page_set_json_name)
gs = gs_utils.GSUtils()
gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX)
if (gs.does_storage_object_exist(gs_bucket, wpr_source) and
gs.does_storage_object_exist(gs_bucket, page_set_source)):
gs.download_file(gs_bucket, wpr_source,
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
wpr_data_file))
gs.download_file(gs_bucket, page_set_source,
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_json_name))
else:
raise Exception('%s and %s do not exist in Google Storage!' % (
wpr_source, page_set_source))
def __init__(self, data_store_url):
self._data_store_url = data_store_url
self._bucket = remove_prefix(self._data_store_url.lstrip(),
gs_utils.GS_PREFIX)
self.gs = gs_utils.GSUtils()
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Download the credentials file if it was not previously downloaded.
if self._skip_all_gs_access:
print """\n\nPlease create a %s file that contains:
{
"google": {
"username": "******",
"password": "******"
},
"facebook": {
"username": "******",
"password": "******"
}
}\n\n""" % CREDENTIALS_FILE_PATH
raw_input("Please press a key when you are ready to proceed...")
elif not os.path.isfile(CREDENTIALS_FILE_PATH):
# Download the credentials.json file from Google Storage.
gs_bucket = remove_prefix(self._dest_gsbase.lstrip(),
gs_utils.GS_PREFIX)
gs_utils.GSUtils().download_file(gs_bucket, CREDENTIALS_GS_PATH,
CREDENTIALS_FILE_PATH)
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0]
page_set_json_name = page_set_basename + '.json'
wpr_data_file = page_set.split(
os.path.sep)[-1].split('.')[0] + '_000.wpr'
page_set_dir = os.path.dirname(page_set)
if self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
'%s_page_set' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir)
for _ in range(RETRY_RECORD_WPR_COUNT):
try:
shell_utils.run(' '.join(record_wpr_cmd), shell=True)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' %
page_set)
else:
if not self._skip_all_gs_access:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file,
page_set_json_name)
run_benchmark_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300',
os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK, '--page-set-name=%s' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir,
'--skp-outdir=%s' % TMP_SKP_DIR, '--also-run-disabled-tests')
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_benchmark_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
if self._record:
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_json_name),
self._local_record_webpages_archive_dir)
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
# If we get here then run_benchmark did not succeed and thus did not
# break out of the loop.
raise Exception('run_benchmark failed for page_set: %s' %
page_set)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'), '-r',
self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'), '-r',
self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code,
_) = shell_utils.log_process_after_completion(proc,
echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(
self._skia_tools, 'debugger'), self._local_skp_dir))
print '\n\n'
if not self._skip_all_gs_access and self._upload_to_gs:
print '\n\n=======Uploading to Google Storage=======\n\n'
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
gs_bucket = remove_prefix(self._dest_gsbase.lstrip(),
gs_utils.GS_PREFIX)
gs_utils.GSUtils().upload_dir_contents(
LOCAL_PLAYBACK_ROOT_DIR,
gs_bucket,
dest_dir_name,
upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
predefined_acl=GS_PREDEFINED_ACL,
fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self._dest_gsbase, dest_dir_name,
SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to Google Storage=======\n\n'
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
