def submit_tries(bots_to_run, dry_run=False):
"""Submit try requests for the current branch on the given bots.
Args:
bots_to_run: (list of strings) bots to request.
dry_run: (bool) whether to actually submit the try request.
"""
git_try = [
git_utils.GIT, 'cl', 'try', '-m', 'tryserver.chromium']
git_try.extend([arg for bot in bots_to_run for arg in ('-b', bot)])
if dry_run:
space = ' '
print 'You should call:'
print space, git_try
print
else:
shell_utils.run(git_try)
def get_running_pid(master):
master_hostname = master.master_fqdn.split('.')[0]
pid_cmd = slave_hosts_cfg.compute_engine_login(master_hostname, None)
pid_cmd.extend(['cat', PID_FILE])
try:
return shell_utils.run(pid_cmd, echo=False).splitlines()[-1]
except shell_utils.CommandFailedException as e:
if 'No such file or directory' in e.output:
return None
raise
def get_running_pid(master):
master_hostname = master.master_fqdn.split('.')[0]
pid_cmd = slave_hosts_cfg.compute_engine_login(master_hostname, None)
pid_cmd.extend(['cat', PID_FILE])
try:
return shell_utils.run(pid_cmd, echo=False).splitlines()[-1]
except shell_utils.CommandFailedException as e:
if 'No such file or directory' in e.output:
return None
raise
def main(chrome_src_path, browser_executable, dry_run):
browser_executable = os.path.realpath(browser_executable)
dry_run = (dry_run == 'True')
skp_version = _get_skp_version()
print 'SKP_VERSION=%d' % skp_version
if os.environ.get('CHROME_HEADLESS'):
# Start Xvfb if running on a bot.
try:
shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &',
shell=True)
except Exception:
# It is ok if the above command fails, it just means that DISPLAY=:0
# is already up.
pass
upload_dir = 'playback_%d' % skp_version
webpages_playback_cmd = [
'python',
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'webpages_playback.py'),
'--page_sets',
'all',
'--browser_executable',
browser_executable,
'--non-interactive',
'--upload',
'--alternate_upload_dir',
upload_dir,
'--chrome_src_path',
chrome_src_path,
]
if not dry_run:
webpages_playback_cmd.append('--upload_to_partner_bucket')
try:
shell_utils.run(webpages_playback_cmd)
finally:
# Clean up any leftover browser instances. This can happen if there are
# telemetry crashes, processes are not always cleaned up appropriately by
# the webpagereplay and telemetry frameworks.
procs = subprocess.check_output(['ps', 'ax'])
for line in procs.splitlines():
if browser_executable in line:
pid = line.strip().split(' ')[0]
if pid != str(os.getpid()) and not 'python' in line:
try:
shell_utils.run(['kill', '-9', pid])
except shell_utils.CommandFailedException as e:
print e
else:
print 'Refusing to kill self.'
print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
with open(SKP_VERSION_FILE, 'w') as f:
f.write(str(skp_version))
def main(chrome_src_path, browser_executable, dry_run):
browser_executable = os.path.realpath(browser_executable)
dry_run = (dry_run == 'True')
skp_version = _get_skp_version()
print 'SKP_VERSION=%d' % skp_version
if os.environ.get('CHROME_HEADLESS'):
# Start Xvfb if running on a bot.
try:
shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True)
except Exception:
# It is ok if the above command fails, it just means that DISPLAY=:0
# is already up.
pass
upload_dir = 'playback_%d' % skp_version
webpages_playback_cmd = [
'python', os.path.join(os.path.dirname(os.path.realpath(__file__)),
'webpages_playback.py'),
'--page_sets', 'all',
'--browser_executable', browser_executable,
'--non-interactive',
'--upload',
'--alternate_upload_dir', upload_dir,
'--chrome_src_path', chrome_src_path,
]
if not dry_run:
webpages_playback_cmd.append('--upload_to_partner_bucket')
try:
shell_utils.run(webpages_playback_cmd)
finally:
# Clean up any leftover browser instances. This can happen if there are
# telemetry crashes, processes are not always cleaned up appropriately by
# the webpagereplay and telemetry frameworks.
procs = subprocess.check_output(['ps', 'ax'])
for line in procs.splitlines():
if browser_executable in line:
pid = line.strip().split(' ')[0]
if pid != str(os.getpid()) and not 'python' in line:
try:
shell_utils.run(['kill', '-9', pid])
except shell_utils.CommandFailedException as e:
print e
else:
print 'Refusing to kill self.'
print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
with open(SKP_VERSION_FILE, 'w') as f:
f.write(str(skp_version))
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Download the credentials file if it was not previously downloaded.
if not os.path.isfile(CREDENTIALS_FILE_PATH):
# Download the credentials.json file from Google Storage.
self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH)
if not os.path.isfile(CREDENTIALS_FILE_PATH):
print """\n\nCould not locate credentials file in the storage.
Please create a %s file that contains:
{
"google": {
"username": "******",
"password": "******"
},
"facebook": {
"username": "******",
"password": "******"
}
}\n\n""" % CREDENTIALS_FILE_PATH
raw_input("Please press a key when you are ready to proceed...")
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0]
page_set_json_name = page_set_basename + '.json'
wpr_data_file = page_set.split(
os.path.sep)[-1].split('.')[0] + '_000.wpr'
page_set_dir = os.path.dirname(page_set)
if self._IsChromiumPageSet(page_set):
print 'Using Chromium\'s captured archives for Chromium\'s page sets.'
elif self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
'%s_page_set' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir)
for _ in range(RETRY_RECORD_WPR_COUNT):
try:
shell_utils.run(' '.join(record_wpr_cmd), shell=True)
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_json_name),
self._local_record_webpages_archive_dir)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' %
page_set)
else:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file,
page_set_json_name)
run_benchmark_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300',
os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK, '--page-set-name=%s' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir,
'--skp-outdir=%s' % TMP_SKP_DIR, '--also-run-disabled-tests')
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_benchmark_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
# If we get here then run_benchmark did not succeed and thus did not
# break out of the loop.
raise Exception('run_benchmark failed for page_set: %s' %
page_set)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'), '-r',
self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'), '-r',
self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code,
_) = shell_utils.log_process_after_completion(proc,
echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(
self._skia_tools, 'debugger'), self._local_skp_dir))
print '\n\n'
if self._upload:
print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type(
)
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
self.gs.upload_dir_contents(
LOCAL_PLAYBACK_ROOT_DIR,
dest_dir=dest_dir_name,
upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
predefined_acl=GS_PREDEFINED_ACL,
fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self.gs.target_name(), dest_dir_name,
SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type(
)
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Download the credentials file if it was not previously downloaded.
if not os.path.isfile(CREDENTIALS_FILE_PATH):
# Download the credentials.json file from Google Storage.
self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH)
if not os.path.isfile(CREDENTIALS_FILE_PATH):
print """\n\nCould not locate credentials file in the storage.
Please create a %s file that contains:
{
"google": {
"username": "******",
"password": "******"
},
"facebook": {
"username": "******",
"password": "******"
}
}\n\n""" % CREDENTIALS_FILE_PATH
raw_input("Please press a key when you are ready to proceed...")
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0]
page_set_json_name = page_set_basename + '.json'
wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
page_set_dir = os.path.dirname(page_set)
if self._IsChromiumPageSet(page_set):
print 'Using Chromium\'s captured archives for Chromium\'s page sets.'
elif self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
'%s_page_set' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir
)
for _ in range(RETRY_RECORD_WPR_COUNT):
try:
shell_utils.run(' '.join(record_wpr_cmd), shell=True)
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_json_name),
self._local_record_webpages_archive_dir)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' % page_set)
else:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name)
run_benchmark_cmd = (
'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
'DISPLAY=%s' % X11_DISPLAY,
'timeout', '300',
os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK,
'--page-set-name=%s' % page_set_basename,
'--page-set-base-dir=%s' % page_set_dir,
'--skp-outdir=%s' % TMP_SKP_DIR,
'--also-run-disabled-tests'
)
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_benchmark_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
# If we get here then run_benchmark did not succeed and thus did not
# break out of the loop.
raise Exception('run_benchmark failed for page_set: %s' % page_set)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'),
'-r', self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'),
'-r', self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code, _) = shell_utils.log_process_after_completion(proc, echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
self._local_skp_dir))
print '\n\n'
if self._upload:
print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type()
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
self.gs.upload_dir_contents(
LOCAL_PLAYBACK_ROOT_DIR, dest_dir=dest_dir_name,
upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
predefined_acl=GS_PREDEFINED_ACL,
fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self.gs.target_name(), dest_dir_name,
SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type()
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
def main(chrome_src_path, browser_executable):
browser_executable = os.path.realpath(browser_executable)
skp_version = _get_skp_version()
print 'SKP_VERSION=%d' % skp_version
if os.environ.get('CHROME_HEADLESS'):
# Start Xvfb if running on a bot.
try:
shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True)
except Exception:
# It is ok if the above command fails, it just means that DISPLAY=:0
# is already up.
pass
upload_dir = 'playback_%d' % skp_version
webpages_playback_cmd = [
'python', os.path.join(os.path.dirname(os.path.realpath(__file__)),
'webpages_playback.py'),
'--page_sets', 'all',
'--browser_executable', browser_executable,
'--non-interactive',
'--upload',
'--alternate_upload_dir', upload_dir,
'--chrome_src_path', chrome_src_path,
]
try:
shell_utils.run(webpages_playback_cmd)
# Temporary change to enable Slimming Paint runs. See skia:3763.
skia_page_sets_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'page_sets')
sp_skia_page_sets = [
os.path.join(skia_page_sets_path, 'skia_carsvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_chalkboard_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_css3gradients_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_espn_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_gmailthread_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_googlehome_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_googlespreadsheet_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_jsfiddlebigcar_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_mapsvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_nytimes_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_samoasvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_tigersvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_ugamsolutions_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_digg_nexus10.py'),
os.path.join(skia_page_sets_path, 'skia_gmail_nexus10.py'),
os.path.join(skia_page_sets_path, 'skia_pravda_nexus10.py'),
]
webpages_playback_cmd.extend([
'--skp_prefix', 'sp_',
'--browser_extra_args', '--enable-slimming-paint',
'--page_sets', '%s' % ','.join(sp_skia_page_sets),
])
shell_utils.run(webpages_playback_cmd)
finally:
# Clean up any leftover browser instances. This can happen if there are
# telemetry crashes, processes are not always cleaned up appropriately by
# the webpagereplay and telemetry frameworks.
procs = subprocess.check_output(['ps', 'ax'])
for line in procs.splitlines():
if browser_executable in line:
pid = line.strip().split(' ')[0]
if pid != str(os.getpid()) and not 'python' in line:
try:
shell_utils.run(['kill', '-9', pid])
except shell_utils.CommandFailedException as e:
print e
else:
print 'Refusing to kill self.'
print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
with open(SKP_VERSION_FILE, 'w') as f:
f.write(str(skp_version))
def main(chrome_src_path, browser_executable):
browser_executable = os.path.realpath(browser_executable)
skp_version = _get_skp_version()
print 'SKP_VERSION=%d' % skp_version
if os.environ.get('CHROME_HEADLESS'):
# Start Xvfb if running on a bot.
try:
shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &',
shell=True)
except Exception:
# It is ok if the above command fails, it just means that DISPLAY=:0
# is already up.
pass
upload_dir = 'playback_%d' % skp_version
webpages_playback_cmd = [
'python',
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'webpages_playback.py'),
'--page_sets',
'all',
'--browser_executable',
browser_executable,
'--non-interactive',
'--upload',
'--alternate_upload_dir',
upload_dir,
'--chrome_src_path',
chrome_src_path,
]
try:
shell_utils.run(webpages_playback_cmd)
# Temporary change to enable Slimming Paint runs. See skia:3763.
skia_page_sets_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'page_sets')
sp_skia_page_sets = [
os.path.join(skia_page_sets_path, 'skia_carsvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_chalkboard_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_css3gradients_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_espn_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_gmailthread_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_googlehome_desktop.py'),
os.path.join(skia_page_sets_path,
'skia_googlespreadsheet_desktop.py'),
os.path.join(skia_page_sets_path,
'skia_jsfiddlebigcar_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_mapsvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_nytimes_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_samoasvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_tigersvg_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_ugamsolutions_desktop.py'),
os.path.join(skia_page_sets_path, 'skia_digg_nexus10.py'),
os.path.join(skia_page_sets_path, 'skia_gmail_nexus10.py'),
os.path.join(skia_page_sets_path, 'skia_pravda_nexus10.py'),
]
webpages_playback_cmd.extend([
'--skp_prefix',
'sp_',
'--browser_extra_args',
'--enable-slimming-paint',
'--page_sets',
'%s' % ','.join(sp_skia_page_sets),
])
shell_utils.run(webpages_playback_cmd)
finally:
# Clean up any leftover browser instances. This can happen if there are
# telemetry crashes, processes are not always cleaned up appropriately by
# the webpagereplay and telemetry frameworks.
procs = subprocess.check_output(['ps', 'ax'])
for line in procs.splitlines():
if browser_executable in line:
pid = line.strip().split(' ')[0]
if pid != str(os.getpid()) and not 'python' in line:
try:
shell_utils.run(['kill', '-9', pid])
except shell_utils.CommandFailedException as e:
print e
else:
print 'Refusing to kill self.'
print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
with open(SKP_VERSION_FILE, 'w') as f:
f.write(str(skp_version))
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0] + '.json'
wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
if self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
page_set
)
for _ in range(RETRY_RECORD_WPR_COUNT):
output = shell_utils.run(' '.join(record_wpr_cmd), shell=True)
if RECORD_FAILURE_MSG in output:
print output
else:
# Break out of the retry loop since there were no errors.
break
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' % page_set)
else:
if not self._skip_all_gs_access:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file, page_set_basename)
page_set_name = os.path.basename(page_set).split('.')[0]
run_benchmark_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
'timeout', '300',
os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK,
page_set_name,
'-o',
'/tmp/test.skp',
'--skp-outdir=%s' % TMP_SKP_DIR
)
page_set_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets',
os.path.basename(page_set))
wpr_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data',
wpr_data_file)
json_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data',
page_set_basename)
copied_page_set = False
if not os.path.exists(page_set_dst):
print 'Copying %s to %s' % (page_set, page_set_dst)
shutil.copyfile(page_set, page_set_dst)
wpr_src = os.path.join(os.path.dirname(page_set), 'data',
wpr_data_file)
print 'Copying %s to %s' % (wpr_src, wpr_dst)
shutil.copyfile(wpr_src, wpr_dst)
json_src = os.path.join(os.path.dirname(page_set), 'data',
page_set_basename)
print 'Copying %s to %s' % (json_src, json_dst)
shutil.copyfile(json_src, json_dst)
copied_page_set = True
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_benchmark_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
if self._record:
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_basename),
self._local_record_webpages_archive_dir)
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
if copied_page_set:
os.remove(page_set_dst)
os.remove(wpr_dst)
os.remove(json_dst)
# If we get here then run_benchmark did not succeed and thus did not
# break out of the loop.
raise Exception('run_benchmark failed for page_set: %s' % page_set)
if copied_page_set:
os.remove(page_set_dst)
os.remove(wpr_dst)
os.remove(json_dst)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'),
'-r', self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'),
self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code, output) = shell_utils.log_process_after_completion(proc,
echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
os.path.join(self._local_skp_dir, '*')))
print '\n\n'
if not self._skip_all_gs_access and self._upload_to_gs:
print '\n\n=======Uploading to Google Storage=======\n\n'
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX)
gs_utils.GSUtils().upload_dir_contents(
LOCAL_PLAYBACK_ROOT_DIR, gs_bucket, dest_dir_name,
upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
predefined_acl=GS_PREDEFINED_ACL,
fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to Google Storage=======\n\n'
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
def roll_deps(config, revision):
"""Upload changed DEPS and a whitespace change.
Given the correct git_hash, create two Reitveld issues.
Args:
config: (roll_deps.DepsRollConfig) object containing options.
revision: (string) Skia Git hash.
Returns:
a tuple containing textual description of the two issues.
Raises:
OSError: failed to execute git or git-cl.
subprocess.CalledProcessError: git returned unexpected status.
"""
with misc.ChDir(config.chromium_path, verbose=config.verbose):
git_utils.Fetch()
output = shell_utils.run([git_utils.GIT, 'show', 'origin/master:DEPS'],
log_in_real_time=False).rstrip()
match = re.search(REGEXP_SKIA_REVISION, output, flags=re.MULTILINE)
old_revision = None
if match:
old_revision = match.group('revision')
assert old_revision
master_hash = git_utils.FullHash('origin/master').rstrip()
# master_hash[8] gives each whitespace CL a unique name.
branch = 'control_%s' % master_hash[:8]
message = ('whitespace change %s\n\n'
'Chromium base revision: %s\n\n'
'This CL was created by Skia\'s roll_deps.py script.\n'
) % (master_hash[:8], master_hash[:8])
with git_utils.GitBranch(branch, message,
delete_when_finished=not config.save_branches,
upload=not config.skip_cl_upload
) as whitespace_branch:
branch = git_utils.GetCurrentBranch()
with open(os.path.join('build', 'whitespace_file.txt'), 'a') as f:
f.write('\nCONTROL\n')
control_url = whitespace_branch.commit_and_upload()
if config.cl_bot_list:
submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
whitespace_cl = control_url
if config.save_branches:
whitespace_cl += '\n branch: %s' % branch
branch = 'roll_%s_%s' % (revision, master_hash[:8])
message = (
'roll skia DEPS to %s\n\n'
'Chromium base revision: %s\n'
'Old Skia revision: %s\n'
'New Skia revision: %s\n'
'Control CL: %s\n\n'
'This CL was created by Skia\'s roll_deps.py script.\n\n'
'Bypassing commit queue trybots:\n'
'NOTRY=true\n'
% (revision, master_hash[:8],
old_revision[:8], revision[:8], control_url))
with git_utils.GitBranch(branch, message,
delete_when_finished=not config.save_branches,
upload=not config.skip_cl_upload
) as roll_branch:
change_skia_deps(revision, 'DEPS')
deps_url = roll_branch.commit_and_upload()
if config.cl_bot_list:
submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
deps_cl = deps_url
if config.save_branches:
deps_cl += '\n branch: %s' % branch
return deps_cl, whitespace_cl