def submit_tries(bots_to_run, dry_run=False): """Submit try requests for the current branch on the given bots. Args: bots_to_run: (list of strings) bots to request. dry_run: (bool) whether to actually submit the try request. """ git_try = [ git_utils.GIT, 'cl', 'try', '-m', 'tryserver.chromium'] git_try.extend([arg for bot in bots_to_run for arg in ('-b', bot)]) if dry_run: space = ' ' print 'You should call:' print space, git_try print else: shell_utils.run(git_try)
def get_running_pid(master): master_hostname = master.master_fqdn.split('.')[0] pid_cmd = slave_hosts_cfg.compute_engine_login(master_hostname, None) pid_cmd.extend(['cat', PID_FILE]) try: return shell_utils.run(pid_cmd, echo=False).splitlines()[-1] except shell_utils.CommandFailedException as e: if 'No such file or directory' in e.output: return None raise
def get_running_pid(master): master_hostname = master.master_fqdn.split('.')[0] pid_cmd = slave_hosts_cfg.compute_engine_login(master_hostname, None) pid_cmd.extend(['cat', PID_FILE]) try: return shell_utils.run(pid_cmd, echo=False).splitlines()[-1] except shell_utils.CommandFailedException as e: if 'No such file or directory' in e.output: return None raise
def main(chrome_src_path, browser_executable, dry_run): browser_executable = os.path.realpath(browser_executable) dry_run = (dry_run == 'True') skp_version = _get_skp_version() print 'SKP_VERSION=%d' % skp_version if os.environ.get('CHROME_HEADLESS'): # Start Xvfb if running on a bot. try: shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True) except Exception: # It is ok if the above command fails, it just means that DISPLAY=:0 # is already up. pass upload_dir = 'playback_%d' % skp_version webpages_playback_cmd = [ 'python', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'webpages_playback.py'), '--page_sets', 'all', '--browser_executable', browser_executable, '--non-interactive', '--upload', '--alternate_upload_dir', upload_dir, '--chrome_src_path', chrome_src_path, ] if not dry_run: webpages_playback_cmd.append('--upload_to_partner_bucket') try: shell_utils.run(webpages_playback_cmd) finally: # Clean up any leftover browser instances. This can happen if there are # telemetry crashes, processes are not always cleaned up appropriately by # the webpagereplay and telemetry frameworks. procs = subprocess.check_output(['ps', 'ax']) for line in procs.splitlines(): if browser_executable in line: pid = line.strip().split(' ')[0] if pid != str(os.getpid()) and not 'python' in line: try: shell_utils.run(['kill', '-9', pid]) except shell_utils.CommandFailedException as e: print e else: print 'Refusing to kill self.' print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version) with open(SKP_VERSION_FILE, 'w') as f: f.write(str(skp_version))
def main(chrome_src_path, browser_executable, dry_run): browser_executable = os.path.realpath(browser_executable) dry_run = (dry_run == 'True') skp_version = _get_skp_version() print 'SKP_VERSION=%d' % skp_version if os.environ.get('CHROME_HEADLESS'): # Start Xvfb if running on a bot. try: shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True) except Exception: # It is ok if the above command fails, it just means that DISPLAY=:0 # is already up. pass upload_dir = 'playback_%d' % skp_version webpages_playback_cmd = [ 'python', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'webpages_playback.py'), '--page_sets', 'all', '--browser_executable', browser_executable, '--non-interactive', '--upload', '--alternate_upload_dir', upload_dir, '--chrome_src_path', chrome_src_path, ] if not dry_run: webpages_playback_cmd.append('--upload_to_partner_bucket') try: shell_utils.run(webpages_playback_cmd) finally: # Clean up any leftover browser instances. This can happen if there are # telemetry crashes, processes are not always cleaned up appropriately by # the webpagereplay and telemetry frameworks. procs = subprocess.check_output(['ps', 'ax']) for line in procs.splitlines(): if browser_executable in line: pid = line.strip().split(' ')[0] if pid != str(os.getpid()) and not 'python' in line: try: shell_utils.run(['kill', '-9', pid]) except shell_utils.CommandFailedException as e: print e else: print 'Refusing to kill self.' print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version) with open(SKP_VERSION_FILE, 'w') as f: f.write(str(skp_version))
def Run(self): """Run the SkPicturePlayback BuildStep.""" # Download the credentials file if it was not previously downloaded. if not os.path.isfile(CREDENTIALS_FILE_PATH): # Download the credentials.json file from Google Storage. self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH) if not os.path.isfile(CREDENTIALS_FILE_PATH): print """\n\nCould not locate credentials file in the storage. Please create a %s file that contains: { "google": { "username": "******", "password": "******" }, "facebook": { "username": "******", "password": "******" } }\n\n""" % CREDENTIALS_FILE_PATH raw_input("Please press a key when you are ready to proceed...") # Delete any left over data files in the data directory. for archive_file in glob.glob( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')): os.remove(archive_file) # Delete the local root directory if it already exists. if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR): shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR) # Create the required local storage directories. self._CreateLocalStorageDirs() # Start the timer. start_time = time.time() # Loop through all page_sets. for page_set in self._page_sets: page_set_basename = os.path.basename(page_set).split('.')[0] page_set_json_name = page_set_basename + '.json' wpr_data_file = page_set.split( os.path.sep)[-1].split('.')[0] + '_000.wpr' page_set_dir = os.path.dirname(page_set) if self._IsChromiumPageSet(page_set): print 'Using Chromium\'s captured archives for Chromium\'s page sets.' elif self._record: # Create an archive of the specified webpages if '--record=True' is # specified. record_wpr_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, os.path.join(self._telemetry_binaries_dir, 'record_wpr'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, '%s_page_set' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir) for _ in range(RETRY_RECORD_WPR_COUNT): try: shell_utils.run(' '.join(record_wpr_cmd), shell=True) # Move over the created archive into the local webpages archive # directory. shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), self._local_record_webpages_archive_dir) shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, page_set_json_name), self._local_record_webpages_archive_dir) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() else: # If we get here then record_wpr did not succeed and thus did not # break out of the loop. raise Exception('record_wpr failed for page_set: %s' % page_set) else: # Get the webpages archive so that it can be replayed. self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name) run_benchmark_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300', os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, SKP_BENCHMARK, '--page-set-name=%s' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir, '--skp-outdir=%s' % TMP_SKP_DIR, '--also-run-disabled-tests') for _ in range(RETRY_RUN_MEASUREMENT_COUNT): try: print '\n\n=======Capturing SKP of %s=======\n\n' % page_set shell_utils.run(' '.join(run_benchmark_cmd), shell=True) except shell_utils.CommandFailedException: # skpicture_printer sometimes fails with AssertionError but the # captured SKP is still valid. This is a known issue. pass # Rename generated SKP files into more descriptive names. try: self._RenameSkpFiles(page_set) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() print '\n\n=======Retrying %s=======\n\n' % page_set time.sleep(10) else: # If we get here then run_benchmark did not succeed and thus did not # break out of the loop. raise Exception('run_benchmark failed for page_set: %s' % page_set) print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % ( time.time() - start_time) if self._skia_tools: render_pictures_cmd = [ os.path.join(self._skia_tools, 'render_pictures'), '-r', self._local_skp_dir ] render_pdfs_cmd = [ os.path.join(self._skia_tools, 'render_pdfs'), '-r', self._local_skp_dir ] for tools_cmd in (render_pictures_cmd, render_pdfs_cmd): print '\n\n=======Running %s=======' % ' '.join(tools_cmd) proc = subprocess.Popen(tools_cmd) (code, _) = shell_utils.log_process_after_completion(proc, echo=False) if code != 0: raise Exception('%s failed!' % ' '.join(tools_cmd)) if not self._non_interactive: print '\n\n=======Running debugger=======' os.system('%s %s' % (os.path.join( self._skia_tools, 'debugger'), self._local_skp_dir)) print '\n\n' if self._upload: print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type( ) # Copy the directory structure in the root directory into Google Storage. dest_dir_name = ROOT_PLAYBACK_DIR_NAME if self._alternate_upload_dir: dest_dir_name = self._alternate_upload_dir self.gs.upload_dir_contents( LOCAL_PLAYBACK_ROOT_DIR, dest_dir=dest_dir_name, upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED, predefined_acl=GS_PREDEFINED_ACL, fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST) print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( posixpath.join(self.gs.target_name(), dest_dir_name, SKPICTURES_DIR_NAME)) else: print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type( ) print 'Generated resources are available in %s\n\n' % ( LOCAL_PLAYBACK_ROOT_DIR) return 0
def Run(self): """Run the SkPicturePlayback BuildStep.""" # Download the credentials file if it was not previously downloaded. if not os.path.isfile(CREDENTIALS_FILE_PATH): # Download the credentials.json file from Google Storage. self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH) if not os.path.isfile(CREDENTIALS_FILE_PATH): print """\n\nCould not locate credentials file in the storage. Please create a %s file that contains: { "google": { "username": "******", "password": "******" }, "facebook": { "username": "******", "password": "******" } }\n\n""" % CREDENTIALS_FILE_PATH raw_input("Please press a key when you are ready to proceed...") # Delete any left over data files in the data directory. for archive_file in glob.glob( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')): os.remove(archive_file) # Delete the local root directory if it already exists. if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR): shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR) # Create the required local storage directories. self._CreateLocalStorageDirs() # Start the timer. start_time = time.time() # Loop through all page_sets. for page_set in self._page_sets: page_set_basename = os.path.basename(page_set).split('.')[0] page_set_json_name = page_set_basename + '.json' wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' page_set_dir = os.path.dirname(page_set) if self._IsChromiumPageSet(page_set): print 'Using Chromium\'s captured archives for Chromium\'s page sets.' elif self._record: # Create an archive of the specified webpages if '--record=True' is # specified. record_wpr_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, os.path.join(self._telemetry_binaries_dir, 'record_wpr'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, '%s_page_set' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir ) for _ in range(RETRY_RECORD_WPR_COUNT): try: shell_utils.run(' '.join(record_wpr_cmd), shell=True) # Move over the created archive into the local webpages archive # directory. shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), self._local_record_webpages_archive_dir) shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, page_set_json_name), self._local_record_webpages_archive_dir) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() else: # If we get here then record_wpr did not succeed and thus did not # break out of the loop. raise Exception('record_wpr failed for page_set: %s' % page_set) else: # Get the webpages archive so that it can be replayed. self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name) run_benchmark_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300', os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, SKP_BENCHMARK, '--page-set-name=%s' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir, '--skp-outdir=%s' % TMP_SKP_DIR, '--also-run-disabled-tests' ) for _ in range(RETRY_RUN_MEASUREMENT_COUNT): try: print '\n\n=======Capturing SKP of %s=======\n\n' % page_set shell_utils.run(' '.join(run_benchmark_cmd), shell=True) except shell_utils.CommandFailedException: # skpicture_printer sometimes fails with AssertionError but the # captured SKP is still valid. This is a known issue. pass # Rename generated SKP files into more descriptive names. try: self._RenameSkpFiles(page_set) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() print '\n\n=======Retrying %s=======\n\n' % page_set time.sleep(10) else: # If we get here then run_benchmark did not succeed and thus did not # break out of the loop. raise Exception('run_benchmark failed for page_set: %s' % page_set) print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % ( time.time() - start_time) if self._skia_tools: render_pictures_cmd = [ os.path.join(self._skia_tools, 'render_pictures'), '-r', self._local_skp_dir ] render_pdfs_cmd = [ os.path.join(self._skia_tools, 'render_pdfs'), '-r', self._local_skp_dir ] for tools_cmd in (render_pictures_cmd, render_pdfs_cmd): print '\n\n=======Running %s=======' % ' '.join(tools_cmd) proc = subprocess.Popen(tools_cmd) (code, _) = shell_utils.log_process_after_completion(proc, echo=False) if code != 0: raise Exception('%s failed!' % ' '.join(tools_cmd)) if not self._non_interactive: print '\n\n=======Running debugger=======' os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'), self._local_skp_dir)) print '\n\n' if self._upload: print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type() # Copy the directory structure in the root directory into Google Storage. dest_dir_name = ROOT_PLAYBACK_DIR_NAME if self._alternate_upload_dir: dest_dir_name = self._alternate_upload_dir self.gs.upload_dir_contents( LOCAL_PLAYBACK_ROOT_DIR, dest_dir=dest_dir_name, upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED, predefined_acl=GS_PREDEFINED_ACL, fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST) print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( posixpath.join(self.gs.target_name(), dest_dir_name, SKPICTURES_DIR_NAME)) else: print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type() print 'Generated resources are available in %s\n\n' % ( LOCAL_PLAYBACK_ROOT_DIR) return 0
def main(chrome_src_path, browser_executable): browser_executable = os.path.realpath(browser_executable) skp_version = _get_skp_version() print 'SKP_VERSION=%d' % skp_version if os.environ.get('CHROME_HEADLESS'): # Start Xvfb if running on a bot. try: shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True) except Exception: # It is ok if the above command fails, it just means that DISPLAY=:0 # is already up. pass upload_dir = 'playback_%d' % skp_version webpages_playback_cmd = [ 'python', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'webpages_playback.py'), '--page_sets', 'all', '--browser_executable', browser_executable, '--non-interactive', '--upload', '--alternate_upload_dir', upload_dir, '--chrome_src_path', chrome_src_path, ] try: shell_utils.run(webpages_playback_cmd) # Temporary change to enable Slimming Paint runs. See skia:3763. skia_page_sets_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'page_sets') sp_skia_page_sets = [ os.path.join(skia_page_sets_path, 'skia_carsvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_chalkboard_desktop.py'), os.path.join(skia_page_sets_path, 'skia_css3gradients_desktop.py'), os.path.join(skia_page_sets_path, 'skia_espn_desktop.py'), os.path.join(skia_page_sets_path, 'skia_gmailthread_desktop.py'), os.path.join(skia_page_sets_path, 'skia_googlehome_desktop.py'), os.path.join(skia_page_sets_path, 'skia_googlespreadsheet_desktop.py'), os.path.join(skia_page_sets_path, 'skia_jsfiddlebigcar_desktop.py'), os.path.join(skia_page_sets_path, 'skia_mapsvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_nytimes_desktop.py'), os.path.join(skia_page_sets_path, 'skia_samoasvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_tigersvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_ugamsolutions_desktop.py'), os.path.join(skia_page_sets_path, 'skia_digg_nexus10.py'), os.path.join(skia_page_sets_path, 'skia_gmail_nexus10.py'), os.path.join(skia_page_sets_path, 'skia_pravda_nexus10.py'), ] webpages_playback_cmd.extend([ '--skp_prefix', 'sp_', '--browser_extra_args', '--enable-slimming-paint', '--page_sets', '%s' % ','.join(sp_skia_page_sets), ]) shell_utils.run(webpages_playback_cmd) finally: # Clean up any leftover browser instances. This can happen if there are # telemetry crashes, processes are not always cleaned up appropriately by # the webpagereplay and telemetry frameworks. procs = subprocess.check_output(['ps', 'ax']) for line in procs.splitlines(): if browser_executable in line: pid = line.strip().split(' ')[0] if pid != str(os.getpid()) and not 'python' in line: try: shell_utils.run(['kill', '-9', pid]) except shell_utils.CommandFailedException as e: print e else: print 'Refusing to kill self.' print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version) with open(SKP_VERSION_FILE, 'w') as f: f.write(str(skp_version))
def main(chrome_src_path, browser_executable): browser_executable = os.path.realpath(browser_executable) skp_version = _get_skp_version() print 'SKP_VERSION=%d' % skp_version if os.environ.get('CHROME_HEADLESS'): # Start Xvfb if running on a bot. try: shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True) except Exception: # It is ok if the above command fails, it just means that DISPLAY=:0 # is already up. pass upload_dir = 'playback_%d' % skp_version webpages_playback_cmd = [ 'python', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'webpages_playback.py'), '--page_sets', 'all', '--browser_executable', browser_executable, '--non-interactive', '--upload', '--alternate_upload_dir', upload_dir, '--chrome_src_path', chrome_src_path, ] try: shell_utils.run(webpages_playback_cmd) # Temporary change to enable Slimming Paint runs. See skia:3763. skia_page_sets_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'page_sets') sp_skia_page_sets = [ os.path.join(skia_page_sets_path, 'skia_carsvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_chalkboard_desktop.py'), os.path.join(skia_page_sets_path, 'skia_css3gradients_desktop.py'), os.path.join(skia_page_sets_path, 'skia_espn_desktop.py'), os.path.join(skia_page_sets_path, 'skia_gmailthread_desktop.py'), os.path.join(skia_page_sets_path, 'skia_googlehome_desktop.py'), os.path.join(skia_page_sets_path, 'skia_googlespreadsheet_desktop.py'), os.path.join(skia_page_sets_path, 'skia_jsfiddlebigcar_desktop.py'), os.path.join(skia_page_sets_path, 'skia_mapsvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_nytimes_desktop.py'), os.path.join(skia_page_sets_path, 'skia_samoasvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_tigersvg_desktop.py'), os.path.join(skia_page_sets_path, 'skia_ugamsolutions_desktop.py'), os.path.join(skia_page_sets_path, 'skia_digg_nexus10.py'), os.path.join(skia_page_sets_path, 'skia_gmail_nexus10.py'), os.path.join(skia_page_sets_path, 'skia_pravda_nexus10.py'), ] webpages_playback_cmd.extend([ '--skp_prefix', 'sp_', '--browser_extra_args', '--enable-slimming-paint', '--page_sets', '%s' % ','.join(sp_skia_page_sets), ]) shell_utils.run(webpages_playback_cmd) finally: # Clean up any leftover browser instances. This can happen if there are # telemetry crashes, processes are not always cleaned up appropriately by # the webpagereplay and telemetry frameworks. procs = subprocess.check_output(['ps', 'ax']) for line in procs.splitlines(): if browser_executable in line: pid = line.strip().split(' ')[0] if pid != str(os.getpid()) and not 'python' in line: try: shell_utils.run(['kill', '-9', pid]) except shell_utils.CommandFailedException as e: print e else: print 'Refusing to kill self.' print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version) with open(SKP_VERSION_FILE, 'w') as f: f.write(str(skp_version))
def Run(self): """Run the SkPicturePlayback BuildStep.""" # Delete any left over data files in the data directory. for archive_file in glob.glob( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')): os.remove(archive_file) # Delete the local root directory if it already exists. if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR): shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR) # Create the required local storage directories. self._CreateLocalStorageDirs() # Start the timer. start_time = time.time() # Loop through all page_sets. for page_set in self._page_sets: page_set_basename = os.path.basename(page_set).split('.')[0] + '.json' wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' if self._record: # Create an archive of the specified webpages if '--record=True' is # specified. record_wpr_cmd = ( 'DISPLAY=%s' % X11_DISPLAY, os.path.join(self._telemetry_binaries_dir, 'record_wpr'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, page_set ) for _ in range(RETRY_RECORD_WPR_COUNT): output = shell_utils.run(' '.join(record_wpr_cmd), shell=True) if RECORD_FAILURE_MSG in output: print output else: # Break out of the retry loop since there were no errors. break else: # If we get here then record_wpr did not succeed and thus did not # break out of the loop. raise Exception('record_wpr failed for page_set: %s' % page_set) else: if not self._skip_all_gs_access: # Get the webpages archive so that it can be replayed. self._DownloadWebpagesArchive(wpr_data_file, page_set_basename) page_set_name = os.path.basename(page_set).split('.')[0] run_benchmark_cmd = ( 'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300', os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, SKP_BENCHMARK, page_set_name, '-o', '/tmp/test.skp', '--skp-outdir=%s' % TMP_SKP_DIR ) page_set_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', os.path.basename(page_set)) wpr_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data', wpr_data_file) json_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data', page_set_basename) copied_page_set = False if not os.path.exists(page_set_dst): print 'Copying %s to %s' % (page_set, page_set_dst) shutil.copyfile(page_set, page_set_dst) wpr_src = os.path.join(os.path.dirname(page_set), 'data', wpr_data_file) print 'Copying %s to %s' % (wpr_src, wpr_dst) shutil.copyfile(wpr_src, wpr_dst) json_src = os.path.join(os.path.dirname(page_set), 'data', page_set_basename) print 'Copying %s to %s' % (json_src, json_dst) shutil.copyfile(json_src, json_dst) copied_page_set = True for _ in range(RETRY_RUN_MEASUREMENT_COUNT): try: print '\n\n=======Capturing SKP of %s=======\n\n' % page_set shell_utils.run(' '.join(run_benchmark_cmd), shell=True) except shell_utils.CommandFailedException: # skpicture_printer sometimes fails with AssertionError but the # captured SKP is still valid. This is a known issue. pass if self._record: # Move over the created archive into the local webpages archive # directory. shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), self._local_record_webpages_archive_dir) shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, page_set_basename), self._local_record_webpages_archive_dir) # Rename generated SKP files into more descriptive names. try: self._RenameSkpFiles(page_set) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() print '\n\n=======Retrying %s=======\n\n' % page_set time.sleep(10) else: if copied_page_set: os.remove(page_set_dst) os.remove(wpr_dst) os.remove(json_dst) # If we get here then run_benchmark did not succeed and thus did not # break out of the loop. raise Exception('run_benchmark failed for page_set: %s' % page_set) if copied_page_set: os.remove(page_set_dst) os.remove(wpr_dst) os.remove(json_dst) print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % ( time.time() - start_time) if self._skia_tools: render_pictures_cmd = [ os.path.join(self._skia_tools, 'render_pictures'), '-r', self._local_skp_dir ] render_pdfs_cmd = [ os.path.join(self._skia_tools, 'render_pdfs'), self._local_skp_dir ] for tools_cmd in (render_pictures_cmd, render_pdfs_cmd): print '\n\n=======Running %s=======' % ' '.join(tools_cmd) proc = subprocess.Popen(tools_cmd) (code, output) = shell_utils.log_process_after_completion(proc, echo=False) if code != 0: raise Exception('%s failed!' % ' '.join(tools_cmd)) if not self._non_interactive: print '\n\n=======Running debugger=======' os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'), os.path.join(self._local_skp_dir, '*'))) print '\n\n' if not self._skip_all_gs_access and self._upload_to_gs: print '\n\n=======Uploading to Google Storage=======\n\n' # Copy the directory structure in the root directory into Google Storage. dest_dir_name = ROOT_PLAYBACK_DIR_NAME if self._alternate_upload_dir: dest_dir_name = self._alternate_upload_dir gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) gs_utils.GSUtils().upload_dir_contents( LOCAL_PLAYBACK_ROOT_DIR, gs_bucket, dest_dir_name, upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED, predefined_acl=GS_PREDEFINED_ACL, fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST) print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) else: print '\n\n=======Not Uploading to Google Storage=======\n\n' print 'Generated resources are available in %s\n\n' % ( LOCAL_PLAYBACK_ROOT_DIR) return 0
def roll_deps(config, revision): """Upload changed DEPS and a whitespace change. Given the correct git_hash, create two Reitveld issues. Args: config: (roll_deps.DepsRollConfig) object containing options. revision: (string) Skia Git hash. Returns: a tuple containing textual description of the two issues. Raises: OSError: failed to execute git or git-cl. subprocess.CalledProcessError: git returned unexpected status. """ with misc.ChDir(config.chromium_path, verbose=config.verbose): git_utils.Fetch() output = shell_utils.run([git_utils.GIT, 'show', 'origin/master:DEPS'], log_in_real_time=False).rstrip() match = re.search(REGEXP_SKIA_REVISION, output, flags=re.MULTILINE) old_revision = None if match: old_revision = match.group('revision') assert old_revision master_hash = git_utils.FullHash('origin/master').rstrip() # master_hash[8] gives each whitespace CL a unique name. branch = 'control_%s' % master_hash[:8] message = ('whitespace change %s\n\n' 'Chromium base revision: %s\n\n' 'This CL was created by Skia\'s roll_deps.py script.\n' ) % (master_hash[:8], master_hash[:8]) with git_utils.GitBranch(branch, message, delete_when_finished=not config.save_branches, upload=not config.skip_cl_upload ) as whitespace_branch: branch = git_utils.GetCurrentBranch() with open(os.path.join('build', 'whitespace_file.txt'), 'a') as f: f.write('\nCONTROL\n') control_url = whitespace_branch.commit_and_upload() if config.cl_bot_list: submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload) whitespace_cl = control_url if config.save_branches: whitespace_cl += '\n branch: %s' % branch branch = 'roll_%s_%s' % (revision, master_hash[:8]) message = ( 'roll skia DEPS to %s\n\n' 'Chromium base revision: %s\n' 'Old Skia revision: %s\n' 'New Skia revision: %s\n' 'Control CL: %s\n\n' 'This CL was created by Skia\'s roll_deps.py script.\n\n' 'Bypassing commit queue trybots:\n' 'NOTRY=true\n' % (revision, master_hash[:8], old_revision[:8], revision[:8], control_url)) with git_utils.GitBranch(branch, message, delete_when_finished=not config.save_branches, upload=not config.skip_cl_upload ) as roll_branch: change_skia_deps(revision, 'DEPS') deps_url = roll_branch.commit_and_upload() if config.cl_bot_list: submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload) deps_cl = deps_url if config.save_branches: deps_cl += '\n branch: %s' % branch return deps_cl, whitespace_cl