def main(): work_dir = os.path.abspath('.') print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL file_name = NACL_SDK_UPDATE_URL.split('/')[-1] response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True) print 'Downloading: %s' % file_name Retrieve(response, file_name) print 'Unzipping %s into %s' % (file_name, work_dir) chromium_utils.ExtractZip(file_name, work_dir, verbose=True) result = chromium_utils.RunCommand([NACL_TOOL, 'update', '--force']) if os.path.exists(CURRENT_PEPPER_BUNDLE): print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE shutil.rmtree(CURRENT_PEPPER_BUNDLE) def PepperDirs(): for x in os.listdir('nacl_sdk'): if re.match('pepper_\d+', x): yield x pepper_rev = max([int(i.split('_')[1]) for i in PepperDirs()]) pepper_rev_dir = os.path.join('nacl_sdk', 'pepper_' + str(pepper_rev)) print 'Copying pepper bundle %d to current' % pepper_rev shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True) return result
def main(): option_parser = optparse.OptionParser() option_parser.add_option('--pepper-channel', default='stable', help='Pepper channel (stable|beta|canary)') options, _ = option_parser.parse_args() work_dir = os.path.abspath('.') print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL file_name = NACL_SDK_UPDATE_URL.split('/')[-1] response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True) file_hash = None if os.path.exists(file_name): file_hash = GetFileHash(file_name) print 'Downloading: %s' % file_name Retrieve(response, file_name) # Only extract if file changed. Extraction overwrites the sdk tools and the # state about which pepper revisions are up to date. if file_hash != GetFileHash(file_name): print 'Unzipping %s into %s' % (file_name, work_dir) chromium_utils.ExtractZip(file_name, work_dir, verbose=True) else: print 'Existing %s is up to date.' % file_name print 'Listing available pepper bundles:' output = chromium_utils.GetCommandOutput([NACL_TOOL, 'list']) print output pepper_rev = GetRevisionName(output, options.pepper_channel) print 'Updating pepper bundle %s' % pepper_rev cmd = [NACL_TOOL, 'update', pepper_rev, '--force'] result = chromium_utils.RunCommand(cmd) if os.path.exists(CURRENT_PEPPER_BUNDLE): print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE shutil.rmtree(CURRENT_PEPPER_BUNDLE) pepper_rev_dir = os.path.join('nacl_sdk', pepper_rev) print 'Copying pepper bundle %s to current' % pepper_rev shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True) return result
def unpack(options): if not options.url and not options.builder: raise StagingError('Either a test url or builder name is required.') # Remove existing test archive. if os.path.exists(AUTO_PATH): print 'Removing existing directory at', AUTO_PATH shutil.rmtree(AUTO_PATH) os.mkdir(AUTO_PATH) # Download archive. if options.url: url = options.url else: # Find latest archive. branch = 'o3d' latest_path = branch + '/latest_' + options.builder latest_url = ARCHIVE_BASE_URL + latest_path local_latest = os.path.join(AUTO_PATH, 'latest') print 'Downloading latest file from', latest_url urllib.urlretrieve(latest_url, local_latest) latest_file = file(local_latest, 'r') url = ARCHIVE_BASE_URL + branch + '/' + latest_file.readline() try: local_archive_path = DownloadTestArchive(url) except IOError: print 'IOError while downloading test archive from', url return 2 # Unzip archive. output_dir = os.path.normpath(os.path.join(O3D_PATH, '..')) print 'Extracting test archive to', output_dir utils.ExtractZip(local_archive_path, output_dir, False) # Copy archive's automation scripts into auto directory. print 'Copying automation scripts from', O3D_SRC_AUTO_PATH, 'to', SCRIPTS_PATH shutil.copytree(O3D_SRC_AUTO_PATH, SCRIPTS_PATH) return 0
def _ArchiveFullLayoutTestResults(staging_dir, dest_dir, diff_file_list, options): # Copy the actual and diff files to the web server. # Don't clobber the staging_dir in the MakeZip call so that it keeps the # files from the previous MakeZip call on diff_file_list. print "archiving results + diffs" full_zip_file = chromium_utils.MakeZip(staging_dir, 'layout-test-results', diff_file_list, options.results_dir, remove_archive_directory=False)[1] slave_utils.CopyFileToArchiveHost(full_zip_file, dest_dir) # Extract the files on the web server. extract_dir = os.path.join(dest_dir, 'results') print 'extracting zip file to %s' % extract_dir if chromium_utils.IsWindows(): chromium_utils.ExtractZip(full_zip_file, extract_dir) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): remote_zip_file = os.path.join(dest_dir, os.path.basename(full_zip_file)) chromium_utils.SshExtractZip(config.Archive.archive_host, remote_zip_file, extract_dir)
def real_main(options): """ Download a build, extract it to build\BuildDir\full-build-win32 and rename it to build\BuildDir\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision base_url, url = GetBuildUrl(options, build_revision, webkit_revision) archive_name = os.path.basename(base_url) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # Check if the url exists. if not handler.is_present(): print '%s is not found' % url failure = True # When 'halt_on_missing_build' is present in factory_properties and if # 'revision' is set in build properties, we assume the build is # triggered automatically and so we halt on a missing build zip. The # other case is if the build is forced, in which case we keep trying # later by looking for the latest build that's available. if (options.factory_properties.get('halt_on_missing_build', False) and 'revision' in options.build_properties and options.build_properties['revision'] != ''): return slave_utils.ERROR_EXIT_CODE # If the url is valid, we download the file. if not failure: if not handler.download(): failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://'): continue else: print 'Fetching latest build at %s' % base_url handler.url = base_url if not handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read( ) build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def real_main(options): """ Download a build, extract it to build\\BuildDir\\full-build-win32 and rename it to build\\BuildDir\\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision and not options.build_archive_url: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision url, archive_name = GetBuildUrl(options, build_revision, webkit_revision) if archive_name is None: archive_name = 'build.zip' base_url = None else: base_url = '/'.join(url.split('/')[:-1] + [archive_name]) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # If the url is valid, we download the file. if not failure: if not handler.download(): if options.halt_on_missing_build: return slave_utils.ERROR_EXIT_CODE failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://') or not base_url: continue else: print 'Fetching latest build at %s' % base_url base_handler = handler.__class__(base_url, handler.archive_name) if not base_handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read() build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def real_main(options): """Download a build and extract. Extract to build\\BuildDir\\full-build-win32 and rename it to build\\BuildDir\\Target """ target_build_output_dir = os.path.join(options.build_dir, options.target) platform = chromium_utils.PlatformName() revision = options.revision if not revision: revision = GetLatestRevision(options.build_url, platform) if not revision: print 'Failed to get revision number.' return slave_utils.ERROR_EXIT_CODE archive_url = GetBuildUrl(options.build_url, platform, revision) archive_name = 'dynamorio.' + os.path.basename(archive_url).split('.', 1)[1] temp_dir = tempfile.mkdtemp() try: # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s' % (tries, archive_url) failure = False try: print '%s/%s' % (archive_url, archive_name) urllib.urlretrieve(archive_url, archive_name) print '\nDownload complete' except IOError: print '\nFailed to download build' failure = True if options.halt_on_missing_build: return slave_utils.ERROR_EXIT_CODE if failure: continue print 'Extracting build %s to %s...' % (archive_name, options.build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, temp_dir) # Look for the top level directory from extracted build. entries = os.listdir(temp_dir) output_dir = temp_dir if (len(entries) == 1 and os.path.isdir( os.path.join(output_dir, entries[0]))): output_dir = os.path.join(output_dir, entries[0]) print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... time.sleep(3) continue return 0 finally: chromium_utils.RemoveDirectory(temp_dir) # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE