def _ArchiveGoodBuild(platform, commit_position): """Archive chromedriver binary if the build is green.""" assert platform != 'android' util.MarkBuildStepStart('archive build') server_name = 'chromedriver' if util.IsWindows(): server_name += '.exe' zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]), server_name)) build_name = 'chromedriver_%s_%s.%s.zip' % ( platform, _GetVersion(), commit_position) build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name) if slave_utils.GSUtilCopy(zip_path, build_url): util.MarkBuildStepError() if util.IsWindows(): zip_path = util.Zip(os.path.join( chrome_paths.GetBuildDir([server_name + '.pdb']), server_name + '.pdb')) pdb_name = 'chromedriver_%s_pdb_%s.%s.zip' % ( platform, _GetVersion(), commit_position) pdb_url = '%s/%s' % (GS_CONTINUOUS_URL, pdb_name) if slave_utils.GSUtilCopy(zip_path, pdb_url): util.MarkBuildStepError() (latest_fd, latest_file) = tempfile.mkstemp() os.write(latest_fd, build_name) os.close(latest_fd) latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform) if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'): util.MarkBuildStepError() os.remove(latest_file)
def _MaybeUpdateLatestRelease(version): """Update the file LATEST_RELEASE with the latest release version number.""" latest_release_fname = 'LATEST_RELEASE' latest_release_url = '%s/%s' % (GS_CHROMEDRIVER_BUCKET, latest_release_fname) # Check if LATEST_RELEASE is up to date. latest_released_version = _GetWebPageContent( '%s/%s' % (GS_CHROMEDRIVER_RELEASE_URL, latest_release_fname)) if version == latest_released_version: return # Check if chromedriver was released on all supported platforms. supported_platforms = ['linux64', 'mac32', 'win32'] for platform in supported_platforms: if not _WasReleased(version, platform): return util.MarkBuildStepStart('updating LATEST_RELEASE to %s' % version) temp_latest_release_fname = tempfile.mkstemp()[1] with open(temp_latest_release_fname, 'w') as f: f.write(version) if slave_utils.GSUtilCopy(temp_latest_release_fname, latest_release_url, mimetype='text/plain'): util.MarkBuildStepError()
def UploadToGomaLogGS(file_path, gs_filename, text_to_append=None): """Upload a file to Google Cloud Storage (gs://chrome-goma-log). Note that the uploaded file would automatically be gzip compressed. Args: file_path: a path of a file to be uploaded. gs_filename: a name of a file in Google Storage. text_to_append: an addtional text to be added to a file in GS. Returns: a stored path name without the bucket name in GS. """ hostname = GetShortHostname() today = datetime.datetime.utcnow().date() log_path = '%s/%s/%s.gz' % ( today.strftime('%Y/%m/%d'), hostname, gs_filename) gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path) temp = tempfile.NamedTemporaryFile(delete=False) try: with temp as f_out: with gzip.GzipFile(fileobj=f_out) as gzipf_out: with open(file_path) as f_in: shutil.copyfileobj(f_in, gzipf_out) if text_to_append: gzipf_out.write(text_to_append) slave_utils.GSUtilCopy(temp.name, gs_path) print "Copied log file to %s" % gs_path finally: os.remove(temp.name) return log_path
def _Release(build, platform): """Releases the given candidate build.""" release_name = 'chromedriver_%s.zip' % platform util.MarkBuildStepStart('releasing %s' % release_name) slave_utils.GSUtilCopy( build, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, _GetVersion(), release_name)) _MaybeUploadReleaseNotes()
def _ArchivePrebuilts(commit_position): """Uploads the prebuilts to google storage.""" util.MarkBuildStepStart('archive prebuilts') zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']), 'chromedriver')) if slave_utils.GSUtilCopy( zip_path, '%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % commit_position)): util.MarkBuildStepError()
def _Release(build, version, platform): """Releases the given candidate build.""" release_name = 'chromedriver_%s.zip' % platform util.MarkBuildStepStart('releasing %s' % release_name) temp_dir = util.MakeTempDir() slave_utils.GSUtilCopy(build, temp_dir) zip_path = os.path.join(temp_dir, os.path.basename(build)) if util.IsLinux(): util.Unzip(zip_path, temp_dir) server_path = os.path.join(temp_dir, 'chromedriver') util.RunCommand(['strip', server_path]) zip_path = util.Zip(server_path) slave_utils.GSUtilCopy( zip_path, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, release_name)) _MaybeUpdateLatestRelease(version)
def _ArchiveServerLogs(): """Uploads chromedriver server logs to google storage.""" util.MarkBuildStepStart('archive chromedriver server logs') for server_log in glob.glob( os.path.join(tempfile.gettempdir(), 'chromedriver_*')): base_name = os.path.basename(server_log) util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name)) slave_utils.GSUtilCopy(server_log, '%s/%s' % (GS_SERVER_LOGS_URL, base_name), mimetype='text/plain')
def download(self): status = slave_utils.GSUtilCopy(self.url, '.') if 0 != status: return False try: shutil.move(os.path.basename(self.url), self.archive_name) except OSError: os.remove(self.archive_name) shutil.move(os.path.basename(self.url), self.archive_name) return True
def CopyToGoogleStorage(src, dst): """Copies a file to the given Google Storage destination url. Args: src: path to file to be copied dst: Google Storage destination url (i.e., gs://...) Returns: whether the copy was successful """ if not os.path.exists(src): print 'No such file', src return False return slave_utils.GSUtilCopy(src, dst, None, 'public-read')
def _ArchiveGoodBuild(platform, revision): assert platform != 'android' util.MarkBuildStepStart('archive build') server_name = 'chromedriver' if util.IsWindows(): server_name += '.exe' zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]), server_name)) build_url = '%s/chromedriver_%s_%s.%s.zip' % ( GS_CONTINUOUS_URL, platform, _GetVersion(), revision) if slave_utils.GSUtilCopy(zip_path, build_url): util.MarkBuildStepError()
def download(self): override_gsutil = None if self.gsutil_py_path: override_gsutil = [sys.executable, self.gsutil_py_path] status = slave_utils.GSUtilCopy( self.url, '.', override_gsutil=override_gsutil) if 0 != status: return False try: shutil.move(os.path.basename(self.url), self.archive_name) except OSError: os.remove(self.archive_name) shutil.move(os.path.basename(self.url), self.archive_name) return True
def _MaybeUploadReleaseNotes(version): """Upload release notes if conditions are right.""" # Check if the current version has already been released. notes_name = 'notes.txt' notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name) prev_version = '.'.join( [version.split('.')[0], str(int(version.split('.')[1]) - 1)]) prev_notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, prev_version, notes_name) result, _ = slave_utils.GSUtilListBucket(notes_url, []) if result == 0: return fixed_issues = [] query = ('https://code.google.com/p/chromedriver/issues/csv?' 'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' % version) issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1]) for issue in csv.reader(issues): if not issue: continue issue_id = issue[0] desc = issue[1] labels = issue[2].split(', ') labels.remove('ChromeDriver-%s' % version) if 'Hotlist-GoodFirstBug' in labels: labels.remove('Hotlist-GoodFirstBug') fixed_issues += [ 'Resolved issue %s: %s [%s]' % (issue_id, desc, labels) ] old_notes = '' temp_notes_fname = tempfile.mkstemp()[1] if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname): with open(temp_notes_fname, 'rb') as f: old_notes = f.read() new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % ( version, datetime.date.today().isoformat(), 'Supports Chrome v%s-%s' % _GetSupportedChromeVersions(), '\n'.join(fixed_issues), old_notes) with open(temp_notes_fname, 'w') as f: f.write(new_notes) if slave_utils.GSUtilCopy(temp_notes_fname, notes_url, mimetype='text/plain'): util.MarkBuildStepError()
def UploadGomaCompilerProxyInfo(): """Upload goma compiler_proxy.INFO to Google Storage.""" latest_info = GetLatestGomaCompilerProxyInfo() today = datetime.datetime.utcnow().date() hostname = GetShortHostname() # Since a filename of compiler_proxy.INFO is fairly unique, # we might be able to upload it as-is. goma_log_gs_path = ( 'gs://chrome-goma-log/%s/%s/%s.gz' % (today.strftime('%Y/%m/%d'), hostname, os.path.basename(latest_info))) try: fd, output_filename = tempfile.mkstemp() with open(latest_info) as f_in: with os.fdopen(fd, 'w') as f_out: with gzip.GzipFile(fileobj=f_out, compresslevel=9) as gzipf: gzipf.writelines(f_in) slave_utils.GSUtilCopy(output_filename, goma_log_gs_path) print "Copied log file to %s" % goma_log_gs_path finally: os.remove(output_filename)
ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % ( hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid) today = datetime.datetime.utcnow().date() ninja_log_gs_path = ( 'gs://chrome-goma-log/%s/%s/%s.gz' % (today.strftime('%Y/%m/%d'), hostname, ninja_log_filename)) try: fd, output_filename = tempfile.mkstemp() with open(ninja_log_path) as f_in: with os.fdopen(fd, 'w') as f_out: with gzip.GzipFile(fileobj=f_out, compresslevel=9) as gzipf: gzipf.writelines(f_in) gzipf.write('# end of ninja log\n') gzipf.write(json.dumps(info)) slave_utils.GSUtilCopy(output_filename, ninja_log_gs_path) print "Copied log file to %s" % ninja_log_gs_path finally: os.remove(output_filename) def common_xcode_settings(command, options, env, compiler=None): """ Sets desirable Mac environment variables and command-line options that are common to the Xcode builds. """ compiler = options.compiler assert compiler in (None, 'clang', 'goma', 'goma-clang') if compiler == 'goma': print 'using goma'