def archive(options, args): build_dir, _ = chromium_utils.ConvertBuildDirToLegacy( options.build_dir, use_out=chromium_utils.IsLinux()) build_dir = os.path.join(build_dir, options.target) src_dir = os.path.abspath(os.path.dirname(options.build_dir)) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir = None # TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not # ready with the server-side change. if chromium_utils.IsMac(): subdir = '%s-%s' % (chromium_utils.PlatformName(), options.target.lower()) prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def UploadGomaCompilerProxyInfo(override_gsutil=None, builder='unknown', master='unknown', slave='unknown', clobber=''): """Upload compiler_proxy{,-subproc}.INFO and gomacc.INFO to Google Storage. Args: override_gsutil: gsutil path to override. builder: a string name of a builder. master: a string name of a master. slave: a string name of a slave. clobber: set something if clobber (to be removed) """ latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo() builderinfo = { 'builder': builder, 'master': master, 'slave': slave, 'clobber': True if clobber else False, 'os': chromium_utils.PlatformName(), } # Needs to begin with x-goog-meta for custom metadata. # https://cloud.google.com/storage/docs/gsutil/addlhelp/WorkingWithObjectMetadata#custom-metadata metadata = {'x-goog-meta-builderinfo': json.dumps(builderinfo)} if latest_subproc_info: UploadToGomaLogGS(latest_subproc_info, os.path.basename(latest_subproc_info), metadata=metadata, override_gsutil=override_gsutil) else: print 'No compiler_proxy-subproc.INFO to upload' latest_info = GetLatestGomaCompilerProxyInfo() if not latest_info: print 'No compiler_proxy.INFO to upload' return # Since a filename of compiler_proxy.INFO is fairly unique, # we might be able to upload it as-is. log_path = UploadToGomaLogGS(latest_info, os.path.basename(latest_info), metadata=metadata, override_gsutil=override_gsutil) viewer_url = ( 'http://chromium-build-stats.appspot.com/compiler_proxy_log/' + log_path) print 'Visualization at %s' % viewer_url gomacc_logs = GetListOfGomaccInfoAfterCompilerProxyStart() if gomacc_logs: for log in gomacc_logs: UploadToGomaLogGS(log, os.path.basename(log), metadata=metadata, override_gsutil=override_gsutil) return viewer_url
def main(argv): o3d_dir = os.path.join(os.getcwd(), 'o3d') staging_dir = slave_utils.GetStagingDir(o3d_dir) # Find builder name and revision #s. builder_name = slave_utils.SlaveBuildName(o3d_dir) o3d_rev = str(slave_utils.SubversionRevision(o3d_dir)) platform = chromium_utils.PlatformName() # Upload zip. local_zip = os.path.join(staging_dir, 'full-build-' + platform + '_' + o3d_rev + '.zip') remote_zip = 'snapshots/o3d/' + o3d_rev + '/' + builder_name + '.zip' archive_file.UploadFile(local_zip, remote_zip) return 0
def MakeGomaExitStatusCounter(goma_stats_file, goma_crash_report, builder='unknown', master='unknown', slave='unknown', clobber=''): """Make Goma exit status counter. This counter indicates compiler_proxy has finished without problem, crashed, or killed. This counter will be used to alert to goma team. Args: goma_stats_file: path to goma stats file if any goma_crash_report: path to goma crash report file if any builder: builder name master: master name slave: slave name clobber: non false if clobber build """ try: counter = { 'name': 'goma/status', 'value': 1, 'builder': builder, 'master': master, 'slave': slave, 'clobber': 1 if clobber else 0, 'os': chromium_utils.PlatformName(), } if goma_stats_file and os.path.exists(goma_stats_file): counter['status'] = 'success' elif goma_crash_report and os.path.exists(goma_crash_report): counter['status'] = 'crashed' elif IsCompilerProxyKilledByFatalError(): counter['status'] = 'killed' else: counter['status'] = 'unknown' start_time = GetCompilerProxyStartTime() if start_time: counter['start_time'] = int(time.mktime(start_time.timetuple())) return counter except Exception as ex: print('error while generating status counter: %s' % ex) return None
def archive(options, args): src_dir = os.path.abspath(os.path.dirname(options.build_dir)) build_dir = os.path.join(src_dir, 'out', options.target) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] prefix = options.factory_properties.get('asan_archive_name', 'asan') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def GetZipFileNames(build_properties, src_dir, webkit_dir=None, extract=False): base_name = 'full-build-%s' % chromium_utils.PlatformName() chromium_revision = GetHashOrRevision(src_dir) if 'try' in build_properties.get('mastername', ''): if extract: if not build_properties.get('parent_buildnumber'): raise Exception('build_props does not have parent data: %s' % build_properties) version_suffix = '_%(parent_buildnumber)s' % build_properties else: version_suffix = '_%(buildnumber)s' % build_properties elif webkit_dir: webkit_revision = SubversionRevision(webkit_dir) version_suffix = '_wk%d_%s' % (webkit_revision, chromium_revision) else: version_suffix = '_%s' % chromium_revision return base_name, version_suffix
def MakeUnversionedArchive(build_dir, staging_dir, zip_file_list): """Creates an unversioned full build archive. Returns the path of the created archive.""" zip_file_name = 'full-build-%s' % chromium_utils.PlatformName() (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big and # can cause bot failures from timeouts during downloads to testers. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size return zip_file
def GetZipFileNames(build_properties, build_revision, webkit_revision=None, extract=False, use_try_buildnumber=True): base_name = 'full-build-%s' % chromium_utils.PlatformName() if 'try' in build_properties.get('mastername', '') and use_try_buildnumber: if extract: if not build_properties.get('parent_buildnumber'): raise Exception('build_props does not have parent data: %s' % build_properties) version_suffix = '_%(parent_buildnumber)s' % build_properties else: version_suffix = '_%(buildnumber)s' % build_properties elif webkit_revision: version_suffix = '_wk%s_%s' % (webkit_revision, build_revision) else: version_suffix = '_%s' % build_revision return base_name, version_suffix
def GetZipFileNames(mastername, buildnumber, parent_buildnumber, build_revision, webkit_revision=None, extract=False, use_try_buildnumber=True): base_name = 'full-build-%s' % chromium_utils.PlatformName() if 'try' in mastername and use_try_buildnumber: if extract: if not parent_buildnumber: raise Exception('missing parent_buildnumber') version_suffix = '_%s' % parent_buildnumber else: version_suffix = '_%s' % buildnumber elif webkit_revision: version_suffix = '_wk%s_%s' % (webkit_revision, build_revision) else: version_suffix = '_%s' % build_revision return base_name, version_suffix
def GetBuildUrl(abs_build_dir, options): """Compute the url to download the build from. This will use as a base string, in order of preference: 1) options.build_url 2) options.factory_properties.build_url 3) build url constructed from build_properties Args: abs_build_dir: Full path to source directory. options: options object as specified by parser below. """ url = options.build_url or options.factory_properties.get('build_url') if not url: url = 'http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % ( options.build_properties.parent_slavename, options.build_properties.parent_builddir, chromium_utils.PlatformName()) if 'parentslavename' in url: parentslavename = options.build_properties.get('parentslavename', '') url = url % {'parentslavename': parentslavename} base_url = url versioned_url = url if options.webkit_dir: webkit_revision = slave_utils.SubversionRevision( os.path.join(abs_build_dir, '..', options.webkit_dir)) versioned_url = versioned_url.replace('.zip', '_wk%d.zip' % webkit_revision) # Find the revision that we need to download. chromium_revision = slave_utils.SubversionRevision(abs_build_dir) versioned_url = versioned_url.replace('.zip', '_%d.zip' % chromium_revision) return base_url, versioned_url
def archive(options, args): # Disable 'unused argument' warning for 'args' | pylint: disable=W0613 build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') primary_project = chromium_utils.GetPrimaryProject(options) build_sortkey_branch, build_sortkey_value = GetBuildSortKey( options, primary_project) build_git_commit = GetGitCommit(options, primary_project) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') sortkey_path = chromium_utils.GetSortableUploadPathForSortKey( build_sortkey_branch, build_sortkey_value) zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, sortkey_path) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) gs_metadata = { GS_COMMIT_POSITION_NUMBER_KEY: build_sortkey_value, } if build_sortkey_branch: gs_metadata[ GS_COMMIT_POSITION_KEY] = chromium_utils.BuildCommitPosition( build_sortkey_branch, build_sortkey_value) if build_git_commit: gs_metadata[GS_GIT_COMMIT_KEY] = build_git_commit status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl, metadata=gs_metadata) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def UploadTests(self, www_dir, gs_base, gs_acl): test_file_list = self._test_files if not test_file_list: return # Make test_file_list contain absolute paths. test_file_list = [ os.path.join(self._build_dir, f) for f in test_file_list ] UPLOAD_DIR = 'chrome-%s.test' % chromium_utils.PlatformName() # Filter out those files that don't exist. base_src_dir = os.path.join(self._build_dir, '') for test_file in test_file_list[:]: if os.path.exists(test_file): relative_dir = os.path.dirname(test_file[len(base_src_dir):]) test_dir = os.path.join(www_dir, UPLOAD_DIR, relative_dir) print 'chromium_utils.CopyFileToDir(%s, %s)' % (test_file, test_dir) else: print '%s does not exist and is skipped.' % test_file test_file_list.remove(test_file) # Extract the list of test paths that will be created. These paths need # to be relative to the archive dir. We have to rebuild the relative # list from the now-pruned absolute test_file_list. relative_file_list = [tf[len(base_src_dir):] for tf in test_file_list] test_dirs = archive_utils.ExtractDirsFromPaths(relative_file_list) test_dirs = [os.path.join(www_dir, UPLOAD_DIR, d) for d in test_dirs] root_test_dir = os.path.join(www_dir, UPLOAD_DIR) print 'chromium_utils.MaybeMakeDirectory(%s)' % root_test_dir for test_dir in test_dirs: print 'chromium_utils.MaybeMakeDirectory(%s)' % test_dir if not self.options.dry_run: if chromium_utils.IsWindows(): # Use Samba on Windows. self.MyMaybeMakeDirectory(root_test_dir, gs_base) for test_dir in test_dirs: self.MyMaybeMakeDirectory(test_dir, gs_base) for test_file in test_file_list: # TODO(robertshield): binaries and symbols are stored in a zip file # via CreateArchiveFile. Tests should be too. relative_dir = os.path.dirname( test_file[len(base_src_dir):]) test_dir = os.path.join(www_dir, UPLOAD_DIR, relative_dir) self.MyCopyFileToDir(test_file, test_dir, gs_base, gs_subdir='/'.join( [UPLOAD_DIR, relative_dir]), gs_acl=gs_acl) else: # Otherwise use scp. self.MySshMakeDirectory(self.options.archive_host, root_test_dir, gs_base) for test_dir in test_dirs: self.MySshMakeDirectory(self.options.archive_host, test_dir, gs_base) for test_file in test_file_list: self.MyMakeWorldReadable(test_file, gs_base) # TODO(robertshield): binaries and symbols are stored in a zip file # via CreateArchiveFile. Tests should be too. relative_dir = os.path.dirname( test_file[len(base_src_dir):]) test_dir = os.path.join(www_dir, UPLOAD_DIR, relative_dir) self.MySshCopyFiles(test_file, self.options.archive_host, test_dir, gs_base, gs_subdir='/'.join( [UPLOAD_DIR, relative_dir]), gs_acl=gs_acl)
def real_main(options): """ Download a build, extract it to build\BuildDir\full-build-win32 and rename it to build\BuildDir\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision base_url, url = GetBuildUrl(options, build_revision, webkit_revision) archive_name = os.path.basename(base_url) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # Check if the url exists. if not handler.is_present(): print '%s is not found' % url failure = True # When 'halt_on_missing_build' is present in factory_properties and if # 'revision' is set in build properties, we assume the build is # triggered automatically and so we halt on a missing build zip. The # other case is if the build is forced, in which case we keep trying # later by looking for the latest build that's available. if (options.factory_properties.get('halt_on_missing_build', False) and 'revision' in options.build_properties and options.build_properties['revision'] != ''): return slave_utils.ERROR_EXIT_CODE # If the url is valid, we download the file. if not failure: if not handler.download(): failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://'): continue else: print 'Fetching latest build at %s' % base_url handler.url = base_url if not handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read( ) build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def SendGomaTsMon(json_file, exit_status): """Send latest Goma status to ts_mon. Args: json_file: json filename string that has goma_ctl.py jsonstatus. exit_status: integer exit status of the build. """ json_statuses = {} try: with open(json_file) as f: json_statuses = json.load(f) if not json_statuses: print('no json status is recorded in %s' % json_file) return if len(json_statuses.get('notice', [])) != 1: print('unknown json statuses style: %s' % json_statuses) return json_status = json_statuses['notice'][0] if json_status['version'] != 1: print('unknown version: %s' % json_status) return infra_status = json_status.get('infra_status') result = 'success' if exit_status != 0: result = 'failure' if (exit_status < 0 or not infra_status or infra_status['ping_status_code'] != 200 or infra_status.get('num_user_error', 0) > 0): result = 'exception' num_failure = 0 ping_status_code = 0 if infra_status: num_failure = infra_status['num_exec_compiler_proxy_failure'] ping_status_code = infra_status['ping_status_code'] clobber = 0 if os.environ.get('BUILDBOT_CLOBBER'): clobber = 1 counter = { 'name': 'goma/failure', 'value': num_failure, 'builder': os.environ.get('BUILDBOT_BUILDERNAME', 'unknown'), 'master': os.environ.get('BUILDBOT_MASTERNAME', 'unknown'), 'slave': os.environ.get('BUILDBOT_SLAVENAME', 'unknown'), 'clobber': clobber, 'os': chromium_utils.PlatformName(), 'ping_status_code': ping_status_code, 'result': result } start_time = GetCompilerProxyStartTime() if start_time: counter['start_time'] = int(time.mktime(start_time.timetuple())) run_cmd = PLATFORM_RUN_CMD.get(os.name) if not run_cmd: print 'Unknown os.name: %s' % os.name return counter_json = json.dumps(counter) # base64 encode on windows because it doesn't like json on the command-line. if os.name == 'nt': counter_json = base64.b64encode(counter_json) cmd = [ sys.executable, run_cmd, 'infra.tools.send_ts_mon_values', '--verbose', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'goma-client', '--ts-mon-task-job-name', 'default', '--counter', counter_json ] cmd_filter = chromium_utils.FilterCapture() retcode = chromium_utils.RunCommand(cmd, filter_obj=cmd_filter, max_time=30) if retcode: print('Execution of send_ts_mon_values failed with code %s' % retcode) print '\n'.join(cmd_filter.text) except Exception as ex: print('error while sending ts mon json_file=%s: %s' % (json_file, ex))
def MakeGomaStatusCounter(json_file, exit_status, builder='unknown', master='unknown', slave='unknown', clobber=''): """Make latest Goma status counter which will be sent to ts_mon. Args: json_file: json filename string that has goma_ctl.py jsonstatus. exit_status: integer exit status of the build. Returns: counter dict if succeeded. None if failed. """ json_statuses = {} try: with open(json_file) as f: json_statuses = json.load(f) if not json_statuses: print('no json status is recorded in %s' % json_file) return None if len(json_statuses.get('notice', [])) != 1: print('unknown json statuses style: %s' % json_statuses) return None json_status = json_statuses['notice'][0] if json_status['version'] != 1: print('unknown version: %s' % json_status) return None infra_status = json_status.get('infra_status') result = 'success' if exit_status is None: result = 'exception' elif exit_status != 0: result = 'failure' if (exit_status < 0 or not infra_status or infra_status['ping_status_code'] != 200 or infra_status.get('num_user_error', 0) > 0): result = 'exception' num_failure = 0 ping_status_code = 0 if infra_status: num_failure = infra_status['num_exec_compiler_proxy_failure'] ping_status_code = infra_status['ping_status_code'] counter = { 'name': 'goma/failure', 'value': num_failure, 'builder': builder, 'master': master, 'slave': slave, 'clobber': 1 if clobber else 0, 'os': chromium_utils.PlatformName(), 'ping_status_code': ping_status_code, 'result': result } start_time = GetCompilerProxyStartTime() if start_time: counter['start_time'] = int(time.mktime(start_time.timetuple())) return counter except Exception as ex: print( 'error while making goma status counter for ts_mon: jons_file=%s: %s' % (json_file, ex)) return None
def archive(options, args): build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') (build_revision, _) = slave_utils.GetBuildRevisions( src_dir, None, revision_dir) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target)] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def ArchiveBuild(self): """Zips build files and uploads them, their symbols, and a change log.""" result = 0 if self._build_revision is None: raise archive_utils.StagingError('No build revision was provided') print 'Staging in %s' % self._staging_dir arch = archive_utils.BuildArch() fparser = archive_utils.FilesCfgParser(self._files_file, self.options.mode, arch) files_list = fparser.ParseLegacyList() self._archive_files = archive_utils.ExpandWildcards( self._build_dir, files_list) archives_list = fparser.ParseArchiveLists() # Check files and revision numbers. all_files_list = self._archive_files + [ item['filename'] for sublist in archives_list.values() for item in sublist ] all_files_list.append(self._version_file) not_found = archive_utils.VerifyFiles(all_files_list, self._build_dir, self.options.ignore) not_found_optional = [] for bad_fn in not_found[:]: if fparser.IsOptional(bad_fn): not_found_optional.append(bad_fn) not_found.remove(bad_fn) # Remove it from all file lists so we don't try to process it. if bad_fn in self._archive_files: self._archive_files.remove(bad_fn) for archive_list in archives_list.values(): archive_list[:] = [ x for x in archive_list if bad_fn != x['filename'] ] # TODO(mmoss): Now that we can declare files optional in FILES.cfg, should # we only allow not_found_optional, and fail on any leftover not_found # files? print 'last change: %d' % self._build_revision previous_revision = self.GetLastBuildRevision() if self._build_revision <= previous_revision: # If there have been no changes, report it but don't raise an exception. # Someone might have pushed the "force build" button. print 'No changes since last build (r%d <= r%d)' % ( self._build_revision, previous_revision) return 0 print 'build name: %s' % self._build_name archive_name = 'chrome-%s.zip' % chromium_utils.PlatformName() archive_file = self.CreateArchiveFile(archive_name, self._archive_files)[1] # Handle any custom archives. # TODO(mmoss): Largely copied from stage_build.py. Maybe refactor more of # this into archive_utils.py. archive_files = [archive_file] for archive_name in archives_list: if fparser.IsDirectArchive(archives_list[archive_name]): fileobj = archives_list[archive_name][0] # Copy the file to the path specified in archive_name, which might be # different than the dirname or basename in 'filename' (allowed by # 'direct_archive'). stage_subdir = os.path.dirname(archive_name) stage_fn = os.path.basename(archive_name) chromium_utils.MaybeMakeDirectory( os.path.join(self._staging_dir, stage_subdir)) print 'chromium_utils.CopyFileToDir(%s, %s, dest_fn=%s)' % ( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), stage_fn) if not self.options.dry_run: chromium_utils.CopyFileToDir( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), dest_fn=stage_fn) archive_files.append( os.path.join(self._staging_dir, archive_name)) else: custom_archive = self.CreateArchiveFile( archive_name, [f['filename'] for f in archives_list[archive_name]])[1] print 'Adding %s to be archived.' % (custom_archive) archive_files.append(custom_archive) # Generate a change log or an error message if no previous revision. changelog_path = os.path.join(self._staging_dir, 'changelog.xml') self._GenerateChangeLog(previous_revision, changelog_path) # Generate a revisions file which contains the Chromium/WebKit/V8's # revision information. self.GenerateRevisionFile() www_dir = os.path.join(self._www_dir_base, str(self._build_revision)) gs_bucket = self.options.factory_properties.get('gs_bucket', None) gs_acl = self.options.factory_properties.get('gs_acl', None) gs_base = None if gs_bucket: gs_base = '/'.join( [gs_bucket, self._build_name, str(self._build_revision)]) self._UploadBuild(www_dir, changelog_path, self.revisions_path, archive_files, gs_base, gs_acl) # Archive Linux packages (if any -- only created for Chrome builds). if chromium_utils.IsLinux(): linux_packages = (glob.glob( os.path.join(self._build_dir, '*-r%s_*.deb' % self._chromium_revision))) linux_packages.extend( glob.glob( os.path.join(self._build_dir, '*-%s.*.rpm' % self._chromium_revision))) for package_file in linux_packages: print 'SshCopyFiles(%s, %s, %s)' % ( package_file, self.options.archive_host, www_dir) if not self.options.dry_run: print 'SshMakeDirectory(%s, %s)' % (self.options.archive_host, www_dir) self.MySshMakeDirectory(self.options.archive_host, www_dir, gs_base) for package_file in linux_packages: self.MyMakeWorldReadable(package_file, gs_base) self.MySshCopyFiles(package_file, self.options.archive_host, www_dir, gs_base, gs_acl=gs_acl) # Cleanup archived packages, otherwise they keep accumlating since # they have different filenames with each build. os.unlink(package_file) self.UploadTests(www_dir, gs_base, gs_acl) if not self.options.dry_run: # Save the current build revision locally so we can compute a changelog # next time self.SaveBuildRevisionToSpecifiedFile(self.last_change_file) # Record the latest revision in the developer archive directory. latest_file_path = os.path.join(self._www_dir_base, 'LATEST') if chromium_utils.IsWindows(): print 'Saving revision to %s' % latest_file_path if gs_base: MyCopyFileToGS(self.last_change_file, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) if not gs_base or self._dual_upload: self.SaveBuildRevisionToSpecifiedFile(latest_file_path) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): # Files are created umask 077 by default, so make it world-readable # before pushing to web server. self.MyMakeWorldReadable(self.last_change_file, gs_base) print 'Saving revision to %s:%s' % (self.options.archive_host, latest_file_path) self.MySshCopyFiles(self.last_change_file, self.options.archive_host, latest_file_path, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) if len(not_found_optional): sys.stderr.write('\n\nINFO: Optional File(s) not found: %s\n' % ', '.join(not_found_optional)) if len(not_found): sys.stderr.write('\n\nWARNING: File(s) not found: %s\n' % ', '.join(not_found)) return result
Args: outdir: a directory that contains .ninja_log. compiler: compiler used for the build. command: command line. exit_status: ninja's exit status. """ ninja_log_path = os.path.join(outdir, '.ninja_log') try: st = os.stat(ninja_log_path) mtime = datetime.datetime.fromtimestamp(st.st_mtime) except OSError, e: print e return cwd = os.getcwd() platform = chromium_utils.PlatformName() # info['cmdline'] should be list of string for # go struct on chromium-build-stats. if isinstance(command, str) or isinstance(command, unicode): command = [command] info = { 'cmdline': command, 'cwd': cwd, 'platform': platform, 'exit': exit_status, 'env': {} } for k, v in os.environ.iteritems(): info['env'][k] = v
def real_main(options): """ Download a build, extract it to build\\BuildDir\\full-build-win32 and rename it to build\\BuildDir\\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision and not options.build_archive_url: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision url, archive_name = GetBuildUrl(options, build_revision, webkit_revision) if archive_name is None: archive_name = 'build.zip' base_url = None else: base_url = '/'.join(url.split('/')[:-1] + [archive_name]) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # If the url is valid, we download the file. if not failure: if not handler.download(): if options.halt_on_missing_build: return slave_utils.ERROR_EXIT_CODE failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://') or not base_url: continue else: print 'Fetching latest build at %s' % base_url base_handler = handler.__class__(base_url, handler.archive_name) if not base_handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read() build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def real_main(options): """Download a build and extract. Extract to build\\BuildDir\\full-build-win32 and rename it to build\\BuildDir\\Target """ target_build_output_dir = os.path.join(options.build_dir, options.target) platform = chromium_utils.PlatformName() revision = options.revision if not revision: revision = GetLatestRevision(options.build_url, platform) if not revision: print 'Failed to get revision number.' return slave_utils.ERROR_EXIT_CODE archive_url = GetBuildUrl(options.build_url, platform, revision) archive_name = 'dynamorio.' + os.path.basename(archive_url).split('.', 1)[1] temp_dir = tempfile.mkdtemp() try: # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s' % (tries, archive_url) failure = False try: print '%s/%s' % (archive_url, archive_name) urllib.urlretrieve(archive_url, archive_name) print '\nDownload complete' except IOError: print '\nFailed to download build' failure = True if options.halt_on_missing_build: return slave_utils.ERROR_EXIT_CODE if failure: continue print 'Extracting build %s to %s...' % (archive_name, options.build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, temp_dir) # Look for the top level directory from extracted build. entries = os.listdir(temp_dir) output_dir = temp_dir if (len(entries) == 1 and os.path.isdir( os.path.join(output_dir, entries[0]))): output_dir = os.path.join(output_dir, entries[0]) print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... time.sleep(3) continue return 0 finally: chromium_utils.RemoveDirectory(temp_dir) # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def TargetPlatformName(self): return self.options.factory_properties.get( 'target_os', chromium_utils.PlatformName())
] # Write out the revision number so we can figure it out in extract_build.py. build_revision_file_name = 'FULL_BUILD_REVISION' build_revision_path = os.path.join(build_dir, build_revision_file_name) try: build_revision_file = open(build_revision_path, 'w') build_revision_file.write('%d' % build_revision) build_revision_file.close() if chromium_utils.IsMac() or chromium_utils.IsLinux(): os.chmod(build_revision_path, 0644) zip_file_list.append(build_revision_file_name) except IOError: print 'Writing to revision file %s failed ' % build_revision_path zip_file_name = 'full-build-%s' % chromium_utils.PlatformName() (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) if chromium_utils.IsMac() or chromium_utils.IsLinux(): os.chmod(zip_file, 0644) # Report the size of the zip file to help catch when it gets too big and # can cause bot failures from timeouts during downloads to testers. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size