def DownloadFuzzerCorpus(fuzzer, dest_directory=None): """Downloads a corpus and returns its path. Downloads a corpus to a subdirectory of dest_directory if specified and returns path on the filesystem of the corpus. Asks users to authenticate if permission to read from bucket is denied. Args: fuzzer: The name of the fuzzer who's corpus we want to download. dest_directory: The directory to download the corpus to. Returns: The path to the downloaded corpus. Raises: gs.NoSuchKey: A corpus for the fuzzer doesn't exist. gs.GSCommandError: The corpus failed to download for another reason. """ if not fuzzer.startswith('chromeos_'): # ClusterFuzz internally appends "chromeos_" to chromeos targets' names. # Therefore we must do so in order to find the corpus. fuzzer = 'chromeos_%s' % fuzzer if dest_directory is None: dest_directory = GetScriptStoragePath(CORPUS_DIRECTORY_NAME).chroot osutils.SafeMakedirsNonRoot(dest_directory) clusterfuzz_gcs_corpus_bucket = 'chromeos-corpus' suburl = 'libfuzzer/%s' % fuzzer gcs_path = gs.GetGsURL(clusterfuzz_gcs_corpus_bucket, for_gsutil=True, public=False, suburl=suburl) dest_path = os.path.join(dest_directory, fuzzer) try: logging.info('Downloading corpus to %s.', dest_path) ctx = gs.GSContext() ctx.Copy(gcs_path, dest_directory, recursive=True, parallel=True, debug_level=logging.DEBUG) logging.info('Finished downloading corpus.') except gs.GSNoSuchKey as exception: logging.error('Corpus for fuzzer: %s does not exist.', fuzzer) raise exception # Try to authenticate if we were denied permission to access the corpus. except gs.GSCommandError as exception: logging.error( 'gsutil failed to download the corpus. You may need to log in. See:\n' 'https://chromium.googlesource.com/chromiumos/docs/+/master/gsutil.md' '#setup\n' 'for instructions on doing this.') raise exception return dest_path
def GetSdkURL(for_gsutil=False, suburl=''): """Construct a Google Storage URL for accessing SDK related archives Args: for_gsutil: Do you want a URL for passing to `gsutil`? suburl: A url fragment to tack onto the end Returns: The fully constructed URL """ return gs.GetGsURL(constants.SDK_GS_BUCKET, for_gsutil=for_gsutil, suburl=suburl)
def PrepareBinhostUploads(input_proto, output_proto, config): """Return a list of files to upload to the binhost. See BinhostService documentation in api/proto/binhost.proto. Args: input_proto (PrepareBinhostUploadsRequest): The input proto. output_proto (PrepareBinhostUploadsResponse): The output proto. config (api_config.ApiConfig): The API call config. """ if input_proto.sysroot.build_target.name: build_target_msg = input_proto.sysroot.build_target else: build_target_msg = input_proto.build_target sysroot_path = input_proto.sysroot.path if not sysroot_path and not build_target_msg.name: cros_build_lib.Die('Sysroot.path is required.') build_target = controller_util.ParseBuildTarget(build_target_msg) chroot = controller_util.ParseChroot(input_proto.chroot) if not sysroot_path: sysroot_path = build_target.root sysroot = sysroot_lib.Sysroot(sysroot_path) uri = input_proto.uri # For now, we enforce that all input URIs are Google Storage buckets. if not gs.PathIsGs(uri): raise ValueError('Upload URI %s must be Google Storage.' % uri) if config.validate_only: return controller.RETURN_CODE_VALID_INPUT parsed_uri = urllib.parse.urlparse(uri) upload_uri = gs.GetGsURL(parsed_uri.netloc, for_gsutil=True).rstrip('/') upload_path = parsed_uri.path.lstrip('/') # Read all packages and update the index. The index must be uploaded to the # binhost for Portage to use it, so include it in upload_targets. uploads_dir = binhost.GetPrebuiltsRoot(chroot, sysroot, build_target) index_path = binhost.UpdatePackageIndex(uploads_dir, upload_uri, upload_path, sudo=True) upload_targets = binhost.GetPrebuiltsFiles(uploads_dir) assert index_path.startswith(uploads_dir), ( 'expected index_path to start with uploads_dir') upload_targets.append(index_path[len(uploads_dir):]) output_proto.uploads_dir = uploads_dir for upload_target in upload_targets: output_proto.upload_targets.add().path = upload_target.strip('/')
def PrepareDevInstallBinhostUploads(input_proto, output_proto, config): """Return a list of files to upload to the binhost" The files will also be copied to the uploads_dir. See BinhostService documentation in api/proto/binhost.proto. Args: input_proto (PrepareDevInstallBinhostUploadsRequest): The input proto. output_proto (PrepareDevInstallBinhostUploadsResponse): The output proto. config (api_config.ApiConfig): The API call config. """ sysroot_path = input_proto.sysroot.path # build_target = build_target_util.BuildTarget(target_name) chroot = controller_util.ParseChroot(input_proto.chroot) sysroot = sysroot_lib.Sysroot(sysroot_path) uri = input_proto.uri # For now, we enforce that all input URIs are Google Storage buckets. if not gs.PathIsGs(uri): raise ValueError('Upload URI %s must be Google Storage.' % uri) if config.validate_only: return controller.RETURN_CODE_VALID_INPUT parsed_uri = urllib.parse.urlparse(uri) upload_uri = gs.GetGsURL(parsed_uri.netloc, for_gsutil=True).rstrip('/') upload_path = parsed_uri.path.lstrip('/') # Calculate the filename for the to-be-created Packages file, which will # contain only devinstall packages. devinstall_package_index_path = os.path.join(input_proto.uploads_dir, 'Packages') upload_targets_list = binhost.ReadDevInstallFilesToCreatePackageIndex( chroot, sysroot, devinstall_package_index_path, upload_uri, upload_path) package_dir = chroot.full_path(sysroot.path, 'packages') for upload_target in upload_targets_list: # Copy each package to target/category/package upload_target = upload_target.strip('/') category = upload_target.split(os.sep)[0] target_dir = os.path.join(input_proto.uploads_dir, category) if not os.path.exists(target_dir): os.makedirs(target_dir) full_src_pkg_path = os.path.join(package_dir, upload_target) full_target_src_path = os.path.join(input_proto.uploads_dir, upload_target) shutil.copyfile(full_src_pkg_path, full_target_src_path) output_proto.upload_targets.add().path = upload_target output_proto.upload_targets.add().path = 'Packages'