def DeleteAndroidVirtualDevices(cfg, instance_names): """Deletes android devices. Args: cfg: An AcloudConfig instance. instance_names: A list of names of the instances to delete. Returns: A Report instance. """ r = report.Report(command="delete") credentials = auth.CreateCredentials(cfg, ALL_SCOPES) compute_client = android_compute_client.AndroidComputeClient(cfg, credentials) try: deleted, failed, error_msgs = compute_client.DeleteInstances( instance_names, cfg.zone) _AddDeletionResultToReport( r, deleted, failed, error_msgs, resource_name="instance") if r.status == report.Status.UNKNOWN: r.SetStatus(report.Status.SUCCESS) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r
def DownloadRemoteArtifact(cfg, build_target, build_id, artifact, extract_path, decompress=False): """Download remote artifact. Args: cfg: An AcloudConfig instance. build_target: String, the build target, e.g. cf_x86_phone-userdebug. build_id: String, Build id, e.g. "2263051", "P2804227" artifact: String, zip image or cvd host package artifact. extract_path: String, a path include extracted files. decompress: Boolean, if true decompress the artifact. """ build_client = android_build_client.AndroidBuildClient( auth.CreateCredentials(cfg)) temp_file = os.path.join(extract_path, artifact) build_client.DownloadArtifact(build_target, build_id, artifact, temp_file) if decompress: utils.Decompress(temp_file, extract_path) try: os.remove(temp_file) logger.debug("Deleted temporary file %s", temp_file) except OSError as e: logger.error("Failed to delete temporary file: %s", str(e))
def __init__(self, cfg, devices=None): self._devices = devices or [] self._cfg = cfg credentials = auth.CreateCredentials(cfg) self._build_client = android_build_client.AndroidBuildClient( credentials) self._storage_client = gstorage_client.StorageClient(credentials) self._compute_client = android_compute_client.AndroidComputeClient( cfg, credentials)
def __init__(self, cfg, build_target, build_id, emulator_build_target, emulator_build_id, kernel_build_id=None, kernel_branch=None, kernel_build_target=None, gpu=None, avd_spec=None, tags=None, branch=None, emulator_branch=None): """Initialize. Args: cfg: An AcloudConfig instance. build_target: String, the build target, e.g. aosp_x86-eng. build_id: String, Build id, e.g. "2263051", "P2804227" emulator_build_target: String, the emulator build target, e.g. aosp_x86-eng. emulator_build_id: String, emulator build id. gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" avd_spec: An AVDSpec instance. tags: A list of tags to associate with the instance. e.g. ["http-server", "https-server"] branch: String, branch of the emulator build target. emulator_branch: String, branch of the emulator. """ self.credentials = auth.CreateCredentials(cfg) compute_client = goldfish_compute_client.GoldfishComputeClient( cfg, self.credentials) super(GoldfishDeviceFactory, self).__init__(compute_client) # Private creation parameters self._cfg = cfg self._gpu = gpu self._avd_spec = avd_spec self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb self._extra_scopes = cfg.extra_scopes self._tags = tags # Configure clients self._build_client = android_build_client.AndroidBuildClient( self.credentials) # Get build info self.build_info = self._build_client.GetBuildInfo( build_target, build_id, branch) self.emulator_build_info = self._build_client.GetBuildInfo( emulator_build_target, emulator_build_id, emulator_branch) self.kernel_build_info = self._build_client.GetBuildInfo( kernel_build_target or cfg.kernel_build_target, kernel_build_id, kernel_branch)
def _ProcessRemoteBuildArgs(self, args): """Get the remote build args. Some of the acloud magic happens here, we will infer some of these values if the user hasn't specified them. Args: args: Namespace object from argparse.parse_args. """ self._remote_image = {} self._remote_image[constants.BUILD_BRANCH] = args.branch if not self._remote_image[constants.BUILD_BRANCH]: self._remote_image[constants.BUILD_BRANCH] = self._GetBuildBranch( args.build_id, args.build_target) self._remote_image[constants.BUILD_TARGET] = args.build_target if not self._remote_image[constants.BUILD_TARGET]: self._remote_image[constants.BUILD_TARGET] = self._GetBuildTarget( args) else: # If flavor isn't specified, try to infer it from build target, # if we can't, just default to phone flavor. self._flavor = args.flavor or self._GetFlavorFromString( self._remote_image[ constants.BUILD_TARGET]) or constants.FLAVOR_PHONE # infer avd_type from build_target. for avd_type, avd_type_abbr in constants.AVD_TYPES_MAPPING.items(): if re.match(r"(.*_)?%s_" % avd_type_abbr, self._remote_image[constants.BUILD_TARGET]): self._avd_type = avd_type break self._remote_image[constants.BUILD_ID] = args.build_id if not self._remote_image[constants.BUILD_ID]: build_client = android_build_client.AndroidBuildClient( auth.CreateCredentials(self._cfg)) self._remote_image[constants.BUILD_ID] = build_client.GetLKGB( self._remote_image[constants.BUILD_TARGET], self._remote_image[constants.BUILD_BRANCH]) self._remote_image[constants.CHEEPS_BETTY_IMAGE] = ( args.cheeps_betty_image) # Process system image and kernel image. self._system_build_info = { constants.BUILD_ID: args.system_build_id, constants.BUILD_BRANCH: args.system_branch, constants.BUILD_TARGET: args.system_build_target } self._kernel_build_info = { constants.BUILD_ID: args.kernel_build_id, constants.BUILD_BRANCH: args.kernel_branch, constants.BUILD_TARGET: args.kernel_build_target }
def __init__(self, cfg, instance_name): """Initialize. Args: cfg: AcloudConfig object, used to create credentials. instance_name: string, instance name. """ credentials = auth.CreateCredentials(cfg, ALL_SCOPES) self._compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) # Name of the Cloud Android instance. self._instance_name = instance_name # IP of the Cloud Android instance. self._target_ip = self._compute_client.GetInstanceIP(instance_name)
def __init__(self, cfg, build_target, build_id, branch=None, kernel_build_id=None, kernel_branch=None, kernel_build_target=None, system_branch=None, system_build_id=None, system_build_target=None, boot_timeout_secs=None, ins_timeout_secs=None, report_internal_ip=None, gpu=None): self.credentials = auth.CreateCredentials(cfg) if cfg.enable_multi_stage: compute_client = cvd_compute_client_multi_stage.CvdComputeClient( cfg, self.credentials, boot_timeout_secs, ins_timeout_secs, report_internal_ip, gpu) else: compute_client = cvd_compute_client.CvdComputeClient( cfg, self.credentials) super(CuttlefishDeviceFactory, self).__init__(compute_client) # Private creation parameters self._cfg = cfg self._build_target = build_target self._build_id = build_id self._branch = branch self._kernel_build_id = kernel_build_id self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb self._extra_scopes = cfg.extra_scopes # Configure clients for interaction with GCE/Build servers self._build_client = android_build_client.AndroidBuildClient( self.credentials) # Get build_info namedtuple for platform, kernel, system build self.build_info = self._build_client.GetBuildInfo( build_target, build_id, branch) self.kernel_build_info = self._build_client.GetBuildInfo( kernel_build_target or cfg.kernel_build_target, kernel_build_id, kernel_branch) self.system_build_info = self._build_client.GetBuildInfo( system_build_target or build_target, system_build_id, system_branch)
def AddPublicSshRsaToInstance(cfg, user, instance_name): """Add the public rsa key to the instance's metadata. When the public key doesn't exist in the metadata, it will add it. Args: cfg: An AcloudConfig instance. user: String, the ssh username to access instance. instance_name: String, instance name. """ credentials = auth.CreateCredentials(cfg) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) compute_client.AddSshRsaInstanceMetadata(user, cfg.ssh_public_key_path, instance_name)
def __init__(self, cfg, avd_spec=None): """Initialize. Args: cfg: An AcloudConfig instance. avd_spec: An AVDSpec instance. """ self.credentials = auth.CreateCredentials(cfg) compute_client = cheeps_compute_client.CheepsComputeClient( cfg, self.credentials) super(CheepsDeviceFactory, self).__init__(compute_client) self._cfg = cfg self._avd_spec = avd_spec
def CheckAccess(cfg): """Check if user has access. Args: cfg: An AcloudConfig instance. """ credentials = auth.CreateCredentials(cfg) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) logger.info("Checking if user has access to project %s", cfg.project) if not compute_client.CheckAccess(): logger.error("User does not have access to project %s", cfg.project) # Print here so that command line user can see it. print("Looks like you do not have access to %s. " % cfg.project) if cfg.project in cfg.no_project_access_msg_map: print(cfg.no_project_access_msg_map[cfg.project])
def _GetBuildBranch(self, build_id, build_target): """Infer build branch if user didn't specify branch name. Args: build_id: String, Build id, e.g. "2263051", "P2804227" build_target: String, the build target, e.g. cf_x86_phone-userdebug Returns: String, name of build branch. """ # Infer branch from build_target and build_id if build_id and build_target: build_client = android_build_client.AndroidBuildClient( auth.CreateCredentials(self._cfg)) return build_client.GetBranch(build_target, build_id) return self._GetBranchFromRepo()
def GetRemoteInstances(cfg): """Look for remote instances. We're going to query the GCP project for all instances that created by user. Args: cfg: AcloudConfig object. Returns: instance_list: List of remote instances. """ credentials = auth.CreateCredentials(cfg) compute_client = gcompute_client.ComputeClient(cfg, credentials) filter_item = "labels.%s=%s" % (constants.LABEL_CREATE_BY, getpass.getuser()) all_instances = compute_client.ListInstances(instance_filter=filter_item) logger.debug("Instance list from: (filter: %s\n%s):", filter_item, all_instances) return _SortInstancesForDisplay(_ProcessInstances(all_instances))
def DeleteAndroidVirtualDevices(cfg, instance_names, default_report=None): """Deletes android devices. Args: cfg: An AcloudConfig instance. instance_names: A list of names of the instances to delete. default_report: A initialized Report instance. Returns: A Report instance. """ # delete, failed, error_msgs are used to record result. deleted = [] failed = [] error_msgs = [] r = default_report if default_report else report.Report(command="delete") credentials = auth.CreateCredentials(cfg) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) zone_instances = compute_client.GetZonesByInstances(instance_names) try: for zone, instances in zone_instances.items(): deleted_ins, failed_ins, error_ins = compute_client.DeleteInstances( instances, zone) deleted.extend(deleted_ins) failed.extend(failed_ins) error_msgs.extend(error_ins) AddDeletionResultToReport(r, deleted, failed, error_msgs, resource_name="instance") if r.status == report.Status.UNKNOWN: r.SetStatus(report.Status.SUCCESS) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r
def AddSshRsa(cfg, user, ssh_rsa_path): """Add public ssh rsa key to the project. Args: cfg: An AcloudConfig instance. user: the name of the user which the key belongs to. ssh_rsa_path: The absolute path to public rsa key. Returns: A Report instance. """ r = report.Report(command="sshkey") try: credentials = auth.CreateCredentials(cfg, ALL_SCOPES) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) compute_client.AddSshRsa(user, ssh_rsa_path) r.SetStatus(report.Status.SUCCESS) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r
def CleanUpRemoteHost(cfg, remote_host, host_user, host_ssh_private_key_path=None): """Clean up the remote host. Args: cfg: An AcloudConfig instance. remote_host : String, ip address or host name of the remote host. host_user: String of user login into the instance. host_ssh_private_key_path: String of host key for logging in to the host. Returns: A Report instance. """ delete_report = report.Report(command="delete") credentials = auth.CreateCredentials(cfg) compute_client = cvd_compute_client_multi_stage.CvdComputeClient( acloud_config=cfg, oauth2_credentials=credentials) ssh = ssh_object.Ssh( ip=ssh_object.IP(ip=remote_host), user=host_user, ssh_private_key_path=( host_ssh_private_key_path or cfg.ssh_private_key_path)) try: compute_client.InitRemoteHost(ssh, remote_host, host_user) delete_report.SetStatus(report.Status.SUCCESS) device_driver.AddDeletionResultToReport( delete_report, [remote_host], failed=[], error_msgs=[], resource_name="remote host") except subprocess.CalledProcessError as e: delete_report.AddError(str(e)) delete_report.SetStatus(report.Status.FAIL) return delete_report
def _FetchBuildIdFromFile(cfg, build_target, build_id, filename): """Parse and fetch build id from a file based on a pattern. Verify if one of the system image or emulator binary build id is missing. If found missing, then update according to the resource file. Args: cfg: An AcloudConfig instance. build_target: Target name. build_id: Build id, a string, e.g. "2263051", "P2804227" filename: Name of file containing the build info. Returns: A build id or None """ build_client = android_build_client.AndroidBuildClient( auth.CreateCredentials(cfg)) with utils.TempDir() as tempdir: temp_filename = os.path.join(tempdir, filename) build_client.DownloadArtifact(build_target, build_id, filename, temp_filename) return ParseBuildInfo(temp_filename)
def Cleanup(cfg, expiration_mins): """Cleans up stale gce images, gce instances, and disk images in storage. Args: cfg: An AcloudConfig instance. expiration_mins: Integer, resources older than |expiration_mins| will be cleaned up. Returns: A Report instance. """ r = report.Report(command="cleanup") try: cut_time = (datetime.datetime.now(dateutil.tz.tzlocal()) - datetime.timedelta(minutes=expiration_mins)) logger.info( "Cleaning up any gce images/instances and cached build artifacts." "in google storage that are older than %s", cut_time) credentials = auth.CreateCredentials(cfg, ALL_SCOPES) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) storage_client = gstorage_client.StorageClient(credentials) # Cleanup expired instances items = compute_client.ListInstances(zone=cfg.zone) cleanup_list = [ item["name"] for item in _FindOldItems(items, cut_time, "creationTimestamp") ] logger.info("Found expired instances: %s", cleanup_list) for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT): result = compute_client.DeleteInstances( instances=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT], zone=cfg.zone) _AddDeletionResultToReport(r, *result, resource_name="instance") # Cleanup expired images items = compute_client.ListImages() skip_list = cfg.precreated_data_image_map.viewvalues() cleanup_list = [ item["name"] for item in _FindOldItems(items, cut_time, "creationTimestamp") if item["name"] not in skip_list ] logger.info("Found expired images: %s", cleanup_list) for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT): result = compute_client.DeleteImages( image_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT]) _AddDeletionResultToReport(r, *result, resource_name="image") # Cleanup expired disks # Disks should have been attached to instances with autoDelete=True. # However, sometimes disks may not be auto deleted successfully. items = compute_client.ListDisks(zone=cfg.zone) cleanup_list = [ item["name"] for item in _FindOldItems(items, cut_time, "creationTimestamp") if not item.get("users") ] logger.info("Found expired disks: %s", cleanup_list) for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT): result = compute_client.DeleteDisks( disk_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT], zone=cfg.zone) _AddDeletionResultToReport(r, *result, resource_name="disk") # Cleanup expired google storage items = storage_client.List(bucket_name=cfg.storage_bucket_name) cleanup_list = [ item["name"] for item in _FindOldItems(items, cut_time, "timeCreated") ] logger.info("Found expired cached artifacts: %s", cleanup_list) for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT): result = storage_client.DeleteFiles( bucket_name=cfg.storage_bucket_name, object_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT]) _AddDeletionResultToReport( r, *result, resource_name="cached_build_artifact") # Everything succeeded, write status to report. if r.status == report.Status.UNKNOWN: r.SetStatus(report.Status.SUCCESS) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r
def CreateAndroidVirtualDevices(cfg, build_target=None, build_id=None, num=1, gce_image=None, local_disk_image=None, cleanup=True, serial_log_file=None, logcat_file=None): """Creates one or multiple android devices. Args: cfg: An AcloudConfig instance. build_target: Target name, e.g. "gce_x86-userdebug" build_id: Build id, a string, e.g. "2263051", "P2804227" num: Number of devices to create. gce_image: string, if given, will use this gce image instead of creating a new one. implies cleanup=False. local_disk_image: string, path to a local disk image, e.g. /tmp/avd-system.tar.gz cleanup: boolean, if True clean up compute engine image and disk image in storage after creating the instance. serial_log_file: A path to a file where serial output should be saved to. logcat_file: A path to a file where logcat logs should be saved. Returns: A Report instance. """ r = report.Report(command="create") credentials = auth.CreateCredentials(cfg, ALL_SCOPES) compute_client = android_compute_client.AndroidComputeClient(cfg, credentials) try: _CreateSshKeyPairIfNecessary(cfg) device_pool = AndroidVirtualDevicePool(cfg) device_pool.CreateDevices( num, build_target, build_id, gce_image, local_disk_image, cleanup, extra_data_disk_size_gb=cfg.extra_data_disk_size_gb, precreated_data_image=cfg.precreated_data_image_map.get( cfg.extra_data_disk_size_gb)) failures = device_pool.WaitForBoot() # Write result to report. for device in device_pool.devices: device_dict = {"ip": device.ip, "instance_name": device.instance_name} if device.instance_name in failures: r.AddData(key="devices_failing_boot", value=device_dict) r.AddError(str(failures[device.instance_name])) else: r.AddData(key="devices", value=device_dict) if failures: r.SetStatus(report.Status.BOOT_FAIL) else: r.SetStatus(report.Status.SUCCESS) # Dump serial and logcat logs. if serial_log_file: _FetchSerialLogsFromDevices( compute_client, instance_names=[d.instance_name for d in device_pool.devices], port=constants.DEFAULT_SERIAL_PORT, output_file=serial_log_file) if logcat_file: _FetchSerialLogsFromDevices( compute_client, instance_names=[d.instance_name for d in device_pool.devices], port=constants.LOGCAT_SERIAL_PORT, output_file=logcat_file) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r
def CreateGCETypeAVD(cfg, build_target=None, build_id=None, num=1, gce_image=None, local_disk_image=None, cleanup=True, serial_log_file=None, autoconnect=False, report_internal_ip=False, avd_spec=None): """Creates one or multiple gce android devices. Args: cfg: An AcloudConfig instance. build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug" build_id: Build id, a string, e.g. "2263051", "P2804227" num: Number of devices to create. gce_image: string, if given, will use this gce image instead of creating a new one. implies cleanup=False. local_disk_image: string, path to a local disk image, e.g. /tmp/avd-system.tar.gz cleanup: boolean, if True clean up compute engine image and disk image in storage after creating the instance. serial_log_file: A path to a file where serial output should be saved to. autoconnect: Create ssh tunnel(s) and adb connect after device creation. report_internal_ip: Boolean to report the internal ip instead of external ip. avd_spec: AVDSpec object for pass hw_property. Returns: A Report instance. """ r = report.Report(command="create") credentials = auth.CreateCredentials(cfg) compute_client = android_compute_client.AndroidComputeClient( cfg, credentials) try: common_operations.CreateSshKeyPairIfNecessary(cfg) device_pool = AndroidVirtualDevicePool(cfg) device_pool.CreateDevices( num, build_target, build_id, gce_image, local_disk_image, cleanup, extra_data_disk_size_gb=cfg.extra_data_disk_size_gb, precreated_data_image=cfg.precreated_data_image_map.get( cfg.extra_data_disk_size_gb), avd_spec=avd_spec, extra_scopes=cfg.extra_scopes) failures = device_pool.WaitForBoot() # Write result to report. for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip else device.ip.external) device_dict = {"ip": ip, "instance_name": device.instance_name} if autoconnect: forwarded_ports = utils.AutoConnect( ip_addr=ip, rsa_key_file=cfg.ssh_private_key_path, target_vnc_port=constants.GCE_VNC_PORT, target_adb_port=constants.GCE_ADB_PORT, ssh_user=_SSH_USER, client_adb_port=avd_spec.client_adb_port, extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port device_dict[constants.ADB_PORT] = forwarded_ports.adb_port if avd_spec.unlock_screen: AdbTools(forwarded_ports.adb_port).AutoUnlockScreen() if device.instance_name in failures: r.AddData(key="devices_failing_boot", value=device_dict) r.AddError(str(failures[device.instance_name])) else: r.AddData(key="devices", value=device_dict) if failures: r.SetStatus(report.Status.BOOT_FAIL) else: r.SetStatus(report.Status.SUCCESS) # Dump serial logs. if serial_log_file: _FetchSerialLogsFromDevices( compute_client, instance_names=[d.instance_name for d in device_pool.devices], port=constants.DEFAULT_SERIAL_PORT, output_file=serial_log_file) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) return r