def DeleteAndroidVirtualDevices(cfg, instance_names):
"""Deletes android devices.
Args:
cfg: An AcloudConfig instance.
instance_names: A list of names of the instances to delete.
Returns:
A Report instance.
"""
r = report.Report(command="delete")
credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
compute_client = android_compute_client.AndroidComputeClient(cfg,
credentials)
try:
deleted, failed, error_msgs = compute_client.DeleteInstances(
instance_names, cfg.zone)
_AddDeletionResultToReport(
r, deleted,
failed, error_msgs,
resource_name="instance")
if r.status == report.Status.UNKNOWN:
r.SetStatus(report.Status.SUCCESS)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
def DownloadRemoteArtifact(cfg,
build_target,
build_id,
artifact,
extract_path,
decompress=False):
"""Download remote artifact.
Args:
cfg: An AcloudConfig instance.
build_target: String, the build target, e.g. cf_x86_phone-userdebug.
build_id: String, Build id, e.g. "2263051", "P2804227"
artifact: String, zip image or cvd host package artifact.
extract_path: String, a path include extracted files.
decompress: Boolean, if true decompress the artifact.
"""
build_client = android_build_client.AndroidBuildClient(
auth.CreateCredentials(cfg))
temp_file = os.path.join(extract_path, artifact)
build_client.DownloadArtifact(build_target, build_id, artifact, temp_file)
if decompress:
utils.Decompress(temp_file, extract_path)
try:
os.remove(temp_file)
logger.debug("Deleted temporary file %s", temp_file)
except OSError as e:
logger.error("Failed to delete temporary file: %s", str(e))
def __init__(self, cfg, devices=None):
self._devices = devices or []
self._cfg = cfg
credentials = auth.CreateCredentials(cfg)
self._build_client = android_build_client.AndroidBuildClient(
credentials)
self._storage_client = gstorage_client.StorageClient(credentials)
self._compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
def __init__(self,
cfg,
build_target,
build_id,
emulator_build_target,
emulator_build_id,
kernel_build_id=None,
kernel_branch=None,
kernel_build_target=None,
gpu=None,
avd_spec=None,
tags=None,
branch=None,
emulator_branch=None):
"""Initialize.
Args:
cfg: An AcloudConfig instance.
build_target: String, the build target, e.g. aosp_x86-eng.
build_id: String, Build id, e.g. "2263051", "P2804227"
emulator_build_target: String, the emulator build target, e.g. aosp_x86-eng.
emulator_build_id: String, emulator build id.
gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80"
avd_spec: An AVDSpec instance.
tags: A list of tags to associate with the instance. e.g.
["http-server", "https-server"]
branch: String, branch of the emulator build target.
emulator_branch: String, branch of the emulator.
"""
self.credentials = auth.CreateCredentials(cfg)
compute_client = goldfish_compute_client.GoldfishComputeClient(
cfg, self.credentials)
super(GoldfishDeviceFactory, self).__init__(compute_client)
# Private creation parameters
self._cfg = cfg
self._gpu = gpu
self._avd_spec = avd_spec
self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb
self._extra_scopes = cfg.extra_scopes
self._tags = tags
# Configure clients
self._build_client = android_build_client.AndroidBuildClient(
self.credentials)
# Get build info
self.build_info = self._build_client.GetBuildInfo(
build_target, build_id, branch)
self.emulator_build_info = self._build_client.GetBuildInfo(
emulator_build_target, emulator_build_id, emulator_branch)
self.kernel_build_info = self._build_client.GetBuildInfo(
kernel_build_target or cfg.kernel_build_target, kernel_build_id,
kernel_branch)
def _ProcessRemoteBuildArgs(self, args):
"""Get the remote build args.
Some of the acloud magic happens here, we will infer some of these
values if the user hasn't specified them.
Args:
args: Namespace object from argparse.parse_args.
"""
self._remote_image = {}
self._remote_image[constants.BUILD_BRANCH] = args.branch
if not self._remote_image[constants.BUILD_BRANCH]:
self._remote_image[constants.BUILD_BRANCH] = self._GetBuildBranch(
args.build_id, args.build_target)
self._remote_image[constants.BUILD_TARGET] = args.build_target
if not self._remote_image[constants.BUILD_TARGET]:
self._remote_image[constants.BUILD_TARGET] = self._GetBuildTarget(
args)
else:
# If flavor isn't specified, try to infer it from build target,
# if we can't, just default to phone flavor.
self._flavor = args.flavor or self._GetFlavorFromString(
self._remote_image[
constants.BUILD_TARGET]) or constants.FLAVOR_PHONE
# infer avd_type from build_target.
for avd_type, avd_type_abbr in constants.AVD_TYPES_MAPPING.items():
if re.match(r"(.*_)?%s_" % avd_type_abbr,
self._remote_image[constants.BUILD_TARGET]):
self._avd_type = avd_type
break
self._remote_image[constants.BUILD_ID] = args.build_id
if not self._remote_image[constants.BUILD_ID]:
build_client = android_build_client.AndroidBuildClient(
auth.CreateCredentials(self._cfg))
self._remote_image[constants.BUILD_ID] = build_client.GetLKGB(
self._remote_image[constants.BUILD_TARGET],
self._remote_image[constants.BUILD_BRANCH])
self._remote_image[constants.CHEEPS_BETTY_IMAGE] = (
args.cheeps_betty_image)
# Process system image and kernel image.
self._system_build_info = {
constants.BUILD_ID: args.system_build_id,
constants.BUILD_BRANCH: args.system_branch,
constants.BUILD_TARGET: args.system_build_target
}
self._kernel_build_info = {
constants.BUILD_ID: args.kernel_build_id,
constants.BUILD_BRANCH: args.kernel_branch,
constants.BUILD_TARGET: args.kernel_build_target
}
def __init__(self, cfg, instance_name):
"""Initialize.
Args:
cfg: AcloudConfig object, used to create credentials.
instance_name: string, instance name.
"""
credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
self._compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
# Name of the Cloud Android instance.
self._instance_name = instance_name
# IP of the Cloud Android instance.
self._target_ip = self._compute_client.GetInstanceIP(instance_name)
def __init__(self,
cfg,
build_target,
build_id,
branch=None,
kernel_build_id=None,
kernel_branch=None,
kernel_build_target=None,
system_branch=None,
system_build_id=None,
system_build_target=None,
boot_timeout_secs=None,
ins_timeout_secs=None,
report_internal_ip=None,
gpu=None):
self.credentials = auth.CreateCredentials(cfg)
if cfg.enable_multi_stage:
compute_client = cvd_compute_client_multi_stage.CvdComputeClient(
cfg, self.credentials, boot_timeout_secs, ins_timeout_secs,
report_internal_ip, gpu)
else:
compute_client = cvd_compute_client.CvdComputeClient(
cfg, self.credentials)
super(CuttlefishDeviceFactory, self).__init__(compute_client)
# Private creation parameters
self._cfg = cfg
self._build_target = build_target
self._build_id = build_id
self._branch = branch
self._kernel_build_id = kernel_build_id
self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb
self._extra_scopes = cfg.extra_scopes
# Configure clients for interaction with GCE/Build servers
self._build_client = android_build_client.AndroidBuildClient(
self.credentials)
# Get build_info namedtuple for platform, kernel, system build
self.build_info = self._build_client.GetBuildInfo(
build_target, build_id, branch)
self.kernel_build_info = self._build_client.GetBuildInfo(
kernel_build_target or cfg.kernel_build_target, kernel_build_id,
kernel_branch)
self.system_build_info = self._build_client.GetBuildInfo(
system_build_target or build_target, system_build_id,
system_branch)
def AddPublicSshRsaToInstance(cfg, user, instance_name):
"""Add the public rsa key to the instance's metadata.
When the public key doesn't exist in the metadata, it will add it.
Args:
cfg: An AcloudConfig instance.
user: String, the ssh username to access instance.
instance_name: String, instance name.
"""
credentials = auth.CreateCredentials(cfg)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
compute_client.AddSshRsaInstanceMetadata(user, cfg.ssh_public_key_path,
instance_name)
def __init__(self, cfg, avd_spec=None):
"""Initialize.
Args:
cfg: An AcloudConfig instance.
avd_spec: An AVDSpec instance.
"""
self.credentials = auth.CreateCredentials(cfg)
compute_client = cheeps_compute_client.CheepsComputeClient(
cfg, self.credentials)
super(CheepsDeviceFactory, self).__init__(compute_client)
self._cfg = cfg
self._avd_spec = avd_spec
def CheckAccess(cfg):
"""Check if user has access.
Args:
cfg: An AcloudConfig instance.
"""
credentials = auth.CreateCredentials(cfg)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
logger.info("Checking if user has access to project %s", cfg.project)
if not compute_client.CheckAccess():
logger.error("User does not have access to project %s", cfg.project)
# Print here so that command line user can see it.
print("Looks like you do not have access to %s. " % cfg.project)
if cfg.project in cfg.no_project_access_msg_map:
print(cfg.no_project_access_msg_map[cfg.project])
def _GetBuildBranch(self, build_id, build_target):
"""Infer build branch if user didn't specify branch name.
Args:
build_id: String, Build id, e.g. "2263051", "P2804227"
build_target: String, the build target, e.g. cf_x86_phone-userdebug
Returns:
String, name of build branch.
"""
# Infer branch from build_target and build_id
if build_id and build_target:
build_client = android_build_client.AndroidBuildClient(
auth.CreateCredentials(self._cfg))
return build_client.GetBranch(build_target, build_id)
return self._GetBranchFromRepo()
def GetRemoteInstances(cfg):
"""Look for remote instances.
We're going to query the GCP project for all instances that created by user.
Args:
cfg: AcloudConfig object.
Returns:
instance_list: List of remote instances.
"""
credentials = auth.CreateCredentials(cfg)
compute_client = gcompute_client.ComputeClient(cfg, credentials)
filter_item = "labels.%s=%s" % (constants.LABEL_CREATE_BY,
getpass.getuser())
all_instances = compute_client.ListInstances(instance_filter=filter_item)
logger.debug("Instance list from: (filter: %s\n%s):", filter_item,
all_instances)
return _SortInstancesForDisplay(_ProcessInstances(all_instances))
def DeleteAndroidVirtualDevices(cfg, instance_names, default_report=None):
"""Deletes android devices.
Args:
cfg: An AcloudConfig instance.
instance_names: A list of names of the instances to delete.
default_report: A initialized Report instance.
Returns:
A Report instance.
"""
# delete, failed, error_msgs are used to record result.
deleted = []
failed = []
error_msgs = []
r = default_report if default_report else report.Report(command="delete")
credentials = auth.CreateCredentials(cfg)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
zone_instances = compute_client.GetZonesByInstances(instance_names)
try:
for zone, instances in zone_instances.items():
deleted_ins, failed_ins, error_ins = compute_client.DeleteInstances(
instances, zone)
deleted.extend(deleted_ins)
failed.extend(failed_ins)
error_msgs.extend(error_ins)
AddDeletionResultToReport(r,
deleted,
failed,
error_msgs,
resource_name="instance")
if r.status == report.Status.UNKNOWN:
r.SetStatus(report.Status.SUCCESS)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
def AddSshRsa(cfg, user, ssh_rsa_path):
"""Add public ssh rsa key to the project.
Args:
cfg: An AcloudConfig instance.
user: the name of the user which the key belongs to.
ssh_rsa_path: The absolute path to public rsa key.
Returns:
A Report instance.
"""
r = report.Report(command="sshkey")
try:
credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
compute_client.AddSshRsa(user, ssh_rsa_path)
r.SetStatus(report.Status.SUCCESS)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
def CleanUpRemoteHost(cfg, remote_host, host_user,
host_ssh_private_key_path=None):
"""Clean up the remote host.
Args:
cfg: An AcloudConfig instance.
remote_host : String, ip address or host name of the remote host.
host_user: String of user login into the instance.
host_ssh_private_key_path: String of host key for logging in to the
host.
Returns:
A Report instance.
"""
delete_report = report.Report(command="delete")
credentials = auth.CreateCredentials(cfg)
compute_client = cvd_compute_client_multi_stage.CvdComputeClient(
acloud_config=cfg,
oauth2_credentials=credentials)
ssh = ssh_object.Ssh(
ip=ssh_object.IP(ip=remote_host),
user=host_user,
ssh_private_key_path=(
host_ssh_private_key_path or cfg.ssh_private_key_path))
try:
compute_client.InitRemoteHost(ssh, remote_host, host_user)
delete_report.SetStatus(report.Status.SUCCESS)
device_driver.AddDeletionResultToReport(
delete_report, [remote_host], failed=[],
error_msgs=[],
resource_name="remote host")
except subprocess.CalledProcessError as e:
delete_report.AddError(str(e))
delete_report.SetStatus(report.Status.FAIL)
return delete_report
def _FetchBuildIdFromFile(cfg, build_target, build_id, filename):
"""Parse and fetch build id from a file based on a pattern.
Verify if one of the system image or emulator binary build id is missing.
If found missing, then update according to the resource file.
Args:
cfg: An AcloudConfig instance.
build_target: Target name.
build_id: Build id, a string, e.g. "2263051", "P2804227"
filename: Name of file containing the build info.
Returns:
A build id or None
"""
build_client = android_build_client.AndroidBuildClient(
auth.CreateCredentials(cfg))
with utils.TempDir() as tempdir:
temp_filename = os.path.join(tempdir, filename)
build_client.DownloadArtifact(build_target, build_id, filename,
temp_filename)
return ParseBuildInfo(temp_filename)
def Cleanup(cfg, expiration_mins):
"""Cleans up stale gce images, gce instances, and disk images in storage.
Args:
cfg: An AcloudConfig instance.
expiration_mins: Integer, resources older than |expiration_mins| will
be cleaned up.
Returns:
A Report instance.
"""
r = report.Report(command="cleanup")
try:
cut_time = (datetime.datetime.now(dateutil.tz.tzlocal()) -
datetime.timedelta(minutes=expiration_mins))
logger.info(
"Cleaning up any gce images/instances and cached build artifacts."
"in google storage that are older than %s", cut_time)
credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
storage_client = gstorage_client.StorageClient(credentials)
# Cleanup expired instances
items = compute_client.ListInstances(zone=cfg.zone)
cleanup_list = [
item["name"]
for item in _FindOldItems(items, cut_time, "creationTimestamp")
]
logger.info("Found expired instances: %s", cleanup_list)
for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
result = compute_client.DeleteInstances(
instances=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT],
zone=cfg.zone)
_AddDeletionResultToReport(r, *result, resource_name="instance")
# Cleanup expired images
items = compute_client.ListImages()
skip_list = cfg.precreated_data_image_map.viewvalues()
cleanup_list = [
item["name"]
for item in _FindOldItems(items, cut_time, "creationTimestamp")
if item["name"] not in skip_list
]
logger.info("Found expired images: %s", cleanup_list)
for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
result = compute_client.DeleteImages(
image_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT])
_AddDeletionResultToReport(r, *result, resource_name="image")
# Cleanup expired disks
# Disks should have been attached to instances with autoDelete=True.
# However, sometimes disks may not be auto deleted successfully.
items = compute_client.ListDisks(zone=cfg.zone)
cleanup_list = [
item["name"]
for item in _FindOldItems(items, cut_time, "creationTimestamp")
if not item.get("users")
]
logger.info("Found expired disks: %s", cleanup_list)
for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
result = compute_client.DeleteDisks(
disk_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT],
zone=cfg.zone)
_AddDeletionResultToReport(r, *result, resource_name="disk")
# Cleanup expired google storage
items = storage_client.List(bucket_name=cfg.storage_bucket_name)
cleanup_list = [
item["name"]
for item in _FindOldItems(items, cut_time, "timeCreated")
]
logger.info("Found expired cached artifacts: %s", cleanup_list)
for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
result = storage_client.DeleteFiles(
bucket_name=cfg.storage_bucket_name,
object_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT])
_AddDeletionResultToReport(
r, *result, resource_name="cached_build_artifact")
# Everything succeeded, write status to report.
if r.status == report.Status.UNKNOWN:
r.SetStatus(report.Status.SUCCESS)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
def CreateAndroidVirtualDevices(cfg,
build_target=None,
build_id=None,
num=1,
gce_image=None,
local_disk_image=None,
cleanup=True,
serial_log_file=None,
logcat_file=None):
"""Creates one or multiple android devices.
Args:
cfg: An AcloudConfig instance.
build_target: Target name, e.g. "gce_x86-userdebug"
build_id: Build id, a string, e.g. "2263051", "P2804227"
num: Number of devices to create.
gce_image: string, if given, will use this gce image
instead of creating a new one.
implies cleanup=False.
local_disk_image: string, path to a local disk image, e.g.
/tmp/avd-system.tar.gz
cleanup: boolean, if True clean up compute engine image and
disk image in storage after creating the instance.
serial_log_file: A path to a file where serial output should
be saved to.
logcat_file: A path to a file where logcat logs should be saved.
Returns:
A Report instance.
"""
r = report.Report(command="create")
credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
compute_client = android_compute_client.AndroidComputeClient(cfg,
credentials)
try:
_CreateSshKeyPairIfNecessary(cfg)
device_pool = AndroidVirtualDevicePool(cfg)
device_pool.CreateDevices(
num,
build_target,
build_id,
gce_image,
local_disk_image,
cleanup,
extra_data_disk_size_gb=cfg.extra_data_disk_size_gb,
precreated_data_image=cfg.precreated_data_image_map.get(
cfg.extra_data_disk_size_gb))
failures = device_pool.WaitForBoot()
# Write result to report.
for device in device_pool.devices:
device_dict = {"ip": device.ip,
"instance_name": device.instance_name}
if device.instance_name in failures:
r.AddData(key="devices_failing_boot", value=device_dict)
r.AddError(str(failures[device.instance_name]))
else:
r.AddData(key="devices", value=device_dict)
if failures:
r.SetStatus(report.Status.BOOT_FAIL)
else:
r.SetStatus(report.Status.SUCCESS)
# Dump serial and logcat logs.
if serial_log_file:
_FetchSerialLogsFromDevices(
compute_client,
instance_names=[d.instance_name for d in device_pool.devices],
port=constants.DEFAULT_SERIAL_PORT,
output_file=serial_log_file)
if logcat_file:
_FetchSerialLogsFromDevices(
compute_client,
instance_names=[d.instance_name for d in device_pool.devices],
port=constants.LOGCAT_SERIAL_PORT,
output_file=logcat_file)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
def CreateGCETypeAVD(cfg,
build_target=None,
build_id=None,
num=1,
gce_image=None,
local_disk_image=None,
cleanup=True,
serial_log_file=None,
autoconnect=False,
report_internal_ip=False,
avd_spec=None):
"""Creates one or multiple gce android devices.
Args:
cfg: An AcloudConfig instance.
build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug"
build_id: Build id, a string, e.g. "2263051", "P2804227"
num: Number of devices to create.
gce_image: string, if given, will use this gce image
instead of creating a new one.
implies cleanup=False.
local_disk_image: string, path to a local disk image, e.g.
/tmp/avd-system.tar.gz
cleanup: boolean, if True clean up compute engine image and
disk image in storage after creating the instance.
serial_log_file: A path to a file where serial output should
be saved to.
autoconnect: Create ssh tunnel(s) and adb connect after device creation.
report_internal_ip: Boolean to report the internal ip instead of
external ip.
avd_spec: AVDSpec object for pass hw_property.
Returns:
A Report instance.
"""
r = report.Report(command="create")
credentials = auth.CreateCredentials(cfg)
compute_client = android_compute_client.AndroidComputeClient(
cfg, credentials)
try:
common_operations.CreateSshKeyPairIfNecessary(cfg)
device_pool = AndroidVirtualDevicePool(cfg)
device_pool.CreateDevices(
num,
build_target,
build_id,
gce_image,
local_disk_image,
cleanup,
extra_data_disk_size_gb=cfg.extra_data_disk_size_gb,
precreated_data_image=cfg.precreated_data_image_map.get(
cfg.extra_data_disk_size_gb),
avd_spec=avd_spec,
extra_scopes=cfg.extra_scopes)
failures = device_pool.WaitForBoot()
# Write result to report.
for device in device_pool.devices:
ip = (device.ip.internal
if report_internal_ip else device.ip.external)
device_dict = {"ip": ip, "instance_name": device.instance_name}
if autoconnect:
forwarded_ports = utils.AutoConnect(
ip_addr=ip,
rsa_key_file=cfg.ssh_private_key_path,
target_vnc_port=constants.GCE_VNC_PORT,
target_adb_port=constants.GCE_ADB_PORT,
ssh_user=_SSH_USER,
client_adb_port=avd_spec.client_adb_port,
extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel)
device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port
device_dict[constants.ADB_PORT] = forwarded_ports.adb_port
if avd_spec.unlock_screen:
AdbTools(forwarded_ports.adb_port).AutoUnlockScreen()
if device.instance_name in failures:
r.AddData(key="devices_failing_boot", value=device_dict)
r.AddError(str(failures[device.instance_name]))
else:
r.AddData(key="devices", value=device_dict)
if failures:
r.SetStatus(report.Status.BOOT_FAIL)
else:
r.SetStatus(report.Status.SUCCESS)
# Dump serial logs.
if serial_log_file:
_FetchSerialLogsFromDevices(
compute_client,
instance_names=[d.instance_name for d in device_pool.devices],
port=constants.DEFAULT_SERIAL_PORT,
output_file=serial_log_file)
except errors.DriverError as e:
r.AddError(str(e))
r.SetStatus(report.Status.FAIL)
return r
