def __init__(self, target):
AmberRepo.__init__(self, target)
self._with_count = 0
self._amber_root = tempfile.mkdtemp()
pm_tool = common.GetHostToolPathFromPlatform('pm')
subprocess.check_call([pm_tool, 'newrepo', '-repo', self._amber_root])
logging.info('Creating and serving temporary Amber root: {}.'.format(
self._amber_root))
serve_port = common.GetAvailableTcpPort()
self._pm_serve_task = subprocess.Popen([
pm_tool, 'serve', '-d',
os.path.join(self._amber_root, 'repository'), '-l',
':%d' % serve_port, '-q'
])
# Block until "pm serve" starts serving HTTP traffic at |serve_port|.
timeout = time.time() + _PM_SERVE_LIVENESS_TIMEOUT_SECS
while True:
try:
urllib2.urlopen('http://localhost:%d' % serve_port,
timeout=1).read()
break
except urllib2.URLError:
logging.info('Waiting until \'pm serve\' is up...')
if time.time() >= timeout:
raise Exception('Timed out while waiting for \'pm serve\'.')
time.sleep(1)
remote_port = common.ConnectPortForwardingTask(target, serve_port, 0)
self._RegisterAmberRepository(self._amber_root, remote_port)
def InstallPackage(self, package_path, package_name, package_deps):
"""Installs a package and it's dependencies on the device. If the package is
already installed then it will be updated to the new version.
package_path: Path to the .far file to be installed.
package_name: Package name.
package_deps: List of .far files with the packages that the main package
depends on. These packages are installed or updated as well.
"""
try:
tuf_root = tempfile.mkdtemp()
pm_serve_task = None
subprocess.check_call([_PM, 'newrepo', '-repo', tuf_root])
# Serve the |tuf_root| using 'pm serve' and configure the target to pull
# from it.
serve_port = common.GetAvailableTcpPort()
pm_serve_task = subprocess.Popen([
_PM, 'serve', '-d',
os.path.join(tuf_root, 'repository'), '-l',
':%d' % serve_port, '-q'
])
# Publish all packages to the serving TUF repository under |tuf_root|.
all_packages = [package_path] + package_deps
for next_package_path in all_packages:
_PublishPackage(tuf_root, next_package_path)
_WaitForPmServeToBeReady(serve_port)
remote_port = common.ConnectPortForwardingTask(self, serve_port, 0)
self._RegisterAmberRepository(tuf_root, remote_port)
# Install all packages.
for next_package_path in all_packages:
install_package_name, package_version = \
_GetPackageInfo(next_package_path)
logging.info('Installing %s version %s.' %
(install_package_name, package_version))
return_code = self.RunCommand(
[
'amberctl', 'get_up', '-n', install_package_name, '-v',
package_version
],
timeout_secs=_INSTALL_TIMEOUT_SECS)
if return_code != 0:
raise Exception('Error while installing %s.' %
install_package_name)
finally:
self._UnregisterAmberRepository()
if pm_serve_task:
pm_serve_task.kill()
shutil.rmtree(tuf_root)
def RunPackage(output_dir, target, package_path, package_name, package_deps,
package_args, args):
"""Copies the Fuchsia package at |package_path| to the target,
executes it with |package_args|, and symbolizes its output.
output_dir: The path containing the build output files.
target: The deployment Target object that will run the package.
package_path: The path to the .far package file.
package_name: The name of app specified by package metadata.
package_args: The arguments which will be passed to the Fuchsia process.
args: Structure of arguments to configure how the package will be run.
Returns the exit code of the remote package process."""
system_logger = (_AttachKernelLogReader(target)
if args.system_logging else None)
try:
if system_logger:
# Spin up a thread to asynchronously dump the system log to stdout
# for easier diagnoses of early, pre-execution failures.
log_output_quit_event = multiprocessing.Event()
log_output_thread = threading.Thread(
target=lambda: _DrainStreamToStdout(system_logger.stdout,
log_output_quit_event))
log_output_thread.daemon = True
log_output_thread.start()
tuf_root = tempfile.mkdtemp()
pm_serve_task = None
# Publish all packages to the serving TUF repository under |tuf_root|.
subprocess.check_call([PM, 'newrepo', '-repo', tuf_root])
all_packages = [package_path] + package_deps
for next_package_path in all_packages:
PublishPackage(tuf_root, next_package_path)
# Serve the |tuf_root| using 'pm serve' and configure the target to pull
# from it.
# TODO(kmarshall): Use -q to suppress pm serve output once blob push
# is confirmed to be running stably on bots.
serve_port = common.GetAvailableTcpPort()
pm_serve_task = subprocess.Popen([
PM, 'serve', '-d',
os.path.join(tuf_root, 'repository'), '-l',
':%d' % serve_port, '-q'
])
remote_port = common.ConnectPortForwardingTask(target, serve_port, 0)
_RegisterAmberRepository(target, tuf_root, remote_port)
# Install all packages.
for next_package_path in all_packages:
install_package_name, package_version = GetPackageInfo(
next_package_path)
logging.info('Installing %s version %s.' %
(install_package_name, package_version))
return_code = target.RunCommand([
'amber_ctl', 'get_up', '-n', install_package_name, '-v',
package_version
])
if return_code != 0:
raise Exception('Error while installing %s.' %
install_package_name)
if system_logger:
log_output_quit_event.set()
log_output_thread.join(timeout=_JOIN_TIMEOUT_SECS)
if args.install_only:
logging.info('Installation complete.')
return
logging.info('Running application.')
command = ['run', _GetComponentUri(package_name)] + package_args
process = target.RunCommandPiped(command,
stdin=open(os.devnull, 'r'),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if system_logger:
task_output = _ReadMergedLines(
[process.stdout, system_logger.stdout])
else:
task_output = process.stdout
if args.symbolizer_config:
# Decorate the process output stream with the symbolizer.
output = FilterStream(task_output, package_name,
args.symbolizer_config, output_dir)
else:
logging.warn('Symbolization is DISABLED.')
output = process.stdout
for next_line in output:
print next_line.rstrip()
process.wait()
if process.returncode == 0:
logging.info('Process exited normally with status code 0.')
else:
# The test runner returns an error status code if *any* tests fail,
# so we should proceed anyway.
logging.warning('Process exited with status code %d.' %
process.returncode)
finally:
if system_logger:
logging.info('Terminating kernel log reader.')
log_output_quit_event.set()
log_output_thread.join()
system_logger.kill()
_UnregisterAmberRepository(target)
if pm_serve_task:
pm_serve_task.kill()
shutil.rmtree(tuf_root)
return process.returncode
def Start(self):
qemu_path = os.path.join(GetQemuRootForPlatform(), 'bin',
'qemu-system-' + self._GetTargetSdkLegacyArch())
kernel_args = boot_data.GetKernelArgs(self._output_dir)
# TERM=dumb tells the guest OS to not emit ANSI commands that trigger
# noisy ANSI spew from the user's terminal emulator.
kernel_args.append('TERM=dumb')
# Enable logging to the serial port. This is a temporary fix to investigate
# the root cause for https://crbug.com/869753 .
kernel_args.append('kernel.serial=legacy')
qemu_command = [qemu_path,
'-m', str(self._ram_size_mb),
'-nographic',
'-kernel', EnsurePathExists(
boot_data.GetTargetFile('qemu-kernel.kernel',
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_QEMU)),
'-initrd', EnsurePathExists(
boot_data.GetBootImage(self._output_dir, self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_QEMU)),
'-smp', str(self._cpu_cores),
# Attach the blobstore and data volumes. Use snapshot mode to discard
# any changes.
'-snapshot',
'-drive', 'file=%s,format=qcow2,if=none,id=blobstore,snapshot=on' %
EnsurePathExists(
os.path.join(self._output_dir, 'fvm.blk.qcow2')),
'-device', 'virtio-blk-pci,drive=blobstore',
# Use stdio for the guest OS only; don't attach the QEMU interactive
# monitor.
'-serial', 'stdio',
'-monitor', 'none',
'-append', ' '.join(kernel_args)
]
# Configure the machine to emulate, based on the target architecture.
if self._target_cpu == 'arm64':
qemu_command.extend([
'-machine','virt',
])
netdev_type = 'virtio-net-pci'
else:
qemu_command.extend([
'-machine', 'q35',
])
netdev_type = 'e1000'
# Configure the CPU to emulate.
# On Linux, we can enable lightweight virtualization (KVM) if the host and
# guest architectures are the same.
enable_kvm = self._require_kvm or (sys.platform.startswith('linux') and (
(self._target_cpu == 'arm64' and platform.machine() == 'aarch64') or
(self._target_cpu == 'x64' and platform.machine() == 'x86_64')) and
os.access('/dev/kvm', os.R_OK | os.W_OK))
if enable_kvm:
qemu_command.extend(['-enable-kvm', '-cpu', 'host,migratable=no'])
else:
logging.warning('Unable to launch QEMU with KVM acceleration.')
if self._target_cpu == 'arm64':
qemu_command.extend(['-cpu', 'cortex-a53'])
else:
qemu_command.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])
# Configure virtual network. It is used in the tests to connect to
# testserver running on the host.
netdev_config = 'user,id=net0,net=%s,dhcpstart=%s,host=%s' % \
(GUEST_NET, GUEST_IP_ADDRESS, HOST_IP_ADDRESS)
self._host_ssh_port = common.GetAvailableTcpPort()
netdev_config += ",hostfwd=tcp::%s-:22" % self._host_ssh_port
qemu_command.extend([
'-netdev', netdev_config,
'-device', '%s,netdev=net0,mac=%s' % (netdev_type, GUEST_MAC_ADDRESS),
])
# We pass a separate stdin stream to qemu. Sharing stdin across processes
# leads to flakiness due to the OS prematurely killing the stream and the
# Python script panicking and aborting.
# The precise root cause is still nebulous, but this fix works.
# See crbug.com/741194.
logging.debug('Launching QEMU.')
logging.debug(' '.join(qemu_command))
# Zircon sends debug logs to serial port (see kernel.serial=legacy flag
# above). Serial port is redirected to a file through QEMU stdout.
# Unless a |_system_log_file| is explicitly set, we output the kernel serial
# log to a temporary file, and print that out if we are unable to connect to
# the QEMU guest, to make it easier to diagnose connectivity issues.
temporary_system_log_file = None
if self._system_log_file:
stdout = self._system_log_file
stderr = subprocess.STDOUT
else:
temporary_system_log_file = tempfile.NamedTemporaryFile('w')
stdout = temporary_system_log_file
stderr = sys.stderr
self._qemu_process = subprocess.Popen(qemu_command, stdin=open(os.devnull),
stdout=stdout, stderr=stderr)
try:
self._WaitUntilReady();
except target.FuchsiaTargetException:
if temporary_system_log_file:
logging.info("Kernel logs:\n" +
open(temporary_system_log_file.name, 'r').read())
raise
def _BuildCommand(self):
boot_data.ProvisionSSH()
self._host_ssh_port = common.GetAvailableTcpPort()
kernel_image = common.EnsurePathExists(
boot_data.GetTargetFile('qemu-kernel.kernel', self._image_arch,
self._image_type))
zbi_image = common.EnsurePathExists(
boot_data.GetTargetFile('zircon-a.zbi', self._image_arch,
self._image_type))
fvm_image = common.EnsurePathExists(
boot_data.GetTargetFile('storage-full.blk', self._image_arch,
self._image_type))
aemu_path = common.EnsurePathExists(
os.path.join(common.GetEmuRootForPlatform(self.EMULATOR_NAME),
'emulator'))
emu_command = [
self._FVDL_PATH,
'--sdk',
'start',
'--nointeractive',
# Host port mapping for user-networking mode.
'--port-map',
'hostfwd=tcp::{}-:22'.format(self._host_ssh_port),
# no-interactive requires a --vdl-output flag to shutdown the emulator.
'--vdl-output',
self._vdl_output_file.name,
'-c',
' '.join(boot_data.GetKernelArgs()),
# Use existing images instead of downloading new ones.
'--kernel-image',
kernel_image,
'--zbi-image',
zbi_image,
'--fvm-image',
fvm_image,
'--image-architecture',
self._target_cpu,
# Use an existing emulator checked out by Chromium.
'--aemu-path',
aemu_path,
# Use this flag and temp file to define ram size.
'--device-proto',
self._device_proto_file.name,
'--cpu-count',
str(self._cpu_cores)
]
self._ConfigureEmulatorLog(emu_command)
if not self._require_kvm:
emu_command.append('--noacceleration')
if not self._enable_graphics:
emu_command.append('--headless')
if self._hardware_gpu:
emu_command.append('--host-gpu')
if self._with_network:
emu_command.append('-N')
return emu_command
def _BuildQemuConfig(self):
boot_data.AssertBootImagesExist(self._GetTargetSdkArch(), 'qemu')
emu_command = [
'-kernel',
EnsurePathExists(
boot_data.GetTargetFile('qemu-kernel.kernel',
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_QEMU)),
'-initrd',
EnsurePathExists(
boot_data.GetBootImage(self._output_dir,
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_QEMU)),
'-m',
str(self._ram_size_mb),
'-smp',
str(self._cpu_cores),
# Attach the blobstore and data volumes. Use snapshot mode to discard
# any changes.
'-snapshot',
'-drive',
'file=%s,format=qcow2,if=none,id=blobstore,snapshot=on' %
_EnsureBlobstoreQcowAndReturnPath(self._output_dir,
self._GetTargetSdkArch(),
self._qemu_img_retries),
'-device',
'virtio-blk-pci,drive=blobstore',
# Use stdio for the guest OS only; don't attach the QEMU interactive
# monitor.
'-serial',
'stdio',
'-monitor',
'none',
]
# Configure the machine to emulate, based on the target architecture.
if self._target_cpu == 'arm64':
emu_command.extend([
'-machine',
'virt,gic_version=3',
])
netdev_type = 'virtio-net-pci'
else:
emu_command.extend([
'-machine',
'q35',
])
netdev_type = 'e1000'
# Configure virtual network. It is used in the tests to connect to
# testserver running on the host.
netdev_config = 'user,id=net0,net=%s,dhcpstart=%s,host=%s' % \
(GUEST_NET, GUEST_IP_ADDRESS, HOST_IP_ADDRESS)
self._host_ssh_port = common.GetAvailableTcpPort()
netdev_config += ",hostfwd=tcp::%s-:22" % self._host_ssh_port
emu_command.extend([
'-netdev',
netdev_config,
'-device',
'%s,netdev=net0,mac=%s' % (netdev_type, GUEST_MAC_ADDRESS),
])
# Configure the CPU to emulate.
# On Linux, we can enable lightweight virtualization (KVM) if the host and
# guest architectures are the same.
if self._IsKvmEnabled():
kvm_command = ['-enable-kvm', '-cpu']
if self._target_cpu == 'arm64':
kvm_command.append('host')
else:
kvm_command.append('host,migratable=no')
else:
logging.warning('Unable to launch %s with KVM acceleration.' %
(self._emu_type) + 'The guest VM will be slow.')
if self._target_cpu == 'arm64':
kvm_command = ['-cpu', 'cortex-a53']
else:
kvm_command = ['-cpu', 'Haswell,+smap,-check,-fsgsbase']
emu_command.extend(kvm_command)
kernel_args = boot_data.GetKernelArgs(self._output_dir)
# TERM=dumb tells the guest OS to not emit ANSI commands that trigger
# noisy ANSI spew from the user's terminal emulator.
kernel_args.append('TERM=dumb')
# Construct kernel cmd line
kernel_args.append('kernel.serial=legacy')
# Don't 'reboot' the emulator if the kernel crashes
kernel_args.append('kernel.halt-on-panic=true')
emu_command.extend(['-append', ' '.join(kernel_args)])
return emu_command
