def GetPackageMerkleRoot(far_file_path):
"""Returns a package's Merkle digest."""
# The digest is the first word on the first line of the merkle tool's output.
merkle_tool = GetHostToolPathFromPlatform('merkleroot')
output = subprocess.check_output([merkle_tool, far_file_path])
return output.splitlines()[0].split()[0]
def RunSymbolizer(input_file, output_file, build_ids_files):
"""Starts a symbolizer process.
input_file: Input file to be symbolized.
output_file: Output file for symbolizer stdout and stderr.
build_ids_file: Path to the ids.txt file which maps build IDs to
unstripped binaries on the filesystem.
Returns a Popen object for the started process."""
if (GetHostArchFromPlatform() == 'arm64' and
os.path.isfile(ARM64_DOCKER_LLVM_SYMBOLIZER_PATH)):
llvm_symbolizer_path = ARM64_DOCKER_LLVM_SYMBOLIZER_PATH
else:
llvm_symbolizer_path = os.path.join(SDK_ROOT, os.pardir, os.pardir,
'llvm-build', 'Release+Asserts', 'bin',
'llvm-symbolizer')
symbolizer = GetHostToolPathFromPlatform('symbolize')
symbolizer_cmd = [symbolizer,
'-ids-rel', '-llvm-symbolizer', llvm_symbolizer_path,
'-build-id-dir', os.path.join(SDK_ROOT, '.build-id')]
for build_ids_file in build_ids_files:
symbolizer_cmd.extend(['-ids', build_ids_file])
logging.info('Running "%s".' % ' '.join(symbolizer_cmd))
return subprocess.Popen(symbolizer_cmd, stdin=input_file, stdout=output_file,
stderr=subprocess.STDOUT, close_fds=True)
def GetCompressedSize(file_path):
"""Measures file size after blobfs compression."""
compressor_path = GetHostToolPathFromPlatform('blobfs-compression')
try:
temp_dir = tempfile.mkdtemp()
compressed_file_path = os.path.join(temp_dir,
os.path.basename(file_path))
proc = subprocess.Popen(
[compressor_path, file_path, compressed_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
proc.wait()
compressor_output = proc.stdout.read()
if proc.returncode != 0:
print(compressor_output, file=sys.stderr)
raise Exception('Error while running %s' % compressor_path)
finally:
shutil.rmtree(temp_dir)
# Match a compressed bytes total from blobfs-compression output like
# Wrote 360830 bytes (40% compression)
blobfs_compressed_bytes_re = r'Wrote\s+(?P<bytes>\d+)\s+bytes'
match = re.search(blobfs_compressed_bytes_re, compressor_output)
if not match:
print(compressor_output, file=sys.stderr)
raise Exception('Could not get compressed bytes for %s' % file_path)
# Round the compressed file size up to an integer number of blobfs blocks.
BLOBFS_BLOCK_SIZE = 8192 # Fuchsia's blobfs file system uses 8KiB blocks.
blob_bytes = int(match.group('bytes'))
return math.ceil(blob_bytes / BLOBFS_BLOCK_SIZE) * BLOBFS_BLOCK_SIZE
def _ProvisionDevice(self):
"""Pave a device with a generic image of Fuchsia."""
bootserver_path = GetHostToolPathFromPlatform('bootserver')
bootserver_command = [
bootserver_path, '-1', '--fvm',
EnsurePathExists(
boot_data.GetTargetFile('storage-sparse.blk',
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC)),
EnsurePathExists(
boot_data.GetBootImage(self._out_dir, self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC))
]
if self._node_name:
bootserver_command += ['-n', self._node_name]
bootserver_command += ['--']
bootserver_command += boot_data.GetKernelArgs()
logging.debug(' '.join(bootserver_command))
_, stdout = SubprocessCallWithTimeout(bootserver_command,
silent=False,
timeout_secs=300)
self._ParseNodename(stdout)
# Update the target's hash to match the current tree's.
self.PutFile(os.path.join(SDK_ROOT, '.hash'), TARGET_HASH_FILE_PATH)
def _Discover(self):
"""Queries mDNS for the IP address of a booted Fuchsia instance whose name
matches |_node_name| on the local area network. If |_node_name| isn't
specified, and there is only one device on the network, then returns the
IP address of that advice.
Sets |_host_name| and returns True if the device was found,
or waits up to |timeout| seconds and returns False if the device couldn't
be found."""
dev_finder_path = GetHostToolPathFromPlatform('device-finder')
if self._node_name:
command = [
dev_finder_path,
'resolve',
'-timeout',
"%ds" % BOOT_DISCOVERY_TIMEOUT_SECS / BOOT_DISCOVERY_ATTEMPTS,
'-device-limit',
'1', # Exit early as soon as a host is found.
self._node_name
]
else:
command = [
dev_finder_path, 'list', '-full', '-timeout',
"%ds" % BOOT_DISCOVERY_TIMEOUT_SECS / BOOT_DISCOVERY_ATTEMPTS
]
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'))
output = set(proc.communicate()[0].strip().split('\n'))
if proc.returncode != 0:
return False
if self._node_name:
# Handle the result of "device-finder resolve".
self._host = output.pop().strip()
else:
name_host_pairs = [x.strip().split(' ') for x in output]
# Handle the output of "device-finder list".
if len(name_host_pairs) > 1:
print('More than one device was discovered on the network.')
print('Use --node-name <name> to specify the device to use.')
print('\nList of devices:')
for pair in name_host_pairs:
print(' ' + pair[1])
print()
raise Exception('Ambiguous target device specification.')
assert len(name_host_pairs) == 1
self._host, self._node_name = name_host_pairs[0]
logging.info('Found device "%s" at address %s.' % (self._node_name,
self._host))
return True
def ExtractFarFile(file_path, extract_dir):
"""Extracts contents of a Fuchsia archive file to the specified directory."""
far_tool = GetHostToolPathFromPlatform('far')
subprocess.check_call([
far_tool, 'extract',
'--archive=%s' % file_path,
'--output=%s' % extract_dir
])
def _Discover(self):
"""Queries mDNS for the IP address of a booted Fuchsia instance whose name
matches |_node_name| on the local area network. If |_node_name| isn't
specified, and there is only one device on the network, then returns the
IP address of that advice.
Sets |_host_name| and returns True if the device was found,
or waits up to |timeout| seconds and returns False if the device couldn't
be found."""
dev_finder_path = GetHostToolPathFromPlatform('device-finder')
with open(os.devnull, 'w') as devnull:
if self._node_name:
command = [
dev_finder_path,
'resolve',
'-device-limit',
'1', # Exit early as soon as a host is found.
self._node_name
]
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=devnull,
text=True)
else:
proc = self.RunFFXCommand(['target', 'list', '-f', 'simple'],
stdout=subprocess.PIPE,
stderr=devnull,
text=True)
output = set(proc.communicate()[0].strip().split('\n'))
if proc.returncode != 0:
return False
if self._node_name:
# Handle the result of "device-finder resolve".
self._host = output.pop().strip()
else:
name_host_pairs = [x.strip().split(' ') for x in output]
if len(name_host_pairs) > 1:
raise Exception('More than one device was discovered on the network. '
'Use --node-name <name> to specify the device to use.'
'List of devices: {}'.format(output))
assert len(name_host_pairs) == 1
# Check if device has both address and name.
if len(name_host_pairs[0]) < 2:
return False
self._host, self._node_name = name_host_pairs[0]
logging.info('Found device "%s" at address %s.' % (self._node_name,
self._host))
return True
def __ProvisionDevice(self):
"""Netboots a device with Fuchsia. If |_node_name| is set, then only a
device with a matching node name is used.
The device is up and reachable via SSH when the function is successfully
completes."""
bootserver_path = GetHostToolPathFromPlatform('bootserver')
bootserver_command = [
bootserver_path, '-1', '--fvm',
EnsurePathExists(
boot_data.GetTargetFile('storage-sparse.blk',
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC)),
EnsurePathExists(
boot_data.GetBootImage(self._output_dir,
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC))
]
if self._node_name:
bootserver_command += ['-n', self._node_name]
bootserver_command += ['--']
bootserver_command += boot_data.GetKernelArgs(self._output_dir)
logging.debug(' '.join(bootserver_command))
stdout = subprocess.check_output(bootserver_command,
stderr=subprocess.STDOUT)
# Parse the nodename from bootserver stdout.
m = re.search(r'.*Proceeding with nodename (?P<nodename>.*)$', stdout,
re.MULTILINE)
if not m:
raise Exception('Couldn\'t parse nodename from bootserver output.')
self._node_name = m.groupdict()['nodename']
logging.info('Booted device "%s".' % self._node_name)
# Repeatdly query mDNS until we find the device, or we hit the timeout of
# DISCOVERY_TIMEOUT_SECS.
logging.info('Waiting for device to join network.')
for _ in xrange(_BOOT_DISCOVERY_ATTEMPTS):
if self.__Discover():
break
if not self._host:
raise Exception('Device %s couldn\'t be discovered via mDNS.' %
self._node_name)
self._WaitUntilReady()
# Update the target's hash to match the current tree's.
self.PutFile(os.path.join(SDK_ROOT, '.hash'), TARGET_HASH_FILE_PATH)
def _Discover(self):
"""Queries mDNS for the IP address of a booted Fuchsia instance whose name
matches |_node_name| on the local area network. If |_node_name| isn't
specified, and there is only one device on the network, then returns the
IP address of that advice.
Sets |_host_name| and returns True if the device was found,
or waits up to |timeout| seconds and returns False if the device couldn't
be found."""
ffx_path = GetHostToolPathFromPlatform('ffx')
# Ffx command to list targets in short form.
# Example output: fe80::4415:3606:fb52:e2bc%zx pecan-guru-clerk-rhyme.
ffx_target_list_command = [
ffx_path, '-T',
"%d" % _FFX_TIMEOUT_LIMIT_SECS, 'target', 'list', '-f', 's'
]
# List only a specific device if node name is present.
if self._node_name:
ffx_target_list_command.append(self._node_name)
proc = subprocess.Popen(ffx_target_list_command,
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'))
output = set(proc.communicate()[0].strip().split('\n'))
if proc.returncode != 0:
return False
name_host_pairs = [x.strip().split(' ') for x in output]
# Handle the output of "ffx target list -f s".
if len(name_host_pairs) > 1:
print('More than one device was discovered on the network.')
print('Use --node-name <name> to specify the device to use.')
print('\nList of devices:')
for pair in name_host_pairs:
print(' ' + pair[1])
print()
raise Exception('Ambiguous target device specification.')
assert len(name_host_pairs) == 1
self._host, self._node_name = name_host_pairs[0]
logging.info('Found device "%s" at address %s.' % (self._node_name,
self._host))
return True
def ExtractFarFile(file_path, extract_dir):
"""Extracts contents of a Fuchsia archive file to the specified directory."""
far_tool = GetHostToolPathFromPlatform('far')
if not os.path.isfile(far_tool):
raise Exception('Could not find FAR host tool "%s".' % far_tool)
if not os.path.isfile(file_path):
raise Exception('Could not find FAR file "%s".' % file_path)
subprocess.check_call([
far_tool, 'extract',
'--archive=%s' % file_path,
'--output=%s' % extract_dir
])
def _ProvisionDevice(self):
"""Pave a device with a generic image of Fuchsia."""
bootserver_path = GetHostToolPathFromPlatform('bootserver')
bootserver_command = [
bootserver_path, '-1', '--fvm',
EnsurePathExists(
boot_data.GetTargetFile('storage-sparse.blk',
self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC)),
EnsurePathExists(
boot_data.GetBootImage(self._output_dir, self._GetTargetSdkArch(),
boot_data.TARGET_TYPE_GENERIC))
]
if self._node_name:
bootserver_command += ['-n', self._node_name]
bootserver_command += ['--']
bootserver_command += boot_data.GetKernelArgs(self._output_dir)
logging.debug(' '.join(bootserver_command))
_, stdout = SubprocessCallWithTimeout(bootserver_command,
silent=False,
timeout_secs=300)
self._ParseNodename(stdout)
# Start loglistener to save system logs.
if self._system_log_file:
self._StartLoglistener()
# Repeatdly query mDNS until we find the device, or we hit
# BOOT_DISCOVERY_ATTEMPTS
logging.info('Waiting for device to join network.')
for _ in xrange(device_target.BOOT_DISCOVERY_ATTEMPTS):
if self.__Discover():
break
if not self._host:
raise Exception("Device %s couldn't be discovered via mDNS." %
self._node_name)
self._WaitUntilReady()
# Update the target's hash to match the current tree's.
self.PutFile(os.path.join(SDK_ROOT, '.hash'), TARGET_HASH_FILE_PATH)
def RunSymbolizer(input_file, output_file, build_ids_files):
"""Starts a symbolizer process.
input_file: Input file to be symbolized.
output_file: Output file for symbolizer stdout and stderr.
build_ids_file: Path to the ids.txt file which maps build IDs to
unstripped binaries on the filesystem.
Returns a Popen object for the started process."""
symbolizer = GetHostToolPathFromPlatform('symbolize')
llvm_symbolizer_path = _GetLLVMSymbolizerPath()
symbolizer_cmd = [symbolizer,
'-ids-rel', '-llvm-symbolizer', llvm_symbolizer_path,
'-build-id-dir', os.path.join(SDK_ROOT, '.build-id')]
for build_ids_file in build_ids_files:
symbolizer_cmd.extend(['-ids', build_ids_file])
logging.info('Running "%s".' % ' '.join(symbolizer_cmd))
return subprocess.Popen(symbolizer_cmd, stdin=input_file, stdout=output_file,
stderr=subprocess.STDOUT, close_fds=True)
def RunSymbolizer(input_fd, output_fd, ids_txt_paths):
"""Starts a symbolizer process.
input_fd: Input file to be symbolized.
output_fd: Output file for symbolizer stdout and stderr.
ids_txt_paths: Path to the ids.txt files which map build IDs to
unstripped binaries on the filesystem.
Returns a Popen object for the started process."""
symbolizer = GetHostToolPathFromPlatform('symbolizer')
symbolizer_cmd = [
symbolizer, '--omit-module-lines', '--build-id-dir',
os.path.join(SDK_ROOT, '.build-id')
]
for ids_txt in ids_txt_paths:
symbolizer_cmd.extend(['--ids-txt', ids_txt])
logging.debug('Running "%s".' % ' '.join(symbolizer_cmd))
return subprocess.Popen(symbolizer_cmd,
stdin=input_fd,
stdout=output_fd,
stderr=subprocess.STDOUT,
close_fds=True)