def _UpdateChroot(self):
"""Update the chroot if needed."""
if self.chroot_update:
# Run chroot update hooks.
cmd = [
os.path.join(constants.CROSUTILS_DIR,
'run_chroot_version_hooks')
]
cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
# Update toolchains.
cmd = [
os.path.join(constants.CHROMITE_BIN_DIR,
'cros_setup_toolchains')
]
cros_build_lib.SudoRunCommand(cmd, debug_level=logging.DEBUG)
# Update the host before updating the board.
self._Emerge(list(_HOST_PKGS))
# Automatically discard all CONFIG_PROTECT'ed files. Those that are
# protected should not be overwritten until the variable is changed.
# Autodiscard is option "-9" followed by the "YES" confirmation.
cros_build_lib.SudoRunCommand(['etc-update'],
input='-9\nYES\n',
debug_level=logging.DEBUG)
self.chroot_update = False
def CopyImageToDevice(self, image, device):
"""Copies |image| to the removable |device|.
Args:
image: Path to the image to copy.
device: Device to copy to.
"""
cmd = [
'dd',
'if=%s' % image,
'of=%s' % device, 'bs=4M', 'iflag=fullblock', 'oflag=sync'
]
if logging.getLogger().getEffectiveLevel() <= logging.NOTICE:
op = UsbImagerOperation(image)
op.Run(cros_build_lib.SudoRunCommand,
cmd,
debug_level=logging.NOTICE,
update_period=0.5)
else:
cros_build_lib.SudoRunCommand(
cmd,
debug_level=logging.NOTICE,
print_cmd=logging.getLogger().getEffectiveLevel() <
logging.NOTICE)
cros_build_lib.SudoRunCommand(['sync'], debug_level=self.debug_level)
def UpdateChroot(brick=None, board=None, update_host_packages=True):
"""Update the chroot."""
# Run chroot update hooks.
logging.notice('Updating the chroot. This may take several minutes.')
cmd = [os.path.join(constants.CROSUTILS_DIR, 'run_chroot_version_hooks')]
cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
# Update toolchains.
cmd = [os.path.join(constants.CHROMITE_BIN_DIR, 'cros_setup_toolchains')]
if brick:
cmd += [
'--targets=bricks',
'--include-bricks=%s' % brick.brick_locator
]
elif board:
cmd += ['--targets=boards', '--include-boards=%s' % board]
cros_build_lib.SudoRunCommand(cmd, debug_level=logging.DEBUG)
# Update the host before updating the board.
if update_host_packages:
Emerge(list(_HOST_PKGS), '/', rebuild_deps=False)
# Automatically discard all CONFIG_PROTECT'ed files. Those that are
# protected should not be overwritten until the variable is changed.
# Autodiscard is option "-9" followed by the "YES" confirmation.
cros_build_lib.SudoRunCommand(['etc-update'],
input='-9\nYES\n',
debug_level=logging.DEBUG)
def _GetFd(self):
if self.world_writable:
create = True
try:
create = stat.S_IMODE(os.stat(self.path).st_mode) != 0o666
except OSError as e:
if e.errno != errno.ENOENT:
raise
if create:
osutils.SafeMakedirs(os.path.dirname(self.path), sudo=True)
cros_build_lib.SudoRunCommand(['touch', self.path],
print_cmd=False)
cros_build_lib.SudoRunCommand(['chmod', '666', self.path],
print_cmd=False)
# If we're on py3.4 and this attribute is exposed, use it to close
# the threading race between open and fcntl setting; this is
# extremely paranoid code, but might as well.
cloexec = getattr(os, 'O_CLOEXEC', 0)
# There exist race conditions where the lock may be created by
# root, thus denying subsequent accesses from others. To prevent
# this, we create the lock with mode 0o666.
try:
value = os.umask(000)
fd = os.open(self.path, os.W_OK | os.O_CREAT | cloexec, 0o666)
finally:
os.umask(value)
return fd
def _CleanupFiles(recreate):
"""Cleanup VM_DIR.
Args:
recreate: recreate VM_DIR.
"""
cros_build_lib.SudoRunCommand(['rm', '-rf', VM.VM_DIR])
if recreate:
cros_build_lib.SudoRunCommand(['mkdir', VM.VM_DIR])
cros_build_lib.SudoRunCommand(['chmod', '777', VM.VM_DIR])
def MountUsb():
"""Mount the USB device."""
# Unmount |device| as it may already be mounted in the wrong location.
cmd = ['umount', device]
cros_build_lib.SudoRunCommand(cmd, error_code_ok=True)
# Mount |device| in the correct location.
cmd = ['mount', device, common.MOBLAB_MOUNTPOINT, '-o',
'exec,dev,nosuid']
cros_build_lib.SudoRunCommand(cmd)
def PerformStage(self):
new_chroot_dir = 'new-sdk-chroot'
tarball_location = os.path.join(self._build_root, SDK_TARBALL_NAME)
new_chroot_args = ['--chroot', new_chroot_dir]
if self._run.options.chrome_root:
new_chroot_args += ['--chrome_root', self._run.options.chrome_root]
# Build a new SDK using the provided tarball.
chroot_args = new_chroot_args + [
'--download', '--replace', '--nousepkg', '--url',
'file://' + tarball_location
]
cros_build_lib.RunCommand([],
cwd=self._build_root,
enter_chroot=True,
chroot_args=chroot_args,
extra_env=self._portage_extra_env)
# Inject the toolchain binpkgs from the previous sdk build. On end user
# systems, they'd be fetched from the binpkg mirror, but we don't have one
# set up for this local build.
pkgdir = os.path.join('var', 'lib', 'portage', 'pkgs')
old_pkgdir = os.path.join(self._build_root,
constants.DEFAULT_CHROOT_DIR, pkgdir)
new_pkgdir = os.path.join(self._build_root, new_chroot_dir, pkgdir)
osutils.SafeMakedirs(new_pkgdir, sudo=True)
cros_build_lib.SudoRunCommand(
['cp', '-r'] + glob.glob(os.path.join(old_pkgdir, '*')) +
[new_pkgdir])
# Now install those toolchains in the new chroot. We skip the chroot
# upgrade below which means we need to install the toolchain manually.
cmd = [
'cros_setup_toolchains', '--targets=boards',
'--include-boards=%s' % ','.join(self._boards)
]
commands.RunBuildScript(self._build_root,
cmd,
chromite_cmd=True,
enter_chroot=True,
sudo=True,
chroot_args=new_chroot_args,
extra_env=self._portage_extra_env)
# Build all the boards with the new sdk.
for board in self._boards:
logging.PrintBuildbotStepText(board)
commands.SetupBoard(self._build_root,
board,
usepkg=True,
chroot_upgrade=False,
extra_env=self._portage_extra_env,
chroot_args=new_chroot_args)
commands.Build(self._build_root,
board,
build_autotest=True,
usepkg=False,
chrome_binhost_only=False,
extra_env=self._portage_extra_env,
chroot_args=new_chroot_args)
def main(argv):
options = ParseArgs(argv)
if not cros_build_lib.IsInsideChroot():
raise commandline.ChrootRequiredError()
if os.geteuid() != 0:
cros_build_lib.SudoRunCommand(sys.argv)
return
# sysroot must have a trailing / as the tree dictionary produced by
# create_trees in indexed with a trailing /.
sysroot = cros_build_lib.GetSysroot(options.board) + '/'
trees = create_trees(target_root=sysroot, config_root=sysroot)
vartree = trees[sysroot]['vartree']
cache_dir = os.path.join(path_util.FindCacheDir(),
'cros_install_debug_syms-v' + CACHE_VERSION)
if options.clearcache:
osutils.RmDir(cache_dir, ignore_missing=True)
binhost_cache = None
if options.cachebinhost:
binhost_cache = cache.DiskCache(cache_dir)
boto_file = vartree.settings['BOTO_CONFIG']
if boto_file:
os.environ['BOTO_CONFIG'] = boto_file
gs_context = gs.GSContext()
symbols_mapping = RemoteSymbols(vartree, binhost_cache)
if options.all:
to_install = vartree.dbapi.cpv_all()
else:
to_install = [GetMatchingCPV(p, vartree.dbapi) for p in options.packages]
to_install = [p for p in to_install
if ShouldGetSymbols(p, vartree.dbapi, symbols_mapping)]
if not to_install:
logging.info('nothing to do, exit')
return
with DebugSymbolsInstaller(vartree, gs_context, sysroot,
not options.debug) as installer:
args = [(p, symbols_mapping[p]) for p in to_install]
parallel.RunTasksInProcessPool(installer.Install, args,
processes=options.jobs)
logging.debug('installation done, updating packages index file')
packages_dir = os.path.join(sysroot, 'packages')
packages_file = os.path.join(packages_dir, 'Packages')
# binpkg will set DEBUG_SYMBOLS automatically if it detects the debug symbols
# in the packages dir.
pkgindex = binpkg.GrabLocalPackageIndex(packages_dir)
with open(packages_file, 'w') as p:
pkgindex.Write(p)
def _Emerge(self, packages, board=None):
"""Emerge the specified packages to the specified board.
Args:
packages: Packages to emerge.
board: Board to emerge to. If None, emerge to host.
"""
modified_packages = self._ListModifiedPackages(board)
cmd = self._GetEmergeCommand(board) + [
'-uNv',
'--reinstall-atoms=%s' % ' '.join(modified_packages),
'--usepkg-exclude=%s' % ' '.join(modified_packages),
]
cmd.append('--deep' if self.options.deps else '--nodeps')
if self.options.binary:
cmd += ['-g', '--with-bdeps=y']
if board is None:
# Only update toolchains in the chroot when binpkgs are available. The
# toolchain rollout process only takes place when the chromiumos sdk
# builder finishes a successful build and pushes out binpkgs.
cmd += [
'--useoldpkg-atoms=%s' % ' '.join(GetToolchainPackages())
]
if self.options.rebuild_deps:
cmd.append('--rebuild-if-unbuilt')
if self.options.jobs:
cmd.append('--jobs=%d' % self.options.jobs)
if self.options.log_level.lower() == 'debug':
cmd.append('--show-output')
cros_build_lib.SudoRunCommand(cmd + packages)
def _InstallBuildDependencies(self):
cros_build_lib.SudoRunCommand([
self.PARALLEL_EMERGE,
'--board=%s' % self.options.board,
'--root=%s' % self.sysroot, '--usepkg', '--onlydeps',
'--usepkg-exclude=%s' % self.options.package, self.options.package
])
def ArchiveTestResults(results_path, archive_dir):
"""Archives the test results to |archive_dir|.
Args:
results_path: Path to test results.
archive_dir: Local directory to archive to.
"""
cros_build_lib.SudoRunCommand(['chmod', '-R', 'a+rw', results_path],
print_cmd=False)
if os.path.exists(archive_dir):
osutils.RmDir(archive_dir)
def _ShouldIgnore(dirname, file_list):
# Note: We exclude VM disk and memory images. Instead, they are
# archived via ArchiveVMFiles. Also skip any symlinks. gsutil
# hangs on broken symlinks.
return [
x for x in file_list if x.startswith(constants.VM_DISK_PREFIX)
or x.startswith(constants.VM_MEM_PREFIX)
or os.path.islink(os.path.join(dirname, x))
]
shutil.copytree(results_path,
archive_dir,
symlinks=False,
ignore=_ShouldIgnore)
def _ArchiveTestResults(self, results_dir):
"""Try to find the results dropped during testing and archive them.
Args:
results_dir: Path to a directory used for creating result files.
"""
results_reldir = 'moblab_vm_test_results'
cros_build_lib.SudoRunCommand(['chmod', '-R', 'a+rw', results_dir],
print_cmd=False)
archive_dir = os.path.join(self.archive_path, results_reldir)
osutils.RmDir(archive_dir, ignore_missing=True)
def _ShouldIgnore(dirname, file_list):
# gsutil hangs on broken symlinks.
return [
x for x in file_list
if os.path.islink(os.path.join(dirname, x))
]
shutil.copytree(results_dir,
archive_dir,
symlinks=False,
ignore=_ShouldIgnore)
self._Upload(results_reldir)
self._PrintDetailedLogLinks(results_reldir)
def Status(self):
# Only bother checking once a day. Our certs are valid in the
# range of weeks, so there's no need to constantly do this.
if (datetime.date.today() <
self.last_notification + datetime.timedelta(days=1)):
return
cmd = ['prodcertstatus', '--check_loas_cert_location', 'sslenrolled']
result = cros_build_lib.SudoRunCommand(cmd,
user=self.user,
error_code_ok=True,
redirect_stdout=True)
# Figure out how many days are left. The command should display:
# SSL-ENROLLED CERT cert expires in about 22 days
m = re.search(r'cert expires in about ([0-9]+) days', result.output)
if m:
days_left = int(m.group(1))
else:
days_left = 0
# Send out one notification a day if there's a week or less left
# before our creds expire.
if days_left <= 7:
alerts.SendEmail(
'Loas certs expiring soon!',
self.email_notify,
server=self.email_server,
message='Please run:\n %s\n\n%s\n%s' % (
self.enroll_msg, result.output, result.error))
self.last_notification = datetime.date.today()
else:
# We won't expire for a while, so stop the periodic polling.
self.last_notification = (
datetime.date.today() + datetime.timedelta(days=days_left - 8))
def SyncSources(self):
repo = repository.RepoRepository(self.manifest_dir,
self.repo_root,
referenced_repo=self.reference,
manifest=MANIFEST_FILE,
branch='master')
# Trigger the network sync
repo.Sync(jobs=multiprocessing.cpu_count() + 1, network_only=True)
projects = [self.CHROMIUM_ROOT]
if self.internal:
projects.append(self.CHROME_ROOT)
for project in projects:
path = os.path.join(self.repo_root, project)
if os.path.exists(path):
try:
git.CleanAndCheckoutUpstream(path, refresh_upstream=False)
continue
except cros_build_lib.RunCommandError:
cros_build_lib.Error("Failed cleaning %r; wiping.", path)
cros_build_lib.SudoRunCommand(['rm', '-rf', path],
print_cmd=False)
cros_build_lib.RunCommand(['repo', 'sync', '-ld', project],
cwd=self.repo_root)
def SetFileContents(path, value, cwd=None):
"""Set a given filepath contents w/ the passed in value."""
cros_build_lib.SudoRunCommand(['tee', path],
redirect_stdout=True,
print_cmd=False,
input=value,
cwd=cwd)
def _DoCleanup(self):
"""Wipe unused repositories."""
# Find all projects, even if they're not in the manifest. Note the find
# trickery this is done to keep it as fast as possible.
repo_path = os.path.join(self.directory, '.repo', 'projects')
current = set(cros_build_lib.RunCommandCaptureOutput(
['find', repo_path, '-type', 'd', '-name', '*.git', '-printf', '%P\n',
'-a', '!', '-wholename', '*.git/*', '-prune'],
print_cmd=False).output.splitlines())
data = {}.fromkeys(current, 0)
path = os.path.join(self.directory, '.repo', 'project.lru')
if os.path.exists(path):
existing = [x.strip().split(None, 1)
for x in osutils.ReadFile(path).splitlines()]
data.update((k, int(v)) for k, v in existing if k in current)
# Increment it all...
data.update((k, v + 1) for k, v in data.iteritems())
# Zero out what is now used.
projects = git.ManifestCheckout.Cached(self.directory).projects
data.update(('%s.git' % x['path'], 0) for x in projects.itervalues())
# Finally... wipe anything that's greater than our threshold.
wipes = [k for k, v in data.iteritems() if v > self.LRU_THRESHOLD]
if wipes:
cros_build_lib.SudoRunCommand(
['rm', '-rf'] + [os.path.join(repo_path, proj) for proj in wipes])
map(data.pop, wipes)
osutils.WriteFile(path, "\n".join('%s %i' % x for x in data.iteritems()))
def _InstallKernelHeaders(self):
cros_build_lib.SudoRunCommand([
self.PARALLEL_EMERGE,
'--board=%s' % self.options.board, '--root-deps=rdeps',
'--getbinpkg', '--usepkg',
'--root=%s' % self.sysroot, 'sys-kernel/linux-headers'
])
def SafeMakedirs(path, mode=0o775, sudo=False):
"""Make parent directories if needed. Ignore if existing.
Arguments:
path: The path to create. Intermediate directories will be created as
needed.
mode: The access permissions in the style of chmod.
sudo: If True, create it via sudo, thus root owned.
Raises:
EnvironmentError: if the makedir failed and it was non sudo.
RunCommandError: If sudo mode, and the command failed for any reason.
Returns:
True if the directory had to be created, False if otherwise.
"""
if sudo:
if os.path.isdir(path):
return False
cros_build_lib.SudoRunCommand(
['mkdir', '-p', '--mode', oct(mode), path],
print_cmd=False,
redirect_stderr=True,
redirect_stdout=True)
return True
try:
os.makedirs(path, mode)
return True
except EnvironmentError as e:
if e.errno != errno.EEXIST or not os.path.isdir(path):
raise
return False
def RmDir(path, ignore_missing=False, sudo=False):
"""Recursively remove a directory.
Arguments:
ignore_missing: Do not error when path does not exist.
sudo: Remove directories as root.
"""
if sudo:
try:
cros_build_lib.SudoRunCommand(
['rm',
'-r%s' % ('f' if ignore_missing else '', ), '--', path],
debug_level=logging.DEBUG,
redirect_stdout=True,
redirect_stderr=True)
except cros_build_lib.RunCommandError as e:
if not ignore_missing or os.path.exists(path):
# If we're not ignoring the rm ENOENT equivalent, throw it;
# if the pathway still exists, something failed, thus throw it.
raise
else:
try:
shutil.rmtree(path)
except EnvironmentError as e:
if not ignore_missing or e.errno != errno.ENOENT:
raise
def Install(self, cpv, url):
"""Install the debug symbols for |cpv|.
This will install the debug symbols tarball in PKGDIR so that it can be
used later.
Args:
cpv: the cpv of the package to build. This assumes that the cpv is
installed in the sysroot.
url: url of the debug symbols archive. This could be a Google Storage url
or a local path.
"""
archive = os.path.join(self._vartree.settings['PKGDIR'],
cpv + DEBUG_SYMS_EXT)
# GsContext does not understand file:// scheme so we need to extract the
# path ourselves.
parsed_url = urlparse.urlsplit(url)
if not parsed_url.scheme or parsed_url.scheme == 'file':
url = parsed_url.path
if not os.path.isfile(archive):
self._gs_context.Copy(url, archive, debug_level=logging.DEBUG)
with osutils.TempDir(sudo_rm=True) as tempdir:
cros_build_lib.SudoRunCommand(['tar', '-I', 'bzip2 -q', '-xf', archive,
'-C', tempdir], quiet=True)
with open(self._vartree.getpath(cpv, filename='CONTENTS'),
'a') as content_file:
# Merge the content of the temporary dir into the sysroot.
# pylint: disable=protected-access
link = self._vartree.dbapi._dblink(cpv)
link.mergeme(tempdir, self._sysroot, content_file, None, '', {}, None)
def _SignalPids(pids, signum):
cros_build_lib.SudoRunCommand(['kill', '-%i' % signum] +
sorted(pids),
print_cmd=False,
error_code_ok=True,
redirect_stdout=True,
combine_stdout_stderr=True)
def main(argv):
opts = ParseArgs(argv)
if not cros_build_lib.IsInsideChroot():
raise commandline.ChrootRequiredError()
if os.geteuid() != 0:
cros_build_lib.SudoRunCommand(sys.argv, print_cmd=False)
return
output = sys.stdout
if opts.out_file:
output = open(opts.out_file, 'w')
sysroot = sysroot_lib.Sysroot(opts.sysroot)
if opts.command == 'create-wrappers':
sysroot.CreateAllWrappers(opts.friendlyname)
elif opts.command == 'generate-config':
config = sysroot.GenerateBoardConfig(opts.board)
output.write('\n' + config)
elif opts.command == 'generate-make-conf':
output.write('\n' + sysroot.GenerateMakeConf(opts.accepted_licenses))
elif opts.command == 'generate-binhosts':
output.write(
'\n' +
sysroot.GenerateBinhostConf(opts.chrome_only, opts.local_only))
def Start(self):
"""Start the VM."""
self.Stop()
if not self.kvm_path:
self.kvm_path = self._FindKVMBinary()
logging.debug('kvm path=%s', self.kvm_path)
if not self.image_path:
self.image_path = os.environ.get('VM_IMAGE_PATH', '')
logging.debug('vm image path=%s', self.image_path)
if not self.image_path or not os.path.exists(self.image_path):
raise VMError('VM image path %s does not exist.' % self.image_path)
self._CleanupFiles(recreate=True)
open(self.kvm_serial, 'w')
for pipe in [self.kvm_pipe_in, self.kvm_pipe_out]:
os.mkfifo(pipe, 0600)
args = [
self.kvm_path, '-m', '2G', '-smp', '4', '-vga', 'cirrus',
'-pidfile', self.pidfile, '-chardev',
'pipe,id=control_pipe,path=%s' % self.kvm_monitor, '-serial',
'file:%s' % self.kvm_serial, '-mon', 'chardev=control_pipe',
'-daemonize', '-net', 'nic,model=virtio', '-net',
'user,hostfwd=tcp::%d-:22' % self.ssh_port, '-drive',
'file=%s,index=0,media=disk,cache=unsafe' % self.image_path
]
logging.info(' '.join(args))
logging.info('Pid file: %s', self.pidfile)
if not self.dry_run:
cros_build_lib.SudoRunCommand(args)
def DowngradePackageVersion(portage_root,
package_cp,
downgrade_to_version='0'):
"""Downgrade the specified portage package version.
Arguments:
portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy'
package_cp: A string similar to 'chromeos-base/autotest-tests'.
downgrade_to_version: String version to downgrade to. Default: '0'
Returns:
True on success. False on failure (nonzero return code from `mv` command).
"""
try:
package, _ = GetPackageAPI(portage_root, package_cp)
except PortagePackageAPIError:
# Unable to fetch a corresponding portage package API for this
# package_cp (either no such package, or name ambigious and matches).
# So, just fail out.
return False
source_directory = package.dbdir
destination_path = os.path.join(package.dbroot,
package_cp + '-' + downgrade_to_version)
if os.path.abspath(source_directory) == os.path.abspath(destination_path):
return True
command = ['mv', source_directory, destination_path]
code = cros_build_lib.SudoRunCommand(command,
error_code_ok=True).returncode
return code == 0
def _RemoveGroupOnDisk(cls, path, strict, sudo_strict=True):
"""Perform the actual group removal.
Args:
namespace: The cgroup namespace to remove
strict: Boolean; if true, then it's an error if the group can't be
removed. This can occur if there are still processes in it, or in
a nested group.
"""
# Depth first recursively remove our children cgroups, then ourselves.
# Allow this to fail since currently it's possible for the cleanup code
# to not fully kill the hierarchy. Note that we must do just rmdirs,
# rm -rf cannot be used- it tries to remove files which are unlinkable
# in cgroup (only namespaces can be removed via rmdir).
# See Documentation/cgroups/ for further details.
path = os.path.normpath(path) + '/'
# Do a sanity check to ensure that we're not touching anything we
# shouldn't.
if not path.startswith(cls.CGROUP_ROOT):
raise RuntimeError("cgroups.py: Was asked to wipe path %s, refusing. "
"strict was %r, sudo_strict was %r"
% (path, strict, sudo_strict))
result = cros_build_lib.SudoRunCommand(
['find', path, '-depth', '-type', 'd', '-exec', 'rmdir', '{}', '+'],
redirect_stderr=True, error_code_ok=not strict,
print_cmd=False, strict=sudo_strict)
if result.returncode == 0:
return True
elif not os.path.isdir(path):
# We were invoked against a nonexistant path.
return True
return False
def _RunIgnoringErrors(cmd):
"""Runs the given command, ignores errors but warns user."""
try:
cros_build_lib.SudoRunCommand(cmd, quiet=True)
except cros_build_lib.RunCommandError as e:
logging.error('Cleanup operation failed. Please run "%s" manually.',
' '.join(cmd))
logging.debug('Encountered error: %s', e)
def _Emerge(self, *args, **kwargs):
"""Emerge the given packages using parallel_emerge."""
cmd = [
self.PARALLEL_EMERGE,
'--board=%s' % self.options.board, '--usepkgonly', '--noreplace'
] + list(args)
kwargs.setdefault('extra_env', self.extra_env)
cros_build_lib.SudoRunCommand(cmd, **kwargs)
def _PingDD(self, dd_pid):
"""Send USR1 signal to dd to get status update."""
try:
cmd = ['kill', '-USR1', str(dd_pid)]
cros_build_lib.SudoRunCommand(cmd, print_cmd=False)
except cros_build_lib.RunCommandError:
# Here we assume that dd finished in the background.
return
def TestStats(self):
"""Collect inodes and blocks usage."""
# Find the loopback device that was mounted to ROOT_A.
loop_device = None
root_path = os.path.abspath(os.readlink(image_test_lib.ROOT_A))
for mtab in osutils.IterateMountPoints():
if mtab.destination == root_path:
loop_device = mtab.source
break
self.assertTrue(loop_device, 'Cannot find loopback device for ROOT_A.')
# Gather file system stats with tune2fs.
cmd = [
'tune2fs',
'-l',
loop_device
]
# tune2fs produces output like this:
#
# tune2fs 1.42 (29-Nov-2011)
# Filesystem volume name: ROOT-A
# Last mounted on: <not available>
# Filesystem UUID: <none>
# Filesystem magic number: 0xEF53
# Filesystem revision #: 1 (dynamic)
# ...
#
# So we need to ignore the first line.
ret = cros_build_lib.SudoRunCommand(cmd, capture_output=True,
extra_env={'LC_ALL': 'C'})
fs_stat = dict(line.split(':', 1) for line in ret.output.splitlines()
if ':' in line)
free_inodes = int(fs_stat['Free inodes'])
free_blocks = int(fs_stat['Free blocks'])
inode_count = int(fs_stat['Inode count'])
block_count = int(fs_stat['Block count'])
block_size = int(fs_stat['Block size'])
sum_file_size = 0
for root, _, filenames in os.walk(image_test_lib.ROOT_A):
for file_name in filenames:
full_name = os.path.join(root, file_name)
file_stat = os.lstat(full_name)
sum_file_size += file_stat.st_size
metadata_size = (block_count - free_blocks) * block_size - sum_file_size
self.OutputPerfValue('free_inodes_over_inode_count',
free_inodes * 100.0 / inode_count, 'percent',
graph='free_over_used_ratio')
self.OutputPerfValue('free_blocks_over_block_count',
free_blocks * 100.0 / block_count, 'percent',
graph='free_over_used_ratio')
self.OutputPerfValue('apparent_size', sum_file_size, 'bytes',
higher_is_better=False, graph='filesystem_stats')
self.OutputPerfValue('metadata_size', metadata_size, 'bytes',
higher_is_better=False, graph='filesystem_stats')
def RsyncQuickmerge(source_path, sysroot_autotest_path,
include_pattern_file=None, pretend=False,
overwrite=False):
"""Run rsync quickmerge command, with specified arguments.
Command will take form `rsync -a [options] --exclude=**.pyc
--exclude=**.pyo
[optional --include-from include_pattern_file]
--exclude=* [source_path] [sysroot_autotest_path]`
Args:
source_path: Directory to rsync from.
sysroot_autotest_path: Directory to rsync too.
include_pattern_file: Optional pattern of files to include in rsync.
pretend: True to use the '-n' option to rsync, to perform dry run.
overwrite: True to omit '-u' option, overwrite all files in sysroot,
not just older files.
Returns:
The cros_build_lib.CommandResult object resulting from the rsync command.
"""
command = ['rsync', '-a']
# For existing files, preserve destination permissions. This ensures that
# existing files end up with the file permissions set by the ebuilds.
# If this script copies over a file that does not exist in the destination
# tree, it will set the least restrictive permissions allowed in the
# destination tree. This could happen if the file copied is not installed by
# *any* ebuild, or if the ebuild that installs the file was never emerged.
command += ['--no-p', '--chmod=ugo=rwX']
if pretend:
command += ['-n']
if not overwrite:
command += ['-u']
command += ['-i']
command += ['--exclude=**.pyc']
command += ['--exclude=**.pyo']
# Exclude files with a specific substring in their name, because
# they create an ambiguous itemized report. (see unit test file for details)
command += ['--exclude=** -> *']
if include_pattern_file:
command += ['--include-from=%s' % include_pattern_file]
command += ['--exclude=*']
# Some tests use symlinks. Follow these.
command += ['-L']
command += [source_path, sysroot_autotest_path]
return cros_build_lib.SudoRunCommand(command, redirect_stdout=True)
