def undo():
if os.path.exists(path_to_orig):
cmd_runner.run(
['mv', '-f', path_to_orig, directory], as_root=True).wait()
else:
cmd_runner.run(
['rm', '-f', oldpath], as_root=True).wait()
def create_partitions(board_config, media, heads, sectors, cylinders=None,
should_align_boot_part=False):
"""Partition the given media according to the board requirements.
:param board_config: A BoardConfig class.
:param media: A setup_partitions.Media object to partition.
:param heads: Number of heads to use in the disk geometry of
partitions.
:param sectors: Number of sectors to use in the disk geometry of
partitions.
:param cylinders: The number of cylinders to pass to sfdisk's -C argument.
If None the -C argument is not passed.
:param should_align_boot_part: Whether to align the boot partition too.
"""
if media.is_block_device:
# Overwrite any existing partition tables with a fresh one.
proc = cmd_runner.run(
['parted', '-s', media.path, 'mklabel', 'msdos'], as_root=True)
proc.wait()
wait_partition_to_settle(media)
sfdisk_cmd = board_config.get_sfdisk_cmd(
should_align_boot_part=should_align_boot_part)
run_sfdisk_commands(sfdisk_cmd, heads, sectors, cylinders, media.path)
# Sync and sleep to wait for the partition to settle.
cmd_runner.run(['sync']).wait()
wait_partition_to_settle(media)
def temporarily_overwrite_file_on_dir(filepath, directory, tmp_dir):
"""Temporarily replace a file on the given directory.
We'll move the existing file on the given directory to a temp dir, then
copy over the given file to that directory and register a function in
local_atexit to move the orig file back to the given directory.
"""
basename = os.path.basename(filepath)
path_to_orig = os.path.join(tmp_dir, basename)
# Move the existing file from the given directory to the temp dir.
oldpath = os.path.join(directory, basename)
if os.path.exists(oldpath):
cmd_runner.run(
['mv', '-f', oldpath, path_to_orig], as_root=True).wait()
# Now copy the given file onto the given directory.
cmd_runner.run(['cp', filepath, directory], as_root=True).wait()
def undo():
if os.path.exists(path_to_orig):
cmd_runner.run(
['mv', '-f', path_to_orig, directory], as_root=True).wait()
else:
cmd_runner.run(
['rm', '-f', oldpath], as_root=True).wait()
local_atexit.append(undo)
def undo():
if os.path.exists(path_to_orig):
cmd_runner.run(
['mv', '-f', path_to_orig, directory], as_root=True).wait()
else:
cmd_runner.run(
['rm', '-f', oldpath], as_root=True).wait()
def temporarily_overwrite_file_on_dir(filepath, directory, tmp_dir):
"""Temporarily replace a file on the given directory.
We'll move the existing file on the given directory to a temp dir, then
copy over the given file to that directory and register a function in
local_atexit to move the orig file back to the given directory.
"""
basename = os.path.basename(filepath)
path_to_orig = os.path.join(tmp_dir, basename)
# Move the existing file from the given directory to the temp dir.
oldpath = os.path.join(directory, basename)
if os.path.exists(oldpath):
cmd_runner.run(['mv', '-f', oldpath, path_to_orig],
as_root=True).wait()
# Now copy the given file onto the given directory.
cmd_runner.run(['cp', filepath, directory], as_root=True).wait()
def undo():
if os.path.exists(path_to_orig):
cmd_runner.run(['mv', '-f', path_to_orig, directory],
as_root=True).wait()
else:
cmd_runner.run(['rm', '-f', oldpath], as_root=True).wait()
local_atexit.append(undo)
def has_command(command):
"""Check the given command is available."""
try:
cmd_runner.run(
['which', command], stdout=open('/dev/null', 'w')).wait()
return True
except cmd_runner.SubcommandNonZeroReturnValue:
return False
def has_command(command):
"""Check the given command is available."""
try:
cmd_runner.run(['which', command], stdout=open('/dev/null',
'w')).wait()
return True
except cmd_runner.SubcommandNonZeroReturnValue:
return False
def install_hwpacks(
rootfs_dir, tmp_dir, tools_dir, hwpack_force_yes, verified_files,
extract_kpkgs=False, *hwpack_files):
"""Install the given hwpacks onto the given rootfs."""
install_command = 'linaro-hwpack-install'
linaro_hwpack_install_path = find_command(
install_command, prefer_dir=tools_dir)
if not linaro_hwpack_install_path:
raise ChrootException("The program linaro-hwpack-install could not "
"be found found: cannot proceed.")
else:
linaro_hwpack_install_path = os.path.abspath(
linaro_hwpack_install_path)
# In case we just want to extract the kernel packages, don't force qemu
# with chroot, as we could have archs without qemu support
if not extract_kpkgs:
prepare_chroot(rootfs_dir, tmp_dir)
# FIXME: shouldn't use chroot/usr/bin as this might conflict with
# installed packages; would be best to use some custom directory like
# chroot/linaro-image-tools/bin
copy_file(linaro_hwpack_install_path,
os.path.join(rootfs_dir, 'usr', 'bin'))
mount_chroot_proc(rootfs_dir)
try:
# Sometimes the host will have qemu-user-static installed but
# another package (i.e. scratchbox) will have mangled its config
# and thus we won't be able to chroot and install the hwpack, so
# we fail here and tell the user to ensure qemu-arm-static is
# setup before trying again.
cmd_runner.run(['true'], as_root=True, chroot=rootfs_dir).wait()
except:
print ("Cannot proceed with hwpack installation because "
"there doesn't seem to be a binfmt interpreter registered "
"to execute arm binaries in the chroot. Please check "
"that qemu-user-static is installed and properly "
"configured before trying again.")
raise
else:
# We are not in the chroot, we do not copy the linaro-hwpack-install
# file, but we might not have l-i-t installed, so we need the full path
# of the linaro-hwpack-install program to run.
install_command = linaro_hwpack_install_path
try:
for hwpack_file in hwpack_files:
hwpack_verified = False
if os.path.basename(hwpack_file) in verified_files:
hwpack_verified = True
install_hwpack(rootfs_dir, hwpack_file, extract_kpkgs,
hwpack_force_yes or hwpack_verified,
install_command)
finally:
run_local_atexit_funcs()
def test_tuple_with_sudo(self):
fixture = self.useFixture(MockCmdRunnerPopenFixture())
self.useFixture(MockSomethingFixture(os, 'getuid', lambda: 1000))
cmd_runner.run((
'foo',
'bar',
), as_root=True).wait()
self.assertEqual(['%s foo bar' % sudo_args],
fixture.mock.commands_executed)
def setup_android_partitions(board_config, media, image_size, bootfs_label,
should_create_partitions,
should_align_boot_part=False):
cylinders = None
if not media.is_block_device:
image_size_in_bytes = get_partition_size_in_bytes(image_size)
cylinders = image_size_in_bytes / CYLINDER_SIZE
proc = cmd_runner.run(
['dd', 'of=%s' % media.path,
'bs=1', 'seek=%s' % image_size_in_bytes, 'count=0'],
stderr=open('/dev/null', 'w'))
proc.wait()
if should_create_partitions:
create_partitions(
board_config, media, HEADS, SECTORS, cylinders,
should_align_boot_part=should_align_boot_part)
if media.is_block_device:
bootfs, system, cache, data, sdcard = \
get_android_partitions_for_media(media, board_config)
ensure_partition_is_not_mounted(bootfs)
ensure_partition_is_not_mounted(system)
ensure_partition_is_not_mounted(cache)
ensure_partition_is_not_mounted(data)
ensure_partition_is_not_mounted(sdcard)
else:
partitions = get_android_loopback_devices(media.path)
bootfs = partitions[0]
system = partitions[1]
cache = partitions[2]
data = partitions[3]
sdcard = partitions[4]
print "\nFormating boot partition\n"
proc = cmd_runner.run(
['mkfs.vfat', '-F', str(board_config.fat_size), bootfs, '-n',
bootfs_label],
as_root=True)
proc.wait()
ext4_partitions = {"system": system, "cache": cache, "userdata": data}
for label, dev in ext4_partitions.iteritems():
mkfs = 'mkfs.%s' % "ext4"
proc = cmd_runner.run(
[mkfs, dev, '-L', label],
as_root=True)
proc.wait()
proc = cmd_runner.run(
['mkfs.vfat', '-F32', sdcard, '-n',
"sdcard"],
as_root=True)
proc.wait()
return bootfs, system, cache, data, sdcard
def setup_android_partitions(board_config, media, image_size, bootfs_label,
should_create_partitions,
should_align_boot_part=False):
cylinders = None
if not media.is_block_device:
image_size_in_bytes = get_partition_size_in_bytes(image_size)
cylinders = image_size_in_bytes / CYLINDER_SIZE
proc = cmd_runner.run(
['dd', 'of=%s' % media.path,
'bs=1', 'seek=%s' % image_size_in_bytes, 'count=0'],
stderr=open('/dev/null', 'w'))
proc.wait()
if should_create_partitions:
create_partitions(
board_config, media, HEADS, SECTORS, cylinders,
should_align_boot_part=should_align_boot_part)
if media.is_block_device:
bootfs, system, cache, data, sdcard = \
get_android_partitions_for_media(media, board_config)
ensure_partition_is_not_mounted(bootfs)
ensure_partition_is_not_mounted(system)
ensure_partition_is_not_mounted(cache)
ensure_partition_is_not_mounted(data)
ensure_partition_is_not_mounted(sdcard)
else:
partitions = get_android_loopback_devices(media.path)
bootfs = partitions[0]
system = partitions[1]
cache = partitions[2]
data = partitions[3]
sdcard = partitions[4]
print "\nFormating boot partition\n"
proc = cmd_runner.run(
['mkfs.vfat', '-F', str(board_config.fat_size), bootfs, '-n',
bootfs_label],
as_root=True)
proc.wait()
ext4_partitions = {"system": system, "cache": cache, "userdata": data}
for label, dev in ext4_partitions.iteritems():
mkfs = 'mkfs.%s' % "ext4"
proc = cmd_runner.run(
[mkfs, '-F', dev, '-L', label],
as_root=True)
proc.wait()
proc = cmd_runner.run(
['mkfs.vfat', '-F32', sdcard, '-n',
"sdcard"],
as_root=True)
proc.wait()
return bootfs, system, cache, data, sdcard
def unpack_package(self, package_file_name):
# We could extract only a single file, but since dpkg will pipe
# the entire package through tar anyway we might as well extract all.
unpack_dir = self.get_path(package_file_name)
if not os.path.isdir(unpack_dir):
os.mkdir(unpack_dir)
p = cmd_runner.run(["tar", "-C", unpack_dir, "-xf", "-"], stdin=PIPE)
cmd_runner.run(["dpkg", "--fsys-tarfile", package_file_name],
stdout=p.stdin).communicate()
p.communicate()
def populate_rootfs(content_dir, root_disk, partition, rootfs_type,
rootfs_id, should_create_swap, swap_size,
mmc_device_id, partition_offset, os_release_id,
board_config=None):
"""Populate the rootfs and make the necessary tweaks to make it usable.
This consists of:
1. Create a directory on the path specified by root_disk
2. Mount the given partition onto the created directory.
3. Setup an atexit handler to unmount the partition mounted above.
4. Move the contents of content_dir to that directory.
5. If should_create_swap, then create it with the given size.
6. Add fstab entries for the / filesystem and swap (if created).
7. Create a /etc/flash-kernel.conf containing the target's boot device.
"""
print "\nPopulating rootfs partition"
print "Be patient, this may take a few minutes\n"
# Create a directory to mount the rootfs partition.
os.makedirs(root_disk)
with partition_mounted(partition, root_disk):
move_contents(content_dir, root_disk)
mount_options = rootfs_mount_options(rootfs_type)
fstab_additions = ["%s / %s %s 0 1" % (
rootfs_id, rootfs_type, mount_options)]
if should_create_swap:
print "\nCreating SWAP File\n"
if has_space_left_for_swap(root_disk, swap_size):
proc = cmd_runner.run([
'dd',
'if=/dev/zero',
'of=%s/SWAP.swap' % root_disk,
'bs=1M',
'count=%s' % swap_size], as_root=True)
proc.wait()
proc = cmd_runner.run(
['mkswap', '%s/SWAP.swap' % root_disk], as_root=True)
proc.wait()
fstab_additions.append("/SWAP.swap none swap sw 0 0")
else:
print ("Swap file is bigger than space left on partition; "
"continuing without swap.")
append_to_fstab(root_disk, fstab_additions)
if os_release_id == 'debian' or os_release_id == 'ubuntu' or \
os.path.exists('%s/etc/debian_version' % root_disk):
print "\nCreating /etc/flash-kernel.conf\n"
create_flash_kernel_config(
root_disk, mmc_device_id, 1 + partition_offset)
if board_config is not None:
print "\nUpdating /etc/network/interfaces\n"
update_network_interfaces(root_disk, board_config)
def move_contents(from_, root_disk):
"""Move everything under from_ to the given root disk.
Uses sudo for moving.
"""
assert os.path.isdir(from_), "%s is not a directory" % from_
files = _list_files(from_)
mv_cmd = ['mv']
mv_cmd.extend(sorted(files))
mv_cmd.append(root_disk)
cmd_runner.run(mv_cmd, as_root=True).wait()
def write_data_to_protected_file(path, data):
"""Write data to the file on the given path.
This is meant to be used when the given file is only writable by root, and
we overcome that by writing the data to a tempfile and then moving the
tempfile on top of the given one using sudo.
"""
_, tmpfile = tempfile.mkstemp()
with open(tmpfile, 'w') as fd:
fd.write(data)
cmd_runner.run(['mv', '-f', tmpfile, path], as_root=True).wait()
def sources_entry_for_debs(self, local_debs, label=None):
tmpdir = self.make_temporary_directory()
with open(os.path.join(tmpdir, 'Packages'), 'w') as packages_file:
packages_file.write(get_packages_file(local_debs, rel_to=tmpdir))
if label:
cmd_runner.run(
['apt-ftparchive',
'-oAPT::FTPArchive::Release::Label=%s' % label,
'release',
tmpdir],
stdout=open(os.path.join(tmpdir, 'Release'), 'w')).wait()
return 'file://%s ./' % (tmpdir, )
def sources_entry_for_debs(self, local_debs, label=None):
tmpdir = self.make_temporary_directory()
with open(os.path.join(tmpdir, 'Packages'), 'w') as packages_file:
packages_file.write(get_packages_file(local_debs, rel_to=tmpdir))
if label:
cmd_runner.run([
'apt-ftparchive',
'-oAPT::FTPArchive::Release::Label=%s' % label, 'release',
tmpdir
],
stdout=open(os.path.join(tmpdir, 'Release'),
'w')).wait()
return 'file://%s ./' % (tmpdir, )
def copy_file(filepath, directory):
"""Copy the given file to the given directory.
The copying of the file is done in a subprocess and run using sudo.
We also register a function in local_atexit to remove the file from the
given directory.
"""
cmd_runner.run(['cp', filepath, directory], as_root=True).wait()
def undo():
new_path = os.path.join(directory, os.path.basename(filepath))
cmd_runner.run(['rm', '-f', new_path], as_root=True).wait()
local_atexit.append(undo)
def copy_file(filepath, directory):
"""Copy the given file to the given directory.
The copying of the file is done in a subprocess and run using sudo.
We also register a function in local_atexit to remove the file from the
given directory.
"""
cmd_runner.run(['cp', filepath, directory], as_root=True).wait()
def undo():
new_path = os.path.join(directory, os.path.basename(filepath))
cmd_runner.run(['rm', '-f', new_path], as_root=True).wait()
local_atexit.append(undo)
def verify_file_integrity(sig_file_list):
"""Verify a list of signature files.
The parameter is a list of filenames of gpg signature files which will be
verified using gpg. For each of the files it is assumed that there is an
sha1 hash file with the same file name minus the '.asc' extension.
Each of the sha1 files will be checked using sha1sums. All files listed in
the sha1 hash file must be found in the same directory as the hash file.
"""
gpg_sig_ok = True
gpg_out = ""
verified_files = []
for sig_file in sig_file_list:
hash_file = sig_file[0:-len('.asc')]
tmp = tempfile.NamedTemporaryFile()
try:
cmd_runner.run(['gpg', '--status-file={0}'.format(tmp.name),
'--verify', sig_file]).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
gpg_sig_ok = False
gpg_out = gpg_out + tmp.read()
tmp.close()
if os.path.dirname(hash_file) == '':
sha_cwd = None
else:
sha_cwd = os.path.dirname(hash_file)
try:
sha1sums_out, _ = cmd_runner.Popen(
['sha1sum', '-c', hash_file],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=sha_cwd
).communicate()
except cmd_runner.SubcommandNonZeroReturnValue as inst:
sha1sums_out = inst.stdout
for line in sha1sums_out.splitlines():
sha1_check = re.search(r'^(.*):\s+OK', line)
if sha1_check:
verified_files.append(sha1_check.group(1))
return verified_files, gpg_sig_ok, gpg_out
def verify_file_integrity(sig_file_list):
"""Verify a list of signature files.
The parameter is a list of filenames of gpg signature files which will be
verified using gpg. For each of the files it is assumed that there is an
sha1 hash file with the same file name minus the '.asc' extension.
Each of the sha1 files will be checked using sha1sums. All files listed in
the sha1 hash file must be found in the same directory as the hash file.
"""
gpg_sig_ok = True
gpg_out = ""
verified_files = []
for sig_file in sig_file_list:
hash_file = sig_file[0:-len('.asc')]
tmp = tempfile.NamedTemporaryFile()
try:
cmd_runner.run([
'gpg', '--status-file={0}'.format(tmp.name), '--verify',
sig_file
]).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
gpg_sig_ok = False
gpg_out = gpg_out + tmp.read()
tmp.close()
if os.path.dirname(hash_file) == '':
sha_cwd = None
else:
sha_cwd = os.path.dirname(hash_file)
try:
sha1sums_out, _ = cmd_runner.Popen(['sha1sum', '-c', hash_file],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=sha_cwd).communicate()
except cmd_runner.SubcommandNonZeroReturnValue as inst:
sha1sums_out = inst.stdout
for line in sha1sums_out.splitlines():
sha1_check = re.search(r'^(.*):\s+OK', line)
if sha1_check:
verified_files.append(sha1_check.group(1))
return verified_files, gpg_sig_ok, gpg_out
def install_package_providing(command):
"""Install a package which provides the given command.
If we can't find any package which provides it, raise
UnableToFindPackageProvidingCommand.
If the user denies installing the package, the program exits.
"""
if CommandNotFound is None:
raise UnableToFindPackageProvidingCommand(
"CommandNotFound python module does not exist.")
packages = CommandNotFound().getPackages(command)
if len(packages) == 0:
raise UnableToFindPackageProvidingCommand(
"Unable to find any package providing %s" % command)
# TODO: Ask the user to pick a package when there's more than one that
# provides the given command.
package, _ = packages[0]
output, _ = cmd_runner.run(['apt-get', '-s', 'install', package],
stdout=subprocess.PIPE).communicate()
to_install = []
for line in output.splitlines():
if line.startswith("Inst"):
to_install.append(line.split()[1])
if not to_install:
raise UnableToFindPackageProvidingCommand(
"Unable to find any package to be installed.")
try:
print(
"In order to use the '%s' command, the following package/s "
"have to be installed: %s" % (command, " ".join(to_install)))
resp = raw_input("Install? (Y/n) ")
if resp.lower() != 'y':
print "Package installation is necessary to continue. Exiting."
sys.exit(1)
print("Installing required command '%s' from package '%s'..." %
(command, package))
cmd_runner.run(['apt-get', '--yes', 'install', package],
as_root=True).wait()
except EOFError:
raise PackageInstallationRefused(
"Package installation interrupted: input error.")
except KeyboardInterrupt:
raise PackageInstallationRefused(
"Package installation interrupted by the user.")
def install_package_providing(command):
"""Install a package which provides the given command.
If we can't find any package which provides it, raise
UnableToFindPackageProvidingCommand.
If the user denies installing the package, the program exits.
"""
if CommandNotFound is None:
raise UnableToFindPackageProvidingCommand(
"CommandNotFound python module does not exist.")
packages = CommandNotFound().getPackages(command)
if len(packages) == 0:
raise UnableToFindPackageProvidingCommand(
"Unable to find any package providing %s" % command)
# TODO: Ask the user to pick a package when there's more than one that
# provides the given command.
package, _ = packages[0]
output, _ = cmd_runner.run(['apt-get', '-s', 'install', package],
stdout=subprocess.PIPE).communicate()
to_install = []
for line in output.splitlines():
if line.startswith("Inst"):
to_install.append(line.split()[1])
if not to_install:
raise UnableToFindPackageProvidingCommand(
"Unable to find any package to be installed.")
try:
print ("In order to use the '%s' command, the following package/s "
"have to be installed: %s" % (command, " ".join(to_install)))
resp = raw_input("Install? (Y/n) ")
if resp.lower() != 'y':
print "Package installation is necessary to continue. Exiting."
sys.exit(1)
print ("Installing required command '%s' from package '%s'..."
% (command, package))
cmd_runner.run(['apt-get', '--yes', 'install', package],
as_root=True).wait()
except EOFError:
raise PackageInstallationRefused(
"Package installation interrupted: input error.")
except KeyboardInterrupt:
raise PackageInstallationRefused(
"Package installation interrupted by the user.")
def get_version():
qemu_path = '/usr/bin/qemu-arm-static'
p = cmd_runner.run(["head", "-n", "1"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
if os.path.exists(qemu_path):
try:
# qemu-arm-static has no --version option so it fails,
# but still prints its version plus usage
cmd_runner.run(["/usr/bin/qemu-arm-static", "--version"],
stdout=p.stdin).communicate()
p.communicate()
except:
qemu_version = p.stdout.read()
else:
qemu_version = "Cannot find %s." % qemu_path
return "%s\n: %s" % (__version__, qemu_version)
def populate_raw_partition(self, media, boot_dir):
# To avoid adding a Snowball specific command line option, we assume
# that the user already has unpacked the startfiles to ./startupfiles
config_files_dir = self.snowball_config(boot_dir)
assert os.path.exists(config_files_dir), (
"You need to unpack the Snowball startupfiles to the directory "
"'startupfiles' in your current working directory. See "
"igloocommunity.org for more information.")
# We copy the u-boot files from the unpacked boot.tar.bz2
# and put it with the startfiles.
boot_files = ['u-boot.bin']
for boot_file in boot_files:
cmd_runner.run(['cp', os.path.join(boot_dir, 'boot', boot_file),
config_files_dir], as_root=True).wait()
super(AndroidSnowballEmmcConfig, self).populate_raw_partition(
media, boot_dir)
def create_partitions(board_config, media, should_align_boot_part=False,
part_table="mbr"):
"""Partition the given media according to the board requirements.
:param board_config: A BoardConfig class.
:param media: A setup_partitions.Media object to partition.
:param should_align_boot_part: Whether to align the boot partition too.
:param part_table Type of partition table, either 'mbr' or 'gpt'.
"""
label = 'msdos'
if part_table == 'gpt':
label = part_table
if media.is_block_device:
# Overwrite any existing partition tables with a fresh one.
proc = cmd_runner.run(
['parted', '-s', media.path, 'mklabel', label], as_root=True)
proc.wait()
wait_partition_to_settle(media, part_table)
if part_table == 'gpt':
sgdisk_cmd = board_config.get_sgdisk_cmd(
should_align_boot_part=should_align_boot_part)
run_sgdisk_commands(sgdisk_cmd, media.path)
else: # default partition table to mbr
sfdisk_cmd = board_config.get_sfdisk_cmd(
should_align_boot_part=should_align_boot_part)
run_sfdisk_commands(sfdisk_cmd, media.path)
# sleep to wait for the partition to settle.
wait_partition_to_settle(media, part_table)
def enable_automount():
"""Re-enables back the desktop environment automount option.
This will work only under GNOME with dconf installed. It should be run
as an atexit function.
"""
logger = logging.getLogger(DEFAULT_LOGGER_NAME)
if has_command('dconf'):
try:
cmd_runner.run(['dconf', 'write', AUTOMOUNT_DCONF_KEY, 'true'],
stdout=open('/dev/null', 'w')).wait()
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_OPEN_DCONF_KEYU, 'true'],
stdout=open('/dev/null', 'w')).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
logger.error("Error enabling back desktop environemnt automount.")
def run_sfdisk_commands(commands, heads, sectors, cylinders, device,
as_root=True, stderr=None):
"""Run the given commands under sfdisk.
Every time sfdisk is invoked it will repartition the device so to create
multiple partitions you should craft a list of newline-separated commands
to be executed in a single sfdisk run.
:param commands: A string of sfdisk commands; each on a separate line.
:return: A 2-tuple containing the subprocess' stdout and stderr.
"""
# --force is unfortunate, but a consequence of having partitions not
# starting on cylinder boundaries: sfdisk will abort with "Warning:
# partition 2 does not start at a cylinder boundary"
args = ['sfdisk',
'--force',
'-D',
'-uS',
'-H', str(heads),
'-S', str(sectors)]
if cylinders is not None:
args.extend(['-C', str(cylinders)])
args.append(device)
proc = cmd_runner.run(
args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr,
as_root=as_root)
return proc.communicate("%s\n" % commands)
def run_sfdisk_commands(commands,
heads,
sectors,
cylinders,
device,
as_root=True,
stderr=None):
"""Run the given commands under sfdisk.
Every time sfdisk is invoked it will repartition the device so to create
multiple partitions you should craft a list of newline-separated commands
to be executed in a single sfdisk run.
:param commands: A string of sfdisk commands; each on a separate line.
:return: A 2-tuple containing the subprocess' stdout and stderr.
"""
# --force is unfortunate, but a consequence of having partitions not
# starting on cylinder boundaries: sfdisk will abort with "Warning:
# partition 2 does not start at a cylinder boundary"
args = [
'sfdisk', '--force', '-D', '-uS', '-H',
str(heads), '-S',
str(sectors)
]
if cylinders is not None:
args.extend(['-C', str(cylinders)])
args.append(device)
proc = cmd_runner.run(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr,
as_root=as_root)
return proc.communicate("%s\n" % commands)
def unpack_android_binary_tarball(tarball, unpack_dir, as_root=True):
if is_tar_support_selinux():
tar_cmd = [
'tar', '--selinux', '--numeric-owner', '-C', unpack_dir, '-jxf',
tarball
]
else:
tar_cmd = ['tar', '--numeric-owner', '-C', unpack_dir, '-jxf', tarball]
proc = cmd_runner.run(tar_cmd, as_root=as_root, stderr=subprocess.PIPE)
stderr = proc.communicate()[1]
selinux_warn_outputted = False
selinux_warn1 = "tar: Ignoring unknown extended header keyword"
selinux_warn2 = "tar: setfileconat: Cannot set SELinux context"
for line in stderr.splitlines():
# following 2 messages will not occur at the same time
index = line.find(selinux_warn1)
index2 = line.find(selinux_warn2)
if index == -1 and index2 == -1:
print line
continue
elif not selinux_warn_outputted:
# either index != -1 or index2 != -1
print line
print(
"WARNING: selinux will not work correctly since the\n"
" --selinux option of tar command in this OS\n"
" is not fully supported\n")
selinux_warn_outputted = True
else:
# same line of selinux_warn1 or selinux_warn2
continue
return proc.returncode
def unpack_android_binary_tarball(tarball, unpack_dir, as_root=True):
if is_tar_support_selinux():
tar_cmd = ['tar', '--selinux', '--numeric-owner', '-C', unpack_dir,
'-jxf', tarball]
else:
tar_cmd = ['tar', '--numeric-owner', '-C', unpack_dir,
'-jxf', tarball]
proc = cmd_runner.run(tar_cmd, as_root=as_root,
stderr=subprocess.PIPE)
stderr = proc.communicate()[1]
selinux_warn_outputted = False
selinux_warn1 = "tar: Ignoring unknown extended header keyword"
selinux_warn2 = "tar: setfileconat: Cannot set SELinux context"
for line in stderr.splitlines():
# following 2 messages will not occur at the same time
index = line.find(selinux_warn1)
index2 = line.find(selinux_warn2)
if index == -1 and index2 == -1:
print line
continue
elif not selinux_warn_outputted:
# either index != -1 or index2 != -1
print line
print ("WARNING: selinux will not work correctly since the\n"
" --selinux option of tar command in this OS\n"
" is not fully supported\n")
selinux_warn_outputted = True
else:
# same line of selinux_warn1 or selinux_warn2
continue
return proc.returncode
def wait_partition_to_settle(media, part_table):
"""Sleep in a loop to wait partition to settle
:param media: A setup_partitions.Media object to partition.
"""
tts = 1
while (tts > 0) and (tts <= MAX_TTS):
try:
logger.info("Sleeping for %s second(s) to wait "
"for the partition to settle" % tts)
time.sleep(tts)
args = ['sfdisk', '-l', media.path]
if part_table == 'gpt':
args = ['sgdisk', '-L', media.path]
proc = cmd_runner.run(args,
as_root=True,
stdout=open('/dev/null', 'w'))
proc.wait()
return 0
except cmd_runner.SubcommandNonZeroReturnValue:
logger.info("Partition table is not available "
"for device %s" % media.path)
tts += 1
logger.error("Couldn't read partition table "
"for a reasonable time for device %s" % media.path)
raise
def wait_partition_to_settle(media, part_table):
"""Sleep in a loop to wait partition to settle
:param media: A setup_partitions.Media object to partition.
"""
tts = 1
while (tts > 0) and (tts <= MAX_TTS):
try:
logger.info("Sleeping for %s second(s) to wait "
"for the partition to settle" % tts)
time.sleep(tts)
args = ['sfdisk', '-l', media.path]
if part_table == 'gpt':
args = ['sgdisk', '-L', media.path]
proc = cmd_runner.run(args, as_root=True,
stdout=open('/dev/null', 'w'))
proc.wait()
return 0
except cmd_runner.SubcommandNonZeroReturnValue:
logger.info("Partition table is not available "
"for device %s" % media.path)
tts += 1
logger.error("Couldn't read partition table "
"for a reasonable time for device %s" % media.path)
raise
def disable_automount():
"""Disables the desktop environment automount option.
This will work only under GNOME with dconf installed.
"""
logger = logging.getLogger(DEFAULT_LOGGER_NAME)
if has_command('dconf'):
logger.info("Disabling desktop environment automount option.")
try:
cmd_runner.run(['dconf', 'write', AUTOMOUNT_DCONF_KEY, 'false'],
stdout=open('/dev/null', 'w')).wait()
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_OPEN_DCONF_KEYU, 'false'],
stdout=open('/dev/null', 'w')).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
logger.error("Error disabling desktop environemnt automount.")
def test_run(self):
fixture = self.useFixture(MockCmdRunnerPopenFixture())
proc = cmd_runner.run(['foo', 'bar', 'baz'])
# Call wait or else MockCmdRunnerPopenFixture() raises an
# AssertionError().
proc.wait()
self.assertEqual(0, proc.returncode)
self.assertEqual(['foo bar baz'], fixture.mock.commands_executed)
def test_run(self):
fixture = self.useFixture(MockCmdRunnerPopenFixture())
proc = cmd_runner.run(['foo', 'bar', 'baz'])
# Call wait or else MockCmdRunnerPopenFixture() raises an
# AssertionError().
proc.wait()
self.assertEqual(0, proc.returncode)
self.assertEqual(['foo bar baz'], fixture.mock.commands_executed)
def get_uuid(partition):
"""Find UUID of the given partition."""
proc = cmd_runner.run(
['blkid', '-o', 'udev', '-p', '-c', '/dev/null', partition],
as_root=True,
stdout=subprocess.PIPE)
blkid_output, _ = proc.communicate()
return _parse_blkid_output(blkid_output)
def enable_automount():
"""Re-enables back the desktop environment automount option.
This will work only under GNOME with dconf installed. It should be run
as an atexit function.
"""
logger = logging.getLogger(DEFAULT_LOGGER_NAME)
if has_command('dconf'):
try:
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_DCONF_KEY, 'true'],
stdout=open('/dev/null', 'w')).wait()
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_OPEN_DCONF_KEYU, 'true'],
stdout=open('/dev/null', 'w')).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
logger.error("Error enabling back desktop environemnt automount.")
def get_version():
qemu_path = '/usr/bin/qemu-arm-static'
p = cmd_runner.run(["head", "-n", "1"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
if os.path.exists(qemu_path):
try:
# qemu-arm-static has no --version option so it fails,
# but still prints its version plus usage
cmd_runner.run(["/usr/bin/qemu-arm-static", "--version"],
stdout=p.stdin).communicate()
p.communicate()
except:
qemu_version = p.stdout.read()
else:
qemu_version = "Cannot find %s." % qemu_path
return "%s\n: %s" % (__version__, qemu_version)
def get_uuid(partition):
"""Find UUID of the given partition."""
proc = cmd_runner.run(
['blkid', '-o', 'udev', '-p', '-c', '/dev/null', partition],
as_root=True,
stdout=subprocess.PIPE)
blkid_output, _ = proc.communicate()
return _parse_blkid_output(blkid_output)
def install_hwpacks(
rootfs_dir, tmp_dir, tools_dir, hwpack_force_yes, verified_files,
extract_kpkgs=False, *hwpack_files):
"""Install the given hwpacks onto the given rootfs."""
# In case we just want to extract the kernel packages, don't force qemu
# with chroot, as we could have archs without qemu support
if not extract_kpkgs:
prepare_chroot(rootfs_dir, tmp_dir)
linaro_hwpack_install_path = find_command(
'linaro-hwpack-install', prefer_dir=tools_dir)
# FIXME: shouldn't use chroot/usr/bin as this might conflict with
# installed packages; would be best to use some custom directory like
# chroot/linaro-image-tools/bin
copy_file(linaro_hwpack_install_path,
os.path.join(rootfs_dir, 'usr', 'bin'))
mount_chroot_proc(rootfs_dir)
try:
# Sometimes the host will have qemu-user-static installed but
# another package (i.e. scratchbox) will have mangled its config
# and thus we won't be able to chroot and install the hwpack, so
# we fail here and tell the user to ensure qemu-arm-static is
# setup before trying again.
cmd_runner.run(['true'], as_root=True, chroot=rootfs_dir).wait()
except:
print ("Cannot proceed with hwpack installation because "
"there doesn't seem to be a binfmt interpreter registered "
"to execute armel binaries in the chroot. Please check "
"that qemu-user-static is installed and properly "
"configured before trying again.")
raise
try:
for hwpack_file in hwpack_files:
hwpack_verified = False
if os.path.basename(hwpack_file) in verified_files:
hwpack_verified = True
install_hwpack(rootfs_dir, hwpack_file, extract_kpkgs,
hwpack_force_yes or hwpack_verified)
finally:
run_local_atexit_funcs()
def disable_automount():
"""Disables the desktop environment automount option.
This will work only under GNOME with dconf installed.
"""
logger = logging.getLogger(DEFAULT_LOGGER_NAME)
if has_command('dconf'):
logger.info("Disabling desktop environment automount option.")
try:
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_DCONF_KEY, 'false'],
stdout=open('/dev/null', 'w')).wait()
cmd_runner.run(
['dconf', 'write', AUTOMOUNT_OPEN_DCONF_KEYU, 'false'],
stdout=open('/dev/null', 'w')).wait()
except cmd_runner.SubcommandNonZeroReturnValue:
logger.error("Error disabling desktop environemnt automount.")
def unpack_binary_tarball(tarball, unpack_dir, as_root=True):
extract_opt = '-xf'
if tarball.endswith('.xz'):
extract_opt = '-Jxf'
proc = cmd_runner.run(
['tar', '--numeric-owner', '-C', unpack_dir, extract_opt, tarball],
as_root=as_root)
proc.wait()
return proc.returncode
def unpack_binary_tarball(tarball, unpack_dir, as_root=True):
extract_opt = '-xf'
if tarball.endswith('.xz'):
extract_opt = '-Jxf'
proc = cmd_runner.run(
['tar', '--numeric-owner', '-C', unpack_dir, extract_opt, tarball],
as_root=as_root)
proc.wait()
return proc.returncode
def populate_raw_partition(self, media, boot_dir):
# To avoid adding a Snowball specific command line option, we assume
# that the user already has unpacked the startfiles to ./startupfiles
config_files_dir = self.snowball_config(boot_dir)
assert os.path.exists(config_files_dir), (
"You need to unpack the Snowball startupfiles to the directory "
"'startupfiles' in your current working directory. See "
"igloocommunity.org for more information.")
# We copy the u-boot files from the unpacked boot.tar.bz2
# and put it with the startfiles.
boot_files = ['u-boot.bin']
for boot_file in boot_files:
cmd_runner.run([
'cp',
os.path.join(boot_dir, 'boot', boot_file), config_files_dir
],
as_root=True).wait()
super(AndroidSnowballEmmcConfig,
self).populate_raw_partition(media, boot_dir)
def _list_files(directory):
"""List the files and dirs under the given directory.
Runs as root because we want to list everything, including stuff that may
not be world-readable.
"""
p = cmd_runner.run(
['find', directory, '-maxdepth', '1', '-mindepth', '1'],
stdout=subprocess.PIPE, as_root=True)
stdout, _ = p.communicate()
return stdout.split()
def install_packages(chroot_dir, tmp_dir, *packages):
"""Install packages in the given chroot.
This does not run apt-get update before hand."""
prepare_chroot(chroot_dir, tmp_dir)
try:
# TODO: Use the partition_mounted() contextmanager here and get rid of
# mount_chroot_proc() altogether.
mount_chroot_proc(chroot_dir)
print "-" * 60
print "Installing (apt-get) %s in target rootfs." % " ".join(packages)
args = ("apt-get", "--yes", "install") + packages
cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait()
print "Cleaning up downloaded packages."
args = ("apt-get", "clean")
cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait()
print "-" * 60
finally:
run_local_atexit_funcs()
def partition_mounted(device, path, *args):
"""A context manager that mounts the given device and umounts when done.
We use a try/finally to make sure the device is umounted even if there's
an uncaught exception in the with block.
:param *args: Extra arguments to the mount command.
"""
subprocess_args = ['mount', device, path]
subprocess_args.extend(args)
cmd_runner.run(subprocess_args, as_root=True).wait()
try:
yield
finally:
try:
umount(path)
except cmd_runner.SubcommandNonZeroReturnValue, e:
logger.warn("Failed to umount %s, but ignoring it because of a "
"previous error" % path)
logger.warn(e)
def test_existing_command(self):
lmc = 'linaro-media-create'
prefer_dir = preferred_tools_dir()
if prefer_dir is None:
expected, _ = cmd_runner.run(
['which', lmc, ],
stdout=subprocess.PIPE).communicate()
expected = expected.strip()
else:
expected = os.path.join(prefer_dir, lmc)
self.assertEquals(expected, find_command(lmc))
def partition_mounted(device, path, *args):
"""A context manager that mounts the given device and umounts when done.
We use a try/finally to make sure the device is umounted even if there's
an uncaught exception in the with block.
:param *args: Extra arguments to the mount command.
"""
subprocess_args = ['mount', device, path]
subprocess_args.extend(args)
cmd_runner.run(subprocess_args, as_root=True).wait()
try:
yield
finally:
try:
umount(path)
except cmd_runner.SubcommandNonZeroReturnValue, e:
logger.warn("Failed to umount %s, but ignoring it because of a "
"previous error" % path)
logger.warn(e)
def install_packages(chroot_dir, tmp_dir, *packages):
"""Install packages in the given chroot.
This does not run apt-get update before hand."""
prepare_chroot(chroot_dir, tmp_dir)
try:
# TODO: Use the partition_mounted() contextmanager here and get rid of
# mount_chroot_proc() altogether.
mount_chroot_proc(chroot_dir)
print "-" * 60
print "Installing (apt-get) %s in target rootfs." % " ".join(packages)
args = ("apt-get", "--yes", "install") + packages
cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait()
print "Cleaning up downloaded packages."
args = ("apt-get", "clean")
cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait()
print "-" * 60
finally:
run_local_atexit_funcs()
def get_version():
qemu_path = '/usr/bin/qemu-arm-static'
if os.path.exists(qemu_path):
# qemu-arm-static has -version option
proc = cmd_runner.run([qemu_path, "-version"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(qemu_version, stderrdata) = proc.communicate()
if (proc.returncode or stderrdata):
qemu_version = "qemu-arm version unknown (%s)" % stderrdata
else:
qemu_version = "Cannot find %s." % qemu_path
return "%s\n* %s" % (__version__, qemu_version)
def register_loopback(image_file, offset, size):
"""Register a loopback device with an atexit handler to de-register it."""
def undo(device):
cmd_runner.run(['losetup', '-d', device], as_root=True).wait()
proc = cmd_runner.run(
['losetup', '-f', '--show', image_file, '--offset',
str(offset), '--sizelimit', str(size)],
stdout=subprocess.PIPE, as_root=True)
device, _ = proc.communicate()
device = device.strip()
atexit.register(undo, device)
return device
def test_existing_command(self):
lmc = 'linaro-media-create'
prefer_dir = preferred_tools_dir()
if prefer_dir is None:
expected, _ = cmd_runner.run([
'which',
lmc,
],
stdout=subprocess.PIPE).communicate()
expected = expected.strip()
else:
expected = os.path.join(prefer_dir, lmc)
self.assertEquals(expected, find_command(lmc))