def test_make_xfs_empty_options(self, mock_exec):
fu.make_fs('xfs', '', '', '/dev/fake')
expected_calls = [
mock.call('mkfs.xfs', '-f', '/dev/fake'),
mock.call('blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', '/dev/fake')
]
self.assertEqual(mock_exec.call_args_list, expected_calls)
def test_make_fs_swap(self, mock_exec):
fu.make_fs('swap', '', 'fake_label', '/dev/fake')
expected_calls = [
mock.call('mkswap', '-f', '-L', 'fake_label', '/dev/fake'),
mock.call('blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', '/dev/fake')
]
self.assertEqual(mock_exec.call_args_list, expected_calls)
def do_clean_filesystems(self):
# All fs without keep_data flag and without image should be cleaned,
# mkfs will clean file table and such flags as --force are not needed
# TODO(asvechnikov): need to refactor processing keep_flag logic when
# data model will become flat
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not fs.keep_data and not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def do_clean_filesystems(self):
# NOTE(agordeev): it turns out that only mkfs.xfs needs '-f' flag in
# order to force recreation of filesystem.
# This option will be added to mkfs.xfs call explicitly in fs utils.
# TODO(asvechnikov): need to refactor processing keep_flag logic when
# data model will become flat
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images if img.target_device == fs.device]
if not fs.keep_data and not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def do_clean_filesystems(self):
# NOTE(agordeev): it turns out that only mkfs.xfs needs '-f' flag in
# order to force recreation of filesystem.
# This option will be added to mkfs.xfs call explicitly in fs utils.
# TODO(asvechnikov): need to refactor processing keep_flag logic when
# data model will become flat
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not fs.keep_data and not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def test_make_fs_bad_swap_failure(self):
# We mock utils.execute to throw an exception on invocations
# of blkid (MAX_MKFS_TRIES times) to see if it fails.
rvs = fu.MAX_MKFS_TRIES * [None, errors.ProcessExecutionError()]
with mock.patch.object(utils, 'execute', side_effect=rvs) as mock_exec:
with self.assertRaises(errors.FsUtilsError):
fu.make_fs('swap', '', 'fake_label', '/dev/fake')
expected_calls = 3 * [
mock.call('mkswap', '-f', '-L', 'fake_label', '/dev/fake'),
mock.call('blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', '/dev/fake')
]
self.assertEqual(mock_exec.call_args_list, expected_calls)
def test_make_fs_bad_swap_retry(self):
# We mock utils.execute to throw an exception on first two
# invocations of blkid to test the retry loop.
rvs = [
None, errors.ProcessExecutionError(),
None, errors.ProcessExecutionError(),
None, None
]
with mock.patch.object(utils, 'execute', side_effect=rvs) as mock_exec:
fu.make_fs('swap', '', 'fake_label', '/dev/fake')
expected_calls = 3 * [
mock.call('mkswap', '-f', '-L', 'fake_label', '/dev/fake'),
mock.call('blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', '/dev/fake')
]
self.assertEqual(mock_exec.call_args_list, expected_calls)
def test_make_fs_swap(self, mock_exec):
fu.make_fs('swap', '-f', '-L fake_label', '/dev/fake')
mock_exec.assert_called_once_with('mkswap', '-f', '-L', 'fake_label',
'/dev/fake')
def test_make_fs(self, mock_exec):
fu.make_fs('ext4', '-F', '-L fake_label', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.ext4', '-F', '-L',
'fake_label', '/dev/fake')
def test_make_xfs_empty_options(self, mock_exec):
fu.make_fs('xfs', '', '', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.xfs', '-f', '/dev/fake')
def test_make_xfs_add_f_flag(self, mock_exec):
fu.make_fs('xfs', '--other-options --passed', '', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.xfs', '--other-options',
'--passed', '-f', '/dev/fake')
def test_make_fs_swap(self, mock_exec):
fu.make_fs('swap', '-f', '-L fake_label', '/dev/fake')
mock_exec.assert_called_once_with('mkswap', '-f', '-L', 'fake_label',
'/dev/fake')
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
# If disks are not wiped out at all, it is likely they contain lvm
# and md metadata which will prevent re-creating a partition table
# with 'device is busy' error.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
# Here is udev's rules blacklisting to be done:
# by adding symlinks to /dev/null in /etc/udev/rules.d for already
# existent rules in /lib/.
# 'parted' generates too many udev events in short period of time
# so we should increase processing speed for those events,
# otherwise partitioning is doomed.
empty_rule_path = os.path.join(CONF.udev_rules_dir,
os.path.basename(CONF.udev_empty_rule))
with open(empty_rule_path, 'w') as f:
f.write('#\n')
LOG.debug("Enabling udev's rules blacklisting")
for rule in os.listdir(CONF.udev_rules_lib_dir):
dst = os.path.join(CONF.udev_rules_dir, rule)
if os.path.isdir(dst):
continue
if dst.endswith('.rules'):
# for successful blacklisting already existent file with name
# from /etc which overlaps with /lib should be renamed prior
# symlink creation.
try:
if os.path.exists(dst):
os.rename(dst, dst[:-len('.rules')] +
CONF.udev_rename_substr)
except OSError:
LOG.debug("Skipping udev rule %s blacklising" % dst)
else:
os.symlink(empty_rule_path, dst)
utils.execute('udevadm', 'control', '--reload-rules',
check_exit_code=[0])
for parted in self.driver.partition_scheme.parteds:
for prt in parted.partitions:
# We wipe out the beginning of every new partition
# right after creating it. It allows us to avoid possible
# interactive dialog if some data (metadata or file system)
# present on this new partition and it also allows udev not
# hanging trying to parse this data.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.begin - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0])
# Also wipe out the ending of every new partition.
# Different versions of md stores metadata in different places.
# Adding exit code 1 to be accepted as for handling situation
# when 'no space left on device' occurs.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.end - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0, 1])
for parted in self.driver.partition_scheme.parteds:
pu.make_label(parted.name, parted.label)
for prt in parted.partitions:
pu.make_partition(prt.device, prt.begin, prt.end, prt.type)
for flag in prt.flags:
pu.set_partition_flag(prt.device, prt.count, flag)
if prt.guid:
pu.set_gpt_type(prt.device, prt.count, prt.guid)
# If any partition to be created doesn't exist it's an error.
# Probably it's again 'device or resource busy' issue.
if not os.path.exists(prt.name):
raise errors.PartitionNotFoundError(
'Partition %s not found after creation' % prt.name)
# disable udev's rules blacklisting
LOG.debug("Disabling udev's rules blacklisting")
for rule in os.listdir(CONF.udev_rules_dir):
src = os.path.join(CONF.udev_rules_dir, rule)
if os.path.isdir(src):
continue
if src.endswith('.rules'):
if os.path.islink(src):
try:
os.remove(src)
except OSError:
LOG.debug(
"Skipping udev rule %s de-blacklisting" % src)
elif src.endswith(CONF.udev_rename_substr):
try:
if os.path.exists(src):
os.rename(src, src[:-len(CONF.udev_rename_substr)] +
'.rules')
except OSError:
LOG.debug("Skipping udev rule %s de-blacklisting" % src)
utils.execute('udevadm', 'control', '--reload-rules',
check_exit_code=[0])
# NOTE(agordeev): re-create all the links which were skipped by udev
# while blacklisted
# NOTE(agordeev): do subsystem match, otherwise it will stuck
utils.execute('udevadm', 'trigger', '--subsystem-match=block',
check_exit_code=[0])
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
# If one creates partitions with the same boundaries as last time,
# there might be md and lvm metadata on those partitions. To prevent
# failing of creating md and lvm devices we need to make sure
# unused metadata are wiped out.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
# creating meta disks
for md in self.driver.partition_scheme.mds:
mu.mdcreate(md.name, md.level, *md.devices)
# creating physical volumes
for pv in self.driver.partition_scheme.pvs:
lu.pvcreate(pv.name, metadatasize=pv.metadatasize,
metadatacopies=pv.metadatacopies)
# creating volume groups
for vg in self.driver.partition_scheme.vgs:
lu.vgcreate(vg.name, *vg.pvnames)
# creating logical volumes
for lv in self.driver.partition_scheme.lvs:
lu.lvcreate(lv.vgname, lv.name, lv.size)
# making file systems
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
if self.driver.partition_scheme.skip_partitioning:
LOG.debug('Some of fs has keep_data flag, '
'partitioning is skiping')
self.do_clean_filesystems()
return
# If disks are not wiped out at all, it is likely they contain lvm
# and md metadata which will prevent re-creating a partition table
# with 'device is busy' error.
mu.mdclean_all(skip_containers=CONF.skip_md_containers)
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
for parted in self.driver.partition_scheme.parteds:
for prt in parted.partitions:
# We wipe out the beginning of every new partition
# right after creating it. It allows us to avoid possible
# interactive dialog if some data (metadata or file system)
# present on this new partition and it also allows udev not
# hanging trying to parse this data.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.begin - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0])
# Also wipe out the ending of every new partition.
# Different versions of md stores metadata in different places.
# Adding exit code 1 to be accepted as for handling situation
# when 'no space left on device' occurs.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.end - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0, 1])
parteds = []
parteds_with_rules = []
for parted in self.driver.partition_scheme.parteds:
if hw.is_multipath_device(parted.name):
parteds_with_rules.append(parted)
else:
parteds.append(parted)
utils.blacklist_udev_rules(udev_rules_dir=CONF.udev_rules_dir,
udev_rules_lib_dir=CONF.udev_rules_lib_dir,
udev_rename_substr=CONF.udev_rename_substr,
udev_empty_rule=CONF.udev_empty_rule)
self._make_partitions(parteds)
utils.unblacklist_udev_rules(
udev_rules_dir=CONF.udev_rules_dir,
udev_rename_substr=CONF.udev_rename_substr)
self._make_partitions(parteds_with_rules)
# If one creates partitions with the same boundaries as last time,
# there might be md and lvm metadata on those partitions. To prevent
# failing of creating md and lvm devices we need to make sure
# unused metadata are wiped out.
mu.mdclean_all(skip_containers=CONF.skip_md_containers)
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
if parteds_with_rules:
utils.refresh_multipath()
# creating meta disks
for md in self.driver.partition_scheme.mds:
mu.mdcreate(md.name, md.level, md.devices, md.metadata)
# creating physical volumes
for pv in self.driver.partition_scheme.pvs:
lu.pvcreate(pv.name, metadatasize=pv.metadatasize,
metadatacopies=pv.metadatacopies)
# creating volume groups
for vg in self.driver.partition_scheme.vgs:
lu.vgcreate(vg.name, *vg.pvnames)
# creating logical volumes
for lv in self.driver.partition_scheme.lvs:
lu.lvcreate(lv.vgname, lv.name, lv.size)
# making file systems
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def test_make_xfs_empty_options(self, mock_exec):
fu.make_fs('xfs', '', '', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.xfs', '-f', '/dev/fake')
def test_make_xfs_add_f_flag(self, mock_exec):
fu.make_fs('xfs', '--other-options --passed', '', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.xfs', '--other-options',
'--passed', '-f', '/dev/fake')
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata['os'] = self.driver.operating_system.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
try:
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(CONF.image_build_dir)
chroot = tempfile.mkdtemp(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug('Temporary chroot: %s', chroot)
proc_path = os.path.join(chroot, 'proc')
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
LOG.debug('Looking for a free loop device')
image.target_device.name = bu.get_free_loop_device(
loop_device_major_number=CONF.loop_device_major_number,
max_loop_devices_count=CONF.max_loop_devices_count)
LOG.debug('Attaching temporary image file to free loop device')
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=str(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('*** Shipping image content ***')
LOG.debug('Installing operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'
'suite={suite}, section={section}'.format(
name=repo.name, uri=repo.uri,
suite=repo.suite, section=repo.section))
bu.add_apt_source(
name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section,
chroot=chroot)
LOG.debug('Adding repository preference: '
'name={name}, priority={priority}'.format(
name=repo.name, priority=repo.priority))
if repo.priority is not None:
bu.add_apt_preference(
name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri)
metadata.setdefault('repos', []).append({
'type': 'deb',
'name': repo.name,
'uri': repo.uri,
'suite': repo.suite,
'section': repo.section,
'priority': repo.priority,
'meta': repo.meta})
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata['packages'] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
LOG.debug('Deattaching loop device from file: %s',
image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container,
chunk_size=CONF.data_chunk_size)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(
img_containerized, container_size)
metadata.setdefault('images', []).append({
'raw_md5': raw_md5,
'raw_size': raw_size,
'raw_name': None,
'container_name': os.path.basename(img_containerized),
'container_md5': container_md5,
'container_size': container_size,
'container': image.container,
'format': image.format})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'wt', encoding='utf-8') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', proc_path)
fu.umount_fs(proc_path)
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
LOG.debug('Finally: detaching loop device: %s',
str(image.target_device))
try:
bu.deattach_loop(str(image.target_device))
except errors.ProcessExecutionError as e:
LOG.warning('Error occured while trying to detach '
'loop device %s. Error message: %s',
str(image.target_device), e)
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug('Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
LOG.debug('Finally: removing chroot directory: %s', chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
if self.driver.partition_scheme.skip_partitioning:
LOG.debug('Some of fs has keep_data flag, '
'partitioning is skiping')
self.do_clean_filesystems()
return
# If disks are not wiped out at all, it is likely they contain lvm
# and md metadata which will prevent re-creating a partition table
# with 'device is busy' error.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
LOG.debug("Enabling udev's rules blacklisting")
utils.blacklist_udev_rules(udev_rules_dir=CONF.udev_rules_dir,
udev_rules_lib_dir=CONF.udev_rules_lib_dir,
udev_rename_substr=CONF.udev_rename_substr,
udev_empty_rule=CONF.udev_empty_rule)
for parted in self.driver.partition_scheme.parteds:
for prt in parted.partitions:
# We wipe out the beginning of every new partition
# right after creating it. It allows us to avoid possible
# interactive dialog if some data (metadata or file system)
# present on this new partition and it also allows udev not
# hanging trying to parse this data.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.begin - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0])
# Also wipe out the ending of every new partition.
# Different versions of md stores metadata in different places.
# Adding exit code 1 to be accepted as for handling situation
# when 'no space left on device' occurs.
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
'seek=%s' % max(prt.end - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0, 1])
for parted in self.driver.partition_scheme.parteds:
pu.make_label(parted.name, parted.label)
for prt in parted.partitions:
pu.make_partition(prt.device, prt.begin, prt.end, prt.type)
for flag in prt.flags:
pu.set_partition_flag(prt.device, prt.count, flag)
if prt.guid:
pu.set_gpt_type(prt.device, prt.count, prt.guid)
# If any partition to be created doesn't exist it's an error.
# Probably it's again 'device or resource busy' issue.
if not os.path.exists(prt.name):
raise errors.PartitionNotFoundError(
'Partition %s not found after creation' % prt.name)
LOG.debug("Disabling udev's rules blacklisting")
utils.unblacklist_udev_rules(
udev_rules_dir=CONF.udev_rules_dir,
udev_rename_substr=CONF.udev_rename_substr)
# If one creates partitions with the same boundaries as last time,
# there might be md and lvm metadata on those partitions. To prevent
# failing of creating md and lvm devices we need to make sure
# unused metadata are wiped out.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
# creating meta disks
for md in self.driver.partition_scheme.mds:
mu.mdcreate(md.name, md.level, md.devices, md.metadata)
# creating physical volumes
for pv in self.driver.partition_scheme.pvs:
lu.pvcreate(pv.name, metadatasize=pv.metadatasize,
metadatacopies=pv.metadatacopies)
# creating volume groups
for vg in self.driver.partition_scheme.vgs:
lu.vgcreate(vg.name, *vg.pvnames)
# creating logical volumes
for lv in self.driver.partition_scheme.lvs:
lu.lvcreate(lv.vgname, lv.name, lv.size)
# making file systems
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
# If disks are not wiped out at all, it is likely they contain lvm
# and md metadata which will prevent re-creating a partition table
# with 'device is busy' error.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
# Here is udev's rules blacklisting to be done:
# by adding symlinks to /dev/null in /etc/udev/rules.d for already
# existent rules in /lib/.
# 'parted' generates too many udev events in short period of time
# so we should increase processing speed for those events,
# otherwise partitioning is doomed.
empty_rule_path = os.path.join(CONF.udev_rules_dir,
os.path.basename(CONF.udev_empty_rule))
with open(empty_rule_path, 'w') as f:
f.write('#\n')
LOG.debug("Enabling udev's rules blacklisting")
for rule in os.listdir(CONF.udev_rules_lib_dir):
dst = os.path.join(CONF.udev_rules_dir, rule)
if os.path.isdir(dst):
continue
if dst.endswith('.rules'):
# for successful blacklisting already existent file with name
# from /etc which overlaps with /lib should be renamed prior
# symlink creation.
try:
if os.path.exists(dst):
os.rename(
dst,
dst[:-len('.rules')] + CONF.udev_rename_substr)
except OSError:
LOG.debug("Skipping udev rule %s blacklising" % dst)
else:
os.symlink(empty_rule_path, dst)
utils.execute('udevadm',
'control',
'--reload-rules',
check_exit_code=[0])
for parted in self.driver.partition_scheme.parteds:
for prt in parted.partitions:
# We wipe out the beginning of every new partition
# right after creating it. It allows us to avoid possible
# interactive dialog if some data (metadata or file system)
# present on this new partition and it also allows udev not
# hanging trying to parse this data.
utils.execute('dd',
'if=/dev/zero',
'bs=1M',
'seek=%s' % max(prt.begin - 3, 0),
'count=5',
'of=%s' % prt.device,
check_exit_code=[0])
# Also wipe out the ending of every new partition.
# Different versions of md stores metadata in different places.
# Adding exit code 1 to be accepted as for handling situation
# when 'no space left on device' occurs.
utils.execute('dd',
'if=/dev/zero',
'bs=1M',
'seek=%s' % max(prt.end - 3, 0),
'count=5',
'of=%s' % prt.device,
check_exit_code=[0, 1])
for parted in self.driver.partition_scheme.parteds:
pu.make_label(parted.name, parted.label)
for prt in parted.partitions:
pu.make_partition(prt.device, prt.begin, prt.end, prt.type)
for flag in prt.flags:
pu.set_partition_flag(prt.device, prt.count, flag)
if prt.guid:
pu.set_gpt_type(prt.device, prt.count, prt.guid)
# If any partition to be created doesn't exist it's an error.
# Probably it's again 'device or resource busy' issue.
if not os.path.exists(prt.name):
raise errors.PartitionNotFoundError(
'Partition %s not found after creation' % prt.name)
# disable udev's rules blacklisting
LOG.debug("Disabling udev's rules blacklisting")
for rule in os.listdir(CONF.udev_rules_dir):
src = os.path.join(CONF.udev_rules_dir, rule)
if os.path.isdir(src):
continue
if src.endswith('.rules'):
if os.path.islink(src):
try:
os.remove(src)
except OSError:
LOG.debug("Skipping udev rule %s de-blacklisting" %
src)
elif src.endswith(CONF.udev_rename_substr):
try:
if os.path.exists(src):
os.rename(
src,
src[:-len(CONF.udev_rename_substr)] + '.rules')
except OSError:
LOG.debug("Skipping udev rule %s de-blacklisting" % src)
utils.execute('udevadm',
'control',
'--reload-rules',
check_exit_code=[0])
# NOTE(agordeev): re-create all the links which were skipped by udev
# while blacklisted
# NOTE(agordeev): do subsystem match, otherwise it will stuck
utils.execute('udevadm',
'trigger',
'--subsystem-match=block',
check_exit_code=[0])
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
# If one creates partitions with the same boundaries as last time,
# there might be md and lvm metadata on those partitions. To prevent
# failing of creating md and lvm devices we need to make sure
# unused metadata are wiped out.
mu.mdclean_all()
lu.lvremove_all()
lu.vgremove_all()
lu.pvremove_all()
# creating meta disks
for md in self.driver.partition_scheme.mds:
mu.mdcreate(md.name, md.level, *md.devices)
# creating physical volumes
for pv in self.driver.partition_scheme.pvs:
lu.pvcreate(pv.name,
metadatasize=pv.metadatasize,
metadatacopies=pv.metadatacopies)
# creating volume groups
for vg in self.driver.partition_scheme.vgs:
lu.vgcreate(vg.name, *vg.pvnames)
# creating logical volumes
for lv in self.driver.partition_scheme.lvs:
lu.lvcreate(lv.vgname, lv.name, lv.size)
# making file systems
for fs in self.driver.partition_scheme.fss:
found_images = [
img for img in self.driver.image_scheme.images
if img.target_device == fs.device
]
if not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def test_make_fs(self, mock_exec):
fu.make_fs('ext4', '-F', '-L fake_label', '/dev/fake')
mock_exec.assert_called_once_with('mkfs.ext4', '-F', '-L',
'fake_label', '/dev/fake')
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([
os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images
]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
try:
LOG.debug('Creating temporary chroot directory')
chroot = tempfile.mkdtemp(dir=CONF.image_build_dir,
suffix=CONF.image_build_suffix)
LOG.debug('Temporary chroot: %s', chroot)
proc_path = os.path.join(chroot, 'proc')
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug(
'Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
LOG.debug('Looking for a free loop device')
image.target_device.name = bu.get_free_loop_device()
LOG.debug('Attaching temporary image file to free loop device')
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=str(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('*** Shipping image content ***')
LOG.debug('Installing operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'
'suite={suite}, section={section}'.format(
name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section))
bu.add_apt_source(name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section,
chroot=chroot)
LOG.debug('Adding repository preference: '
'name={name}, priority={priority}'.format(
name=repo.name, priority=repo.priority))
if repo.priority is not None:
bu.add_apt_preference(name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri)
metadata.setdefault('repos', []).append({
'type': 'deb',
'name': repo.name,
'uri': repo.uri,
'suite': repo.suite,
'section': repo.section,
'priority': repo.priority,
'meta': repo.meta
})
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata['packages'] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, '/proc')
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot,
signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
LOG.debug('Deattaching loop device from file: %s',
image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(img_containerized,
container_size)
metadata.setdefault('images', []).append({
'raw_md5':
raw_md5,
'raw_size':
raw_size,
'raw_name':
None,
'container_name':
os.path.basename(img_containerized),
'container_md5':
container_md5,
'container_size':
container_size,
'container':
image.container,
'format':
image.format
})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'w') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', proc_path)
fu.umount_fs(proc_path)
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
for image in self.driver.image_scheme.images:
LOG.debug('Finally: detaching loop device: %s',
str(image.target_device))
try:
bu.deattach_loop(str(image.target_device))
except errors.ProcessExecutionError as e:
LOG.warning(
'Error occured while trying to detach '
'loop device %s. Error message: %s',
str(image.target_device), e)
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug(
'Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
LOG.debug('Finally: removing chroot directory: %s', chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug(
'Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info("--- Building image (do_build_image) ---")
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata["os"] = self.driver.operating_system.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info("*** Checking if image exists ***")
if all([os.path.exists(img.uri.split("file://", 1)[1]) for img in self.driver.image_scheme.images]):
LOG.debug("All necessary images are available. " "Nothing needs to be done.")
return
LOG.debug("At least one of the necessary images is unavailable. " "Starting build process.")
try:
LOG.debug("Creating temporary chroot directory")
utils.makedirs_if_not_exists(CONF.image_build_dir)
chroot = tempfile.mkdtemp(dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug("Temporary chroot: %s", chroot)
proc_path = os.path.join(chroot, "proc")
LOG.info("*** Preparing image space ***")
for image in self.driver.image_scheme.images:
LOG.debug("Creating temporary sparsed file for the " "image: %s", image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix, size=CONF.sparse_file_size
)
LOG.debug("Temporary file: %s", img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image,
)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
LOG.debug("Creating file system on the image")
fu.make_fs(fs_type=fs.type, fs_options=fs.options, fs_label=fs.label, dev=str(fs.device))
if fs.type == "ext4":
LOG.debug("Trying to disable journaling for ext4 " "in order to speed up the build")
utils.execute("tune2fs", "-O", "^has_journal", str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info("*** Shipping image content ***")
LOG.debug("Installing operating system into image")
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug("Preventing services from being get started")
bu.suppress_services_start(chroot)
LOG.debug("Installing base operating system using debootstrap")
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, attempts=CONF.fetch_packages_attempts)
# APT-GET
LOG.debug("Configuring apt inside chroot")
LOG.debug("Setting environment variables")
bu.set_apt_get_env()
LOG.debug("Allowing unauthenticated repos")
bu.pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)
for repo in self.driver.operating_system.repos:
LOG.debug(
"Adding repository source: name={name}, uri={uri},"
"suite={suite}, section={section}".format(
name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section
)
)
bu.add_apt_source(name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section, chroot=chroot)
LOG.debug(
"Adding repository preference: "
"name={name}, priority={priority}".format(name=repo.name, priority=repo.priority)
)
if repo.priority is not None:
bu.add_apt_preference(
name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri,
)
metadata.setdefault("repos", []).append(
{
"type": "deb",
"name": repo.name,
"uri": repo.uri,
"suite": repo.suite,
"section": repo.section,
"priority": repo.priority,
"meta": repo.meta,
}
)
LOG.debug("Preventing services from being get started")
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata["packages"] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, "/proc")
bu.populate_basic_dev(chroot)
LOG.debug("Installing packages using apt-get: %s", " ".join(packages))
bu.run_apt_get(chroot, packages=packages, attempts=CONF.fetch_packages_attempts)
LOG.debug("Post-install OS configuration")
bu.do_post_inst(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)
LOG.debug("Making sure there are no running processes " "inside chroot before trying to umount chroot")
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
"Stopping chrooted processes failed. " "There are some processes running in chroot %s", chroot
)
LOG.info("*** Finalizing image space ***")
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
if fs.type == "ext4":
LOG.debug("Trying to re-enable journaling for ext4")
utils.execute("tune2fs", "-O", "has_journal", str(fs.device))
LOG.debug("Deattaching loop device from file: %s", image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug("Shrinking temporary image file: %s", image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug("Containerizing temporary image file: %s", image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container, chunk_size=CONF.data_chunk_size
)
img_containerized = image.uri.split("file://", 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug("Moving image file to the final location: %s", img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(img_containerized, container_size)
metadata.setdefault("images", []).append(
{
"raw_md5": raw_md5,
"raw_size": raw_size,
"raw_name": None,
"container_name": os.path.basename(img_containerized),
"container_md5": container_md5,
"container_size": container_size,
"container": image.container,
"format": image.format,
}
)
# NOTE(kozhukalov): implement abstract publisher
LOG.debug("Image metadata: %s", metadata)
with open(self.driver.metadata_uri.split("file://", 1)[1], "wt", encoding="utf-8") as f:
yaml.safe_dump(metadata, stream=f)
LOG.info("--- Building image END (do_build_image) ---")
except Exception as exc:
LOG.error("Failed to build image: %s", exc)
raise
finally:
LOG.debug("Finally: stopping processes inside chroot: %s", chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug("Finally: umounting procfs %s", proc_path)
fu.umount_fs(proc_path)
LOG.debug("Finally: umounting chroot tree %s", chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
if image.target_device.name:
LOG.debug("Finally: detaching loop device: %s", image.target_device.name)
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning(
"Error occured while trying to detach " "loop device %s. Error message: %s",
image.target_device.name,
e,
)
if image.img_tmp_file:
LOG.debug("Finally: removing temporary file: %s", image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug("Finally: file %s seems does not exist " "or can not be removed", image.img_tmp_file)
LOG.debug("Finally: removing chroot directory: %s", chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug("Finally: directory %s seems does not exist " "or can not be removed", chroot)