def test_umount_fs_ok(self, mock_exec):
fu.umount_fs('/fake')
expected_calls = [
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
mock.call('umount', '/fake', check_exit_code=[0])
]
self.assertEqual(expected_calls, mock_exec.call_args_list)
def destroy_chroot(self, chroot):
# Umount chroot tree and remove images tmp files
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', os.path.join(chroot, 'proc'))
fu.umount_fs(os.path.join(chroot, 'proc'))
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
if image.target_device.name:
LOG.debug('Finally: detaching loop device: %s',
image.target_device.name)
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning(
'Error occured while trying to detach '
'loop device %s. Error message: %s',
image.target_device.name, e)
if image.img_tmp_file:
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug(
'Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
try:
os.rmdir(chroot)
except OSError:
LOG.debug(
'Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def test_umount_fs_not_mounted(self, mock_exec):
mock_exec.side_effect = errors.ProcessExecutionError
fu.umount_fs('/fake')
mock_exec.assert_called_once_with('mountpoint',
'-q',
'/fake',
check_exit_code=[0])
def test_umount_fs_ok(self, mock_exec):
fu.umount_fs('/fake')
expected_calls = [
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
mock.call('umount', '/fake', check_exit_code=[0])
]
self.assertEqual(expected_calls, mock_exec.call_args_list)
def destroy_chroot(self, chroot):
# Umount chroot tree and remove images tmp files
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', os.path.join(chroot, 'proc'))
fu.umount_fs(os.path.join(chroot, 'proc'))
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
if image.target_device.name:
LOG.debug('Finally: detaching loop device: %s',
image.target_device.name)
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning('Error occured while trying to detach '
'loop device %s. Error message: %s',
image.target_device.name, e)
if image.img_tmp_file:
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug('Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
try:
os.rmdir(chroot)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def _make_configdrive_image(self, src_files):
bs = 4096
configdrive_device = self.driver.partition_scheme.configdrive_device()
size = utils.execute('blockdev', '--getsize64', configdrive_device)[0]
size = int(size.strip())
utils.execute('truncate', '--size=%d' % size, CONF.config_drive_path)
fu.make_fs(
fs_type='ext2',
fs_options=' -b %d -F ' % bs,
fs_label='config-2',
dev=six.text_type(CONF.config_drive_path))
mount_point = tempfile.mkdtemp(dir=CONF.tmp_path)
try:
fu.mount_fs('ext2', CONF.config_drive_path, mount_point)
for file_path in src_files:
name = os.path.basename(file_path)
if os.path.isdir(file_path):
shutil.copytree(file_path, os.path.join(mount_point, name))
else:
shutil.copy2(file_path, mount_point)
except Exception as exc:
LOG.error('Error copying files to configdrive: %s', exc)
raise
finally:
fu.umount_fs(mount_point)
os.rmdir(mount_point)
def umount_target_flat(self, mount_map):
"""Umount file systems previously mounted into temporary directories.
:param mount_map: Mount map dict
"""
for mount_point in six.itervalues(mount_map):
fu.umount_fs(mount_point)
shutil.rmtree(mount_point)
def test_umount_fs_error(self, mock_exec):
mock_exec.side_effect = [
None, errors.ProcessExecutionError('message'), ('', '')]
fu.umount_fs('/fake', try_lazy_umount=True)
expected_calls = [
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
mock.call('umount', '/fake', check_exit_code=[0]),
mock.call('umount', '-l', '/fake', check_exit_code=[0])
]
self.assertEqual(expected_calls, mock_exec.call_args_list)
def umount_target(self, chroot, pseudo=True):
LOG.debug('Umounting target file systems: %s', chroot)
if pseudo:
# umount fusectl (typically mounted at /sys/fs/fuse/connections)
for path in ('/proc', '/dev', '/sys/fs/fuse/connections', '/sys'):
fu.umount_fs(chroot + path)
for fs in self.driver.partition_scheme.fs_sorted_by_depth(
reverse=True):
if fs.mount == 'swap':
continue
fu.umount_fs(chroot + fs.mount)
def test_umount_fs_error(self, mock_exec):
mock_exec.side_effect = [
None, errors.ProcessExecutionError('message'), ('', '')
]
fu.umount_fs('/fake', try_lazy_umount=True)
expected_calls = [
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
mock.call('umount', '/fake', check_exit_code=[0]),
mock.call('umount', '-l', '/fake', check_exit_code=[0])
]
self.assertEqual(expected_calls, mock_exec.call_args_list)
def _umount_target(self, mount_dir, os_id=None, pseudo=True):
LOG.debug('Umounting target file systems: %s', mount_dir)
if pseudo:
# umount fusectl (typically mounted at /sys/fs/fuse/connections)
for path in ('/proc', '/dev', '/sys/fs/fuse/connections', '/sys'):
fu.umount_fs(os.path.join(mount_dir, path.strip(os.sep)),
try_lazy_umount=True)
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id, True):
if fs.mount == 'swap':
continue
fu.umount_fs(os.path.join(mount_dir, fs.mount.strip(os.sep)))
def umount_target(self, chroot, pseudo=True):
LOG.debug('Umounting target file systems: %s', chroot)
if pseudo:
# umount fusectl (typically mounted at /sys/fs/fuse/connections)
for path in ('/proc', '/dev', '/sys/fs/fuse/connections', '/sys'):
fu.umount_fs(chroot + path)
for fs in self.driver.partition_scheme.fs_sorted_by_depth(
reverse=True):
if fs.mount == 'swap':
continue
fu.umount_fs(chroot + fs.mount)
def _umount_target(self, mount_dir, os_id=None, pseudo=True):
LOG.debug('Umounting target file systems: %s', mount_dir)
if pseudo:
# umount fusectl (typically mounted at /sys/fs/fuse/connections)
for path in ('/proc', '/dev', '/sys/fs/fuse/connections', '/sys'):
fu.umount_fs(os.path.join(mount_dir, path.strip(os.sep)),
try_lazy_umount=True)
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id,
True):
if fs.mount == 'swap':
continue
fu.umount_fs(os.path.join(mount_dir, fs.mount.strip(os.sep)))
def _mount_bootloader(self, mount_dir):
fs = filter(lambda fss: fss.mount == 'multiboot',
self.driver.partition_scheme.fss)
if len(fs) > 1:
raise errors.WrongPartitionSchemeError(
'Multiple multiboot partitions found')
utils.makedirs_if_not_exists(mount_dir)
fu.mount_fs(fs[0].type, str(fs[0].device), mount_dir)
yield pu.get_uuid(fs[0].device)
fu.umount_fs(mount_dir)
def _mount_bootloader(self, mount_dir):
fs = filter(lambda fss: fss.mount == 'multiboot',
self.driver.partition_scheme.fss)
if len(fs) > 1:
raise errors.WrongPartitionSchemeError(
'Multiple multiboot partitions found')
utils.makedirs_if_not_exists(mount_dir)
fu.mount_fs(fs[0].type, str(fs[0].device), mount_dir)
yield pu.get_uuid(fs[0].device)
fu.umount_fs(mount_dir)
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs('tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute('chroot',
chroot,
'mksquashfs',
'/mnt/src',
chroot_squash,
'-comp',
compression_algorithm,
'-no-progress',
'-noappend',
logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs(
'tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute(
'chroot', chroot, 'mksquashfs', '/mnt/src',
chroot_squash,
'-comp', compression_algorithm,
'-no-progress', '-noappend', logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def test_umount_fs_not_mounted(self, mock_exec):
mock_exec.side_effect = errors.ProcessExecutionError
fu.umount_fs('/fake')
mock_exec.assert_called_once_with(
'mountpoint', '-q', '/fake', check_exit_code=[0])
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) install operating system (install_base_os)
6) configure apt-get sources,and perform package install.
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
driver_os = self.driver.operating_system
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata['os'] = driver_os.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([
os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images
]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
chroot = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix)
try:
self.install_base_os(chroot)
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot,
driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot,
packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot,
signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(os.path.join(chroot, 'proc'))
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
if image.target_device.name:
LOG.debug('Finally: detaching loop device: {0}'.format(
image.target_device.name))
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning(
'Error occured while trying to detach '
'loop device {0}. Error message: {1}'.format(
image.target_device.name, e))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file,
image.container,
chunk_size=CONF.data_chunk_size)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(img_containerized,
container_size)
metadata.setdefault('images', []).append({
'raw_md5':
raw_md5,
'raw_size':
raw_size,
'raw_name':
None,
'container_name':
os.path.basename(img_containerized),
'container_md5':
container_md5,
'container_size':
container_size,
'container':
image.container,
'format':
image.format
})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'wt',
encoding='utf-8') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) install operating system (install_base_os)
6) configure apt-get sources,and perform package install.
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
driver_os = self.driver.operating_system
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata['os'] = driver_os.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
chroot = bu.mkdtemp_smart(
CONF.image_build_dir, CONF.image_build_suffix)
try:
self.install_base_os(chroot)
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot, driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(
metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
root = driver_os.get_user_by_name('root')
bu.do_post_inst(chroot,
hashed_root_password=root.hashed_password,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(os.path.join(chroot, 'proc'))
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
if image.target_device.name:
LOG.debug('Finally: detaching loop device: {0}'.format(
image.target_device.name))
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning('Error occured while trying to detach '
'loop device {0}. Error message: {1}'.
format(image.target_device.name, e))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container,
chunk_size=CONF.data_chunk_size)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(
img_containerized, container_size)
metadata.setdefault('images', []).append({
'raw_md5': raw_md5,
'raw_size': raw_size,
'raw_name': None,
'container_name': os.path.basename(img_containerized),
'container_md5': container_md5,
'container_size': container_size,
'container': image.container,
'format': image.format})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'wt', encoding='utf-8') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
