def parse_image_scheme(self):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_meta = self._image_meta
image_scheme = objects.ImageScheme()
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all images in provision data')
for mount_point, image_data in six.iteritems(
data['ks_meta']['image_data']):
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(mount_point, image_data['uri'],
image_data['format'], image_data['container']))
iname = os.path.basename(urlparse(image_data['uri']).path)
imeta = next(itertools.chain(
(img for img in image_meta.get('images', [])
if img['container_name'] == iname), [{}]))
image_scheme.add_image(
uri=image_data['uri'],
target_device=self.partition_scheme.fs_by_mount(
mount_point).device,
format=image_data['format'],
container=image_data['container'],
size=imeta.get('raw_size'),
md5=imeta.get('raw_md5'),
)
return image_scheme
def __init__(self, data):
super(NailgunBuildImage, self).__init__(data)
self._image_scheme = objects.ImageScheme()
self._partition_scheme = objects.PartitionScheme()
self.parse_schemes()
self._operating_system = self.parse_operating_system()
def image_scheme(self, partition_scheme):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all file systems in partition scheme')
for fs in partition_scheme.fss:
LOG.debug('Processing fs %s' % fs.mount)
if fs.mount not in data['ks_meta']['image_data']:
LOG.debug('There is no image for fs %s. Skipping.' % fs.mount)
continue
image_data = data['ks_meta']['image_data'][fs.mount]
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(fs.mount, image_data['uri'], image_data['format'],
image_data['container']))
image_scheme.add_image(
uri=image_data['uri'],
target_device=fs.device,
# In the future we will get format and container
# from provision.json, but currently it is hard coded.
format=image_data['format'],
container=image_data['container'],
)
return image_scheme
def parse_schemes(self):
self.image_scheme = objects.ImageScheme()
self.partition_scheme = objects.PartitionScheme()
for mount, image in six.iteritems(self.data['image_data']):
filename = os.path.basename(urlsplit(image['uri']).path)
# Loop does not allocate any loop device
# during initialization.
device = objects.Loop()
self.image_scheme.add_image(
uri='file://' + os.path.join(self.data['output'], filename),
format=image['format'],
container=image['container'],
target_device=device)
self.partition_scheme.add_fs(
device=device,
mount=mount,
fs_type=image['format'])
if mount == '/':
metadata_filename = filename.split('.', 1)[0] + '.yaml'
self.metadata_uri = 'file://' + os.path.join(
self.data['output'], metadata_filename)
def image_scheme(self, partition_scheme):
data = self.data
image_scheme = objects.ImageScheme()
root_image_uri = 'http://%s/targetimages/%s.img.gz' % (
data['ks_meta']['master_ip'], data['profile'].split('_')[0])
image_scheme.add_image(
uri=root_image_uri,
target_device=partition_scheme.root_device(),
image_format='ext4',
container='gzip',
)
return image_scheme
def parse_image_scheme(self):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
# FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all images in provision data')
for mount_point, image_data in six.iteritems(
data['ks_meta']['image_data']):
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(mount_point, image_data['uri'],
image_data['format'], image_data['container']))
iname = os.path.basename(urlparse(image_data['uri']).path)
imeta = next(itertools.chain(
(img for img in image_meta.get('images', [])
if img['container_name'] == iname), [{}]))
image_scheme.add_image(
uri=image_data['uri'],
target_device=self.partition_scheme.fs_by_mount(
mount_point).device,
format=image_data['format'],
container=image_data['container'],
size=imeta.get('raw_size'),
md5=imeta.get('raw_md5'),
)
return image_scheme
def image_scheme(self, partition_scheme):
data = self.data
image_scheme = objects.ImageScheme()
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
for fs in partition_scheme.fss:
if fs.mount not in data['ks_meta']['image_data']:
continue
image_data = data['ks_meta']['image_data'][fs.mount]
image_scheme.add_image(
uri=image_data['uri'],
target_device=fs.device,
# In the future we will get image_format and container format
# from provision.json, but currently it is hard coded.
image_format=image_data['format'],
container=image_data['container'],
)
return image_scheme
def test_do_build_image(self, mock_umount_target, mock_mount_target,
mock_yaml_dump, mock_mkdtemp, mock_open,
mock_shutil_move, mock_os, mock_utils, mock_fu,
mock_bu):
loops = [objects.Loop(), objects.Loop()]
self.mgr.driver.image_scheme = objects.ImageScheme([
objects.Image('file:///fake/img.img.gz', loops[0], 'ext4', 'gzip'),
objects.Image('file:///fake/img-boot.img.gz', loops[1], 'ext2',
'gzip')
])
self.mgr.driver.partition_scheme = objects.PartitionScheme()
self.mgr.driver.partition_scheme.add_fs(device=loops[0],
mount='/',
fs_type='ext4')
self.mgr.driver.partition_scheme.add_fs(device=loops[1],
mount='/boot',
fs_type='ext2')
self.mgr.driver.metadata_uri = 'file:///fake/img.yaml'
self.mgr.driver.operating_system = objects.Ubuntu(
repos=[
objects.DEBRepo('ubuntu',
'http://fakeubuntu',
'trusty',
'fakesection',
priority=900),
objects.DEBRepo('ubuntu_zero',
'http://fakeubuntu_zero',
'trusty',
'fakesection',
priority=None),
objects.DEBRepo('mos',
'http://fakemos',
'mosX.Y',
'fakesection',
priority=1000)
],
packages=['fakepackage1', 'fakepackage2'])
mock_os.path.exists.return_value = False
mock_os.path.join.return_value = '/tmp/imgdir/proc'
mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz']
mock_bu.create_sparse_tmp_file.side_effect = \
['/tmp/img', '/tmp/img-boot']
mock_bu.get_free_loop_device.side_effect = ['/dev/loop0', '/dev/loop1']
mock_mkdtemp.return_value = '/tmp/imgdir'
getsize_side = [20, 2, 10, 1]
mock_os.path.getsize.side_effect = getsize_side
md5_side = [
'fakemd5_raw', 'fakemd5_gzip', 'fakemd5_raw_boot',
'fakemd5_gzip_boot'
]
mock_utils.calculate_md5.side_effect = md5_side
mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz']
mock_bu.stop_chrooted_processes.side_effect = [
False, True, False, True
]
self.mgr.do_build_image()
self.assertEqual([
mock.call('/fake/img.img.gz'),
mock.call('/fake/img-boot.img.gz')
], mock_os.path.exists.call_args_list)
self.assertEqual([
mock.call(dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
] * 2, mock_bu.create_sparse_tmp_file.call_args_list)
self.assertEqual([mock.call()] * 2,
mock_bu.get_free_loop_device.call_args_list)
self.assertEqual([
mock.call('/tmp/img', '/dev/loop0'),
mock.call('/tmp/img-boot', '/dev/loop1')
], mock_bu.attach_file_to_loop.call_args_list)
self.assertEqual([
mock.call(
fs_type='ext4', fs_options='', fs_label='', dev='/dev/loop0'),
mock.call(
fs_type='ext2', fs_options='', fs_label='', dev='/dev/loop1')
], mock_fu.make_fs.call_args_list)
mock_mkdtemp.assert_called_once_with(dir=CONF.image_build_dir,
suffix=CONF.image_build_suffix)
mock_mount_target.assert_called_once_with('/tmp/imgdir',
treat_mtab=False,
pseudo=False)
self.assertEqual([mock.call('/tmp/imgdir')] * 2,
mock_bu.suppress_services_start.call_args_list)
mock_bu.run_debootstrap.assert_called_once_with(
uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir')
mock_bu.set_apt_get_env.assert_called_once_with()
mock_bu.pre_apt_get.assert_called_once_with('/tmp/imgdir')
self.assertEqual([
mock.call(name='ubuntu',
uri='http://fakeubuntu',
suite='trusty',
section='fakesection',
chroot='/tmp/imgdir'),
mock.call(name='ubuntu_zero',
uri='http://fakeubuntu_zero',
suite='trusty',
section='fakesection',
chroot='/tmp/imgdir'),
mock.call(name='mos',
uri='http://fakemos',
suite='mosX.Y',
section='fakesection',
chroot='/tmp/imgdir')
], mock_bu.add_apt_source.call_args_list)
# we don't call add_apt_preference for ubuntu_zero
# because it has priority == None
self.assertEqual([
mock.call(name='ubuntu',
priority=900,
suite='trusty',
section='fakesection',
chroot='/tmp/imgdir',
uri='http://fakeubuntu'),
mock.call(name='mos',
priority=1000,
suite='mosX.Y',
section='fakesection',
chroot='/tmp/imgdir',
uri='http://fakemos')
], mock_bu.add_apt_preference.call_args_list)
mock_utils.makedirs_if_not_exists.assert_called_once_with(
'/tmp/imgdir/proc')
self.assertEqual([
mock.call('tune2fs', '-O', '^has_journal', '/dev/loop0'),
mock.call('tune2fs', '-O', 'has_journal', '/dev/loop0')
], mock_utils.execute.call_args_list)
mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc')
mock_bu.run_apt_get.assert_called_once_with(
'/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'])
mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir')
signal_calls = mock_bu.stop_chrooted_processes.call_args_list
self.assertEqual(
2 * [
mock.call('/tmp/imgdir', signal=signal.SIGTERM),
mock.call('/tmp/imgdir', signal=signal.SIGKILL)
], signal_calls)
self.assertEqual([mock.call('/tmp/imgdir/proc')] * 2,
mock_fu.umount_fs.call_args_list)
self.assertEqual(
[mock.call('/tmp/imgdir', try_lazy_umount=False, pseudo=False)] *
2, mock_umount_target.call_args_list)
self.assertEqual([mock.call('/dev/loop0'),
mock.call('/dev/loop1')] * 2,
mock_bu.deattach_loop.call_args_list)
self.assertEqual([mock.call('/tmp/img'),
mock.call('/tmp/img-boot')],
mock_bu.shrink_sparse_file.call_args_list)
self.assertEqual([
mock.call('/tmp/img'),
mock.call('/fake/img.img.gz'),
mock.call('/tmp/img-boot'),
mock.call('/fake/img-boot.img.gz')
], mock_os.path.getsize.call_args_list)
self.assertEqual([
mock.call('/tmp/img', 20),
mock.call('/fake/img.img.gz', 2),
mock.call('/tmp/img-boot', 10),
mock.call('/fake/img-boot.img.gz', 1)
], mock_utils.calculate_md5.call_args_list)
self.assertEqual([
mock.call('/tmp/img', 'gzip'),
mock.call('/tmp/img-boot', 'gzip')
], mock_bu.containerize.call_args_list)
mock_open.assert_called_once_with('/fake/img.yaml', 'w')
self.assertEqual([
mock.call('/tmp/img.gz', '/fake/img.img.gz'),
mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')
], mock_shutil_move.call_args_list)
metadata = {}
for repo in self.mgr.driver.operating_system.repos:
metadata.setdefault('repos', []).append({
'type': 'deb',
'name': repo.name,
'uri': repo.uri,
'suite': repo.suite,
'section': repo.section,
'priority': repo.priority,
'meta': repo.meta
})
metadata['packages'] = self.mgr.driver.operating_system.packages
metadata['images'] = [{
'raw_md5':
md5_side[0],
'raw_size':
getsize_side[0],
'raw_name':
None,
'container_name':
os.path.basename(self.mgr.driver.image_scheme.images[0].uri.split(
'file://', 1)[1]),
'container_md5':
md5_side[1],
'container_size':
getsize_side[1],
'container':
self.mgr.driver.image_scheme.images[0].container,
'format':
self.mgr.driver.image_scheme.images[0].format
}, {
'raw_md5':
md5_side[2],
'raw_size':
getsize_side[2],
'raw_name':
None,
'container_name':
os.path.basename(self.mgr.driver.image_scheme.images[1].uri.split(
'file://', 1)[1]),
'container_md5':
md5_side[3],
'container_size':
getsize_side[3],
'container':
self.mgr.driver.image_scheme.images[1].container,
'format':
self.mgr.driver.image_scheme.images[1].format
}]
mock_yaml_dump.assert_called_once_with(metadata, stream=mock_open())
import mock
import requests_mock
import unittest2
from fuel_agent.drivers import simple
from fuel_agent import objects
from fuel_agent.tests import base
@mock.patch.multiple(
simple.NailgunSimpleDriver,
parse_operating_system=lambda x: objects.OperatingSystem(None, None),
parse_image_meta=lambda x: {},
parse_grub=lambda x: objects.Grub(),
parse_configdrive_scheme=lambda x: objects.ConfigDriveScheme(),
parse_image_scheme=lambda x: objects.ImageScheme())
class TestObjectDeserialization(unittest2.TestCase):
def test_driver_always_has_correct_objects(self):
driver = simple.NailgunSimpleDriver({})
assert isinstance(driver.partition_scheme, objects.PartitionScheme)
def test_lv_data_is_loaded(self):
lv_data = {
'partitioning': {
'lvs': [
{
'name': 'lv-name',
'size': 12345,
'vgname': 'vg-name',
},
]