def test_flatpak_create_oci(tmpdir, docker_tasker, config_name, breakage,
mock_flatpak):
if not mock_flatpak:
# Check that we actually have flatpak available
have_flatpak = False
try:
output = subprocess.check_output(['flatpak', '--version'],
universal_newlines=True)
m = re.search('(\d+)\.(\d+)\.(\d+)', output)
if m and (int(m.group(1)), int(m.group(2)), int(
m.group(3))) >= (0, 9, 7):
have_flatpak = True
except (subprocess.CalledProcessError, OSError):
pass
if not have_flatpak:
return
config = CONFIGS[config_name]
if mock_flatpak:
(flexmock(subprocess).should_receive("check_call").replace_with(
mocked_check_call))
(flexmock(subprocess).should_receive("check_output").replace_with(
mocked_check_output))
workflow = DockerBuildWorkflow({
"provider": "git",
"uri": "asd"
}, TEST_IMAGE)
setattr(workflow, 'builder', X)
setattr(workflow.builder, 'tasker', docker_tasker)
filesystem_dir = os.path.join(str(tmpdir), 'filesystem')
os.mkdir(filesystem_dir)
filesystem_contents = config['filesystem_contents']
for path, contents in filesystem_contents.items():
parts = path.split(':', 1)
path = parts[0]
mode = parts[1] if len(parts) == 2 else None
fullpath = os.path.join(filesystem_dir, path[1:])
parent_dir = os.path.dirname(fullpath)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
if contents is None:
os.mkdir(fullpath)
else:
with open(fullpath, 'w') as f:
f.write(contents)
if mode is not None:
os.chmod(fullpath, int(mode, 8))
if breakage == 'no_runtime':
# Copy the parts of the config we are going to change
config = dict(config)
config['modules'] = dict(config['modules'])
config['modules']['eog'] = dict(config['modules']['eog'])
module_config = config['modules']['eog']
mmd = Modulemd.Module.new_from_string(module_config['metadata'])
# Clear out all dependencies. Setting via the property causes a crash
# https://gitlab.gnome.org/GNOME/pygobject/issues/37
# mmd.props.dependencies = [Modulemd.Dependencies()]
mmd.set_dependencies([Modulemd.Dependencies()])
module_config['metadata'] = mmd.dumps()
expected_exception = 'Failed to identify runtime module'
else:
assert breakage is None
expected_exception = None
filesystem_tar = os.path.join(filesystem_dir, 'tar')
with open(filesystem_tar, "wb") as f:
with tarfile.TarFile(fileobj=f, mode='w') as tf:
for f in os.listdir(filesystem_dir):
tf.add(os.path.join(filesystem_dir, f), f)
export_stream = open(filesystem_tar, "rb")
def stream_to_generator(s):
while True:
# Yield small chunks to test the StreamAdapter code better
buf = s.read(100)
if len(buf) == 0:
return
yield buf
export_generator = stream_to_generator(export_stream)
(flexmock(
docker_tasker.d.wrapped).should_receive('create_container').with_args(
workflow.image,
command=["/bin/bash"]).and_return({'Id': CONTAINER_ID}))
(flexmock(docker_tasker.d.wrapped).should_receive('export').with_args(
CONTAINER_ID).and_return(export_generator))
(flexmock(docker_tasker.d.wrapped).should_receive(
'remove_container').with_args(CONTAINER_ID))
setup_flatpak_source_info(workflow, config)
runner = PrePublishPluginsRunner(docker_tasker, workflow,
[{
'name': FlatpakCreateOciPlugin.key,
'args': {}
}])
if expected_exception:
with pytest.raises(PluginFailedException) as ex:
runner.run()
assert expected_exception in str(ex)
else:
runner.run()
dir_metadata = workflow.exported_image_sequence[-2]
assert dir_metadata['type'] == IMAGE_TYPE_OCI
tar_metadata = workflow.exported_image_sequence[-1]
assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR
# Check that the expected files ended up in the flatpak
if mock_flatpak:
inspector = MockInspector(tmpdir, dir_metadata)
else:
inspector = DefaultInspector(tmpdir, dir_metadata)
files = inspector.list_files()
assert sorted(files) == config['expected_contents']
components = {c['name'] for c in workflow.image_components}
for n in config['expected_components']:
assert n in components
for n in config['unexpected_components']:
assert n not in components
metadata_lines = inspector.cat_file('/metadata').split('\n')
assert any(
re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', l)
for l in metadata_lines)
assert any(
re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', l)
for l in metadata_lines)
if config_name == 'app':
# Check that the desktop file was rewritten
output = inspector.cat_file(
'/export/share/applications/org.gnome.eog.desktop')
lines = output.split('\n')
assert 'Icon=org.gnome.eog' in lines
assert 'name=org.gnome.eog' in metadata_lines
assert 'tags=Viewer' in metadata_lines
assert 'command=eog2' in metadata_lines
elif config_name == 'runtime': # runtime
# Check that permissions have been normalized
assert inspector.get_file_perms('/files/etc/shadow') == '-00644'
assert inspector.get_file_perms('/files/bin/mount') == '-00755'
assert inspector.get_file_perms('/files/share/foo') == 'd00755'
assert 'name=org.fedoraproject.Platform' in metadata_lines
else: # SDK
assert 'name=org.fedoraproject.Sdk' in metadata_lines
def test_flatpak_create_oci(tmpdir, docker_tasker, config_name,
flatpak_metadata, breakage):
# Check that we actually have flatpak available
have_flatpak = False
try:
output = subprocess.check_output(['flatpak', '--version'],
universal_newlines=True)
m = re.search(r'(\d+)\.(\d+)\.(\d+)', output)
if m and (int(m.group(1)), int(m.group(2)), int(
m.group(3))) >= (0, 9, 7):
have_flatpak = True
except (subprocess.CalledProcessError, OSError):
pytest.skip(msg='flatpak not available')
if not have_flatpak:
return
config = CONFIGS[config_name]
workflow = DockerBuildWorkflow(TEST_IMAGE,
source={
"provider": "git",
"uri": "asd"
})
setattr(workflow, 'builder', X)
X.df_path = write_docker_file(config, str(tmpdir))
setattr(workflow.builder, 'tasker', docker_tasker)
make_and_store_reactor_config_map(workflow, flatpak_metadata)
filesystem_dir = os.path.join(str(tmpdir), 'filesystem')
os.mkdir(filesystem_dir)
filesystem_contents = config['filesystem_contents']
for path, contents in filesystem_contents.items():
parts = path.split(':', 1)
path = parts[0]
mode = parts[1] if len(parts) == 2 else None
fullpath = os.path.join(filesystem_dir, path[1:])
parent_dir = os.path.dirname(fullpath)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
if contents is None:
os.mkdir(fullpath)
else:
with open(fullpath, 'wb') as f:
f.write(contents)
if mode is not None:
os.chmod(fullpath, int(mode, 8))
if breakage == 'no_runtime':
# Copy the parts of the config we are going to change
config = dict(config)
config['modules'] = dict(config['modules'])
config['modules']['eog'] = dict(config['modules']['eog'])
module_config = config['modules']['eog']
mmd = Modulemd.ModuleStream.read_string(module_config['metadata'],
strict=True)
mmd.clear_dependencies()
mmd.add_dependencies(Modulemd.Dependencies())
mmd_index = Modulemd.ModuleIndex.new()
mmd_index.add_module_stream(mmd)
module_config['metadata'] = mmd_index.dump_to_string()
expected_exception = 'Failed to identify runtime module'
else:
assert breakage is None
expected_exception = None
filesystem_tar = os.path.join(filesystem_dir, 'tar')
with open(filesystem_tar, "wb") as f:
with tarfile.TarFile(fileobj=f, mode='w') as tf:
for f in os.listdir(filesystem_dir):
tf.add(os.path.join(filesystem_dir, f), f)
export_stream = open(filesystem_tar, "rb")
def stream_to_generator(s):
while True:
# Yield small chunks to test the StreamAdapter code better
buf = s.read(100)
if len(buf) == 0:
return
yield buf
export_generator = stream_to_generator(export_stream)
(flexmock(
docker_tasker.tasker).should_receive('export_container').with_args(
CONTAINER_ID).and_return(export_generator))
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'create_container').with_args(workflow.image,
command=["/bin/bash"]).and_return(
{'Id': CONTAINER_ID}))
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'remove_container').with_args(CONTAINER_ID, force=False))
setup_flatpak_source_info(workflow, config)
runner = PrePublishPluginsRunner(docker_tasker, workflow,
[{
'name': FlatpakCreateOciPlugin.key,
'args': {}
}])
if expected_exception:
with pytest.raises(PluginFailedException) as ex:
runner.run()
assert expected_exception in str(ex.value)
else:
runner.run()
dir_metadata = workflow.exported_image_sequence[-2]
assert dir_metadata['type'] == IMAGE_TYPE_OCI
tar_metadata = workflow.exported_image_sequence[-1]
assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR
# Check that the correct labels and annotations were written
labels, annotations = load_labels_and_annotations(dir_metadata)
if config_name == 'app':
assert labels['name'] == 'eog'
assert labels['com.redhat.component'] == 'eog'
assert labels['version'] == 'f28'
assert labels['release'] == '20170629213428'
elif config_name == 'runtime': # runtime
assert labels['name'] == 'flatpak-runtime'
assert labels[
'com.redhat.component'] == 'flatpak-runtime-container'
assert labels['version'] == 'f28'
assert labels['release'] == '20170701152209'
else:
assert labels['name'] == 'flatpak-sdk'
assert labels['com.redhat.component'] == 'flatpak-sdk-container'
assert labels['version'] == 'f28'
assert labels['release'] == '20170701152209'
if flatpak_metadata == 'annotations':
assert annotations.get(
'org.flatpak.ref') == config['expected_ref_name']
assert 'org.flatpak.ref' not in labels
elif flatpak_metadata == 'labels':
assert 'org.flatpak.ref' not in annotations
assert labels.get('org.flatpak.ref') == config['expected_ref_name']
elif flatpak_metadata == 'both':
assert annotations.get(
'org.flatpak.ref') == config['expected_ref_name']
assert labels.get('org.flatpak.ref') == config['expected_ref_name']
# Check that the expected files ended up in the flatpak
# Flatpak versions before 1.6 require annotations to be present, since we don't
# require such a new Flatpak, skip remaining checks in the label-only case
if flatpak_metadata == 'labels':
return
inspector = DefaultInspector(tmpdir, dir_metadata)
files = inspector.list_files()
assert sorted(files) == config['expected_contents']
components = {c['name'] for c in workflow.image_components} # noqa:E501; pylint: disable=not-an-iterable
for n in config['expected_components']:
assert n in components
for n in config['unexpected_components']:
assert n not in components
metadata_lines = inspector.cat_file('/metadata').split('\n')
assert any(
re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', l)
for l in metadata_lines)
assert any(
re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', l)
for l in metadata_lines)
if config_name == 'app':
# Check that the desktop file was rewritten
output = inspector.cat_file(
'/export/share/applications/org.gnome.eog.desktop')
lines = output.split('\n')
assert 'Icon=org.gnome.eog' in lines
assert 'name=org.gnome.eog' in metadata_lines
assert 'tags=Viewer' in metadata_lines
assert 'command=eog2' in metadata_lines
elif config_name == 'runtime': # runtime
# Check that permissions have been normalized
assert inspector.get_file_perms('/files/etc/shadow') == '-00644'
assert inspector.get_file_perms('/files/bin/mount') == '-00755'
assert inspector.get_file_perms('/files/share/foo') == 'd00755'
assert 'name=org.fedoraproject.Platform' in metadata_lines
else: # SDK
assert 'name=org.fedoraproject.Sdk' in metadata_lines
def test_flatpak_create_oci(tmpdir, docker_tasker, config_name, breakage, mock_flatpak):
if not mock_flatpak:
# Check that we actually have flatpak available
have_flatpak = False
try:
output = subprocess.check_output(['flatpak', '--version'],
universal_newlines=True)
m = re.search('(\d+)\.(\d+)\.(\d+)', output)
if m and (int(m.group(1)), int(m.group(2)), int(m.group(3))) >= (0, 9, 7):
have_flatpak = True
except (subprocess.CalledProcessError, OSError):
pytest.skip(msg='flatpak not available')
if not have_flatpak:
return
config = CONFIGS[config_name]
if mock_flatpak:
(flexmock(subprocess)
.should_receive("check_call")
.replace_with(mocked_check_call))
(flexmock(subprocess)
.should_receive("check_output")
.replace_with(mocked_check_output))
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE)
setattr(workflow, 'builder', X)
setattr(workflow.builder, 'tasker', docker_tasker)
filesystem_dir = os.path.join(str(tmpdir), 'filesystem')
os.mkdir(filesystem_dir)
filesystem_contents = config['filesystem_contents']
for path, contents in filesystem_contents.items():
parts = path.split(':', 1)
path = parts[0]
mode = parts[1] if len(parts) == 2 else None
fullpath = os.path.join(filesystem_dir, path[1:])
parent_dir = os.path.dirname(fullpath)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
if contents is None:
os.mkdir(fullpath)
else:
with open(fullpath, 'wb') as f:
f.write(contents)
if mode is not None:
os.chmod(fullpath, int(mode, 8))
if breakage == 'no_runtime':
# Copy the parts of the config we are going to change
config = dict(config)
config['modules'] = dict(config['modules'])
config['modules']['eog'] = dict(config['modules']['eog'])
module_config = config['modules']['eog']
mmd = Modulemd.Module.new_from_string(module_config['metadata'])
# Clear out all dependencies. Setting via the property causes a crash
# https://gitlab.gnome.org/GNOME/pygobject/issues/37
# mmd.props.dependencies = [Modulemd.Dependencies()]
mmd.set_dependencies([Modulemd.Dependencies()])
module_config['metadata'] = mmd.dumps()
expected_exception = 'Failed to identify runtime module'
else:
assert breakage is None
expected_exception = None
filesystem_tar = os.path.join(filesystem_dir, 'tar')
with open(filesystem_tar, "wb") as f:
with tarfile.TarFile(fileobj=f, mode='w') as tf:
for f in os.listdir(filesystem_dir):
tf.add(os.path.join(filesystem_dir, f), f)
export_stream = open(filesystem_tar, "rb")
def stream_to_generator(s):
while True:
# Yield small chunks to test the StreamAdapter code better
buf = s.read(100)
if len(buf) == 0:
return
yield buf
export_generator = stream_to_generator(export_stream)
(flexmock(docker_tasker.d)
.should_receive('export')
.with_args(CONTAINER_ID)
.and_return(export_generator))
(flexmock(docker_tasker.d.wrapped)
.should_receive('create_container')
.with_args(workflow.image, command=["/bin/bash"])
.and_return({'Id': CONTAINER_ID}))
(flexmock(docker_tasker.d.wrapped)
.should_receive('remove_container')
.with_args(CONTAINER_ID))
setup_flatpak_source_info(workflow, config)
runner = PrePublishPluginsRunner(
docker_tasker,
workflow,
[{
'name': FlatpakCreateOciPlugin.key,
'args': {}
}]
)
if expected_exception:
with pytest.raises(PluginFailedException) as ex:
runner.run()
assert expected_exception in str(ex)
else:
runner.run()
dir_metadata = workflow.exported_image_sequence[-2]
assert dir_metadata['type'] == IMAGE_TYPE_OCI
tar_metadata = workflow.exported_image_sequence[-1]
assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR
# Check that the expected files ended up in the flatpak
if mock_flatpak:
inspector = MockInspector(tmpdir, dir_metadata)
else:
inspector = DefaultInspector(tmpdir, dir_metadata)
files = inspector.list_files()
assert sorted(files) == config['expected_contents']
components = {c['name'] for c in workflow.image_components}
for n in config['expected_components']:
assert n in components
for n in config['unexpected_components']:
assert n not in components
metadata_lines = inspector.cat_file('/metadata').split('\n')
assert any(re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', l)
for l in metadata_lines)
assert any(re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', l)
for l in metadata_lines)
if config_name == 'app':
# Check that the desktop file was rewritten
output = inspector.cat_file('/export/share/applications/org.gnome.eog.desktop')
lines = output.split('\n')
assert 'Icon=org.gnome.eog' in lines
assert 'name=org.gnome.eog' in metadata_lines
assert 'tags=Viewer' in metadata_lines
assert 'command=eog2' in metadata_lines
elif config_name == 'runtime': # runtime
# Check that permissions have been normalized
assert inspector.get_file_perms('/files/etc/shadow') == '-00644'
assert inspector.get_file_perms('/files/bin/mount') == '-00755'
assert inspector.get_file_perms('/files/share/foo') == 'd00755'
assert 'name=org.fedoraproject.Platform' in metadata_lines
else: # SDK
assert 'name=org.fedoraproject.Sdk' in metadata_lines
def test_flatpak_create_oci(workflow, config_name, flatpak_metadata, breakage):
# Check that we actually have flatpak available
have_flatpak = False
try:
output = subprocess.check_output(['flatpak', '--version'],
universal_newlines=True)
m = re.search(r'(\d+)\.(\d+)\.(\d+)', output)
if m and (int(m.group(1)), int(m.group(2)), int(
m.group(3))) >= (0, 9, 7):
have_flatpak = True
except (subprocess.CalledProcessError, OSError):
pytest.skip(msg='flatpak not available')
if not have_flatpak:
return
# Check if we have skopeo
try:
subprocess.check_output(['skopeo', '--version'])
except (subprocess.CalledProcessError, OSError):
pytest.skip(msg='skopeo not available')
config = CONFIGS[config_name]
platforms = ['x86_64', 'aarch64', 's390x', 'ppc64le']
workflow.user_params['flatpak'] = True
write_docker_file(config, workflow.source.path)
workflow.build_dir.init_build_dirs(platforms, workflow.source)
mock_extract_filesystem_call = functools.partial(mock_extract_filesystem,
config)
(flexmock(ImageUtil).should_receive(
'extract_filesystem_layer').replace_with(mock_extract_filesystem_call))
make_and_store_reactor_config_map(workflow, flatpak_metadata)
if breakage == 'no_runtime':
# Copy the parts of the config we are going to change
config = dict(config)
config['modules'] = dict(config['modules'])
config['modules']['eog'] = dict(config['modules']['eog'])
module_config = config['modules']['eog']
mmd = Modulemd.ModuleStream.read_string(module_config['metadata'],
strict=True)
mmd.clear_dependencies()
mmd.add_dependencies(Modulemd.Dependencies())
mmd_index = Modulemd.ModuleIndex.new()
mmd_index.add_module_stream(mmd)
module_config['metadata'] = mmd_index.dump_to_string()
expected_exception = 'Failed to identify runtime module'
else:
assert breakage is None
expected_exception = None
runner = PostBuildPluginsRunner(workflow, [{
'name': FlatpakCreateOciPlugin.key,
'args': {}
}])
setup_flatpak_composes(workflow)
source = setup_flatpak_source_info(config)
(flexmock(FlatpakUtil).should_receive(
'get_flatpak_source_info').and_return(source))
if expected_exception:
with pytest.raises(PluginFailedException) as ex:
runner.run()
assert expected_exception in str(ex.value)
else:
builder = FlatpakBuilder(source,
workflow.build_dir.any_platform.path,
'var/tmp/flatpak-build',
parse_manifest=parse_rpm_output,
flatpak_metadata=FLATPAK_METADATA_ANNOTATIONS)
with NamedTemporaryFile(dir=workflow.build_dir.any_platform.path) as f:
f.write(
config['filesystem_contents']['/var/tmp/flatpak-build.rpm_qf'])
f.flush()
expected_components = builder.get_components(f.name)
results = runner.run()
x86_64_results = results[FlatpakCreateOciPlugin.key][platforms[0]]
dir_metadata = x86_64_results['metadata']
components = x86_64_results['components']
assert components == expected_components
assert dir_metadata['type'] == IMAGE_TYPE_OCI
# Check that the correct labels and annotations were written
labels, annotations = load_labels_and_annotations(dir_metadata)
if config_name == 'app':
assert labels['name'] == 'eog'
assert labels['com.redhat.component'] == 'eog'
assert labels['version'] == 'f28'
assert labels['release'] == '20170629213428'
elif config_name == 'runtime': # runtime
assert labels['name'] == 'flatpak-runtime'
assert labels[
'com.redhat.component'] == 'flatpak-runtime-container'
assert labels['version'] == 'f28'
assert labels['release'] == '20170701152209'
else:
assert labels['name'] == 'flatpak-sdk'
assert labels['com.redhat.component'] == 'flatpak-sdk-container'
assert labels['version'] == 'f28'
assert labels['release'] == '20170701152209'
if flatpak_metadata == 'annotations':
assert annotations.get(
'org.flatpak.ref') == config['expected_ref_name']
assert 'org.flatpak.ref' not in labels
elif flatpak_metadata == 'labels':
assert 'org.flatpak.ref' not in annotations
assert labels.get('org.flatpak.ref') == config['expected_ref_name']
elif flatpak_metadata == 'both':
assert annotations.get(
'org.flatpak.ref') == config['expected_ref_name']
assert labels.get('org.flatpak.ref') == config['expected_ref_name']
# Check that the expected files ended up in the flatpak
# Flatpak versions before 1.6 require annotations to be present, and Flatpak
# versions 1.6 and later require labels to be present. Skip the remaining
# checks unless we have both annotations and labels.
if flatpak_metadata != 'both':
return
inspector = DefaultInspector(str(workflow.build_dir.any_platform.path),
dir_metadata)
files = inspector.list_files()
assert sorted(files) == config['expected_contents']
metadata_lines = inspector.cat_file('/metadata').split('\n')
assert any(
re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', line)
for line in metadata_lines)
assert any(
re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', line)
for line in metadata_lines)
if config_name == 'app':
# Check that the desktop file was rewritten
output = inspector.cat_file(
'/export/share/applications/org.gnome.eog.desktop')
lines = output.split('\n')
assert 'Icon=org.gnome.eog' in lines
assert 'name=org.gnome.eog' in metadata_lines
assert 'tags=Viewer' in metadata_lines
assert 'command=eog2' in metadata_lines
elif config_name == 'runtime': # runtime
# Check that permissions have been normalized
assert inspector.get_file_perms('/files/etc/shadow') == '-00644'
assert inspector.get_file_perms('/files/bin/mount') == '-00755'
assert inspector.get_file_perms('/files/share/foo') == 'd00755'
assert 'name=org.fedoraproject.Platform' in metadata_lines
else: # SDK
assert 'name=org.fedoraproject.Sdk' in metadata_lines