def test_deduplicate(self):
merger = Modulemd.ModuleIndexMerger()
idx1 = Modulemd.ModuleIndex()
idx2 = Modulemd.ModuleIndex()
res, failures = idx1.update_from_file(
path.join(self.source_root,
"modulemd/v2/tests/test_data/long-valid.yaml"), True)
self.assertTrue(res)
self.assertEqual(len(failures), 0)
res, failures = idx2.update_from_file(
path.join(self.source_root,
"modulemd/v2/tests/test_data/long-valid.yaml"), True)
self.assertTrue(res)
self.assertEqual(len(failures), 0)
# Save the original document for comparing later
baseline = idx1.dump_to_string()
# Add the same index twice
merger.associate_index(idx1, 0)
merger.associate_index(idx2, 0)
deduplicated_idx = merger.resolve()
final = deduplicated_idx.dump_to_string()
# Verify that it outputs the same content as the baseline
print("Baseline:\n%s" % baseline, file=sys.stderr)
print("Resolved:\n%s" % final, file=sys.stderr)
self.assertEqual(baseline, deduplicated_idx.dump_to_string())
def test_deduplicate(self):
merger = Modulemd.ModuleIndexMerger()
idx1 = Modulemd.ModuleIndex()
idx2 = Modulemd.ModuleIndex()
res, failures = idx1.update_from_file(
path.join(self.test_data_path, "long-valid.yaml"), True)
self.assertTrue(res)
self.assertEqual(len(failures), 0)
res, failures = idx2.update_from_file(
path.join(self.test_data_path, "long-valid.yaml"), True)
self.assertTrue(res)
self.assertEqual(len(failures), 0)
# Save the original document for comparing later
baseline = idx1.dump_to_string()
# Add the same index twice
merger.associate_index(idx1, 0)
merger.associate_index(idx2, 0)
deduplicated_idx = merger.resolve()
final = deduplicated_idx.dump_to_string()
# Verify that it outputs the same content as the baseline
self.assertEqual(baseline, final)
def test_merge_incompatible_streams(self):
# This test verifies that if we encounter two streams with the same
# NSVCA, but different content, we only retain one of them.
# Note: the specification of the merger states that the behavior is
# undefined, so we will only validate that the merge completes and
# it only contains a single stream
merger = Modulemd.ModuleIndexMerger.new()
base_idx = Modulemd.ModuleIndex()
self.assertTrue(
base_idx.update_from_file(
path.join(self.test_data_path, "merger", "conflict_base.yaml"),
True,
)
)
merger.associate_index(base_idx, 0)
bad_idx = Modulemd.ModuleIndex()
self.assertTrue(
bad_idx.update_from_file(
path.join(self.test_data_path, "merger", "conflict_base.yaml"),
True,
)
)
# Modify the stream to have a different summary
stream = bad_idx.search_streams_by_nsvca_glob()[0]
stream.set_summary("Invalid summary, should not merge successfully")
merger.associate_index(bad_idx, 0)
merged_idx = merger.resolve()
all_streams = merged_idx.search_streams_by_nsvca_glob()
self.assertEqual(len(all_streams), 1)
self.assertEqual(all_streams[0].props.module_name, "nodejs")
def test_merger_with_real_world_data(self):
fedora_index = Modulemd.ModuleIndex()
fedora_index.update_from_file(
path.join(self.test_data_path, "f29.yaml"), True)
updates_index = Modulemd.ModuleIndex()
updates_index.update_from_file(
path.join(self.test_data_path, "f29-updates.yaml"), True)
merger = Modulemd.ModuleIndexMerger()
merger.associate_index(fedora_index, 0)
merger.associate_index(updates_index, 0)
merged_index = merger.resolve()
self.assertIsNotNone(merged_index)
def test_xmd(self):
if "_overrides_module" in dir(Modulemd):
# The XMD python tests can only be run against the installed lib
# because the overrides that translate between python and GVariant
# must be installed in /usr/lib/python*/site-packages/gi/overrides
# or they are not included when importing Modulemd
stream = Modulemd.ModuleStreamV2.new("foo", "bar")
# An empty dictionary should be returned if no xmd value is set
assert stream.get_xmd() == {}
xmd = {"outer_key": {"inner_key": ["scalar", "another_scalar"]}}
stream.set_xmd(xmd)
xmd_copy = stream.get_xmd()
assert xmd_copy
assert "outer_key" in xmd_copy
assert "inner_key" in xmd_copy["outer_key"]
assert "scalar" in xmd_copy["outer_key"]["inner_key"]
assert "another_scalar" in xmd_copy["outer_key"]["inner_key"]
# Verify that we can add content and save it back
xmd["something"] = ["foo", "bar"]
stream.set_xmd(xmd)
stream.set_summary("foo")
stream.set_description("bar")
stream.add_module_license("MIT")
# Verify that we can output the XMD successfully
index = Modulemd.ModuleIndex()
index.add_module_stream(stream)
out_yaml = index.dump_to_string()
self.assertIsNotNone(out_yaml)
def test_strict_default_streams(self):
merger = Modulemd.ModuleIndexMerger.new()
for stream in ("27", "38"):
default = """
---
document: modulemd-defaults
version: 1
data:
module: python
stream: %s
...
""" % (
stream
)
index = Modulemd.ModuleIndex()
index.update_from_string(default, strict=True)
merger.associate_index(index, 0)
with self.assertRaisesRegexp(
gi.repository.GLib.GError,
"Default stream mismatch in module python",
):
merger.resolve_ext(True)
def test_merge_add_conflicting_stream_and_profile_modified(self):
base_idx = Modulemd.ModuleIndex()
self.assertTrue(
base_idx.update_from_file(
path.join(self.test_data_path, "merger", "base.yaml"), True
)
)
add_conflicting_idx = Modulemd.ModuleIndex()
self.assertTrue(
add_conflicting_idx.update_from_file(
path.join(
self.test_data_path,
"merger",
"add_conflicting_stream_and_profile_modified.yaml",
),
True,
)
)
merger = Modulemd.ModuleIndexMerger()
merger.associate_index(base_idx, 0)
merger.associate_index(add_conflicting_idx, 0)
merged_idx = merger.resolve()
self.assertIsNotNone(merged_idx)
psql = merged_idx.get_module("postgresql")
self.assertIsNotNone(psql)
psql_defs = psql.get_defaults()
self.assertIsNotNone(psql_defs)
self.assertEqual(psql_defs.get_default_stream(), "8.2")
expected_profile_defs = {
"8.1": set(["client", "server"]),
"8.2": set(["client", "server", "foo"]),
"8.3": set(["client", "server"]),
}
for stream in expected_profile_defs:
self.assertEqual(
set(psql_defs.get_default_profiles_for_stream(stream)),
expected_profile_defs[stream],
)
def test_merge_add_only(self):
base_idx = Modulemd.ModuleIndex()
self.assertTrue(
base_idx.update_from_file(
path.join(self.test_data_path, "merger", "base.yaml"), True))
add_only_idx = Modulemd.ModuleIndex()
self.assertTrue(
add_only_idx.update_from_file(
path.join(self.test_data_path, "merger", "add_only.yaml"),
True))
merger = Modulemd.ModuleIndexMerger()
merger.associate_index(base_idx, 0)
merger.associate_index(add_only_idx, 0)
merged_idx = merger.resolve()
self.assertIsNotNone(merged_idx)
httpd = merged_idx.get_module('httpd')
self.assertIsNotNone(httpd)
httpd_defs = httpd.get_defaults()
self.assertEqual(httpd_defs.get_default_stream(), "2.8")
expected_profile_defs = {
'2.2': set(['client', 'server']),
'2.8': set([
'notreal',
]),
'2.10': set([
'notreal',
])
}
for stream in expected_profile_defs.keys():
self.assertEqual(
set(httpd_defs.get_default_profiles_for_stream(stream)),
expected_profile_defs[stream])
self.assertEqual(httpd_defs.get_default_stream("workstation"), "2.4")
def index_modulemd_files(repo_path):
merger = Modulemd.ModuleIndexMerger()
for fn in sorted(os.listdir(repo_path)):
if not fn.endswith(".yaml"):
continue
yaml_path = os.path.join(repo_path, fn)
mmd = Modulemd.ModuleIndex()
mmd.update_from_file(yaml_path, strict=True)
merger.associate_index(mmd, 0)
return merger.resolve()
def test_merge_add_conflicting_stream(self):
base_idx = Modulemd.ModuleIndex()
self.assertTrue(
base_idx.update_from_file(
path.join(self.test_data_path, "merger", "base.yaml"), True))
add_only_idx = Modulemd.ModuleIndex()
self.assertTrue(
add_only_idx.update_from_file(
path.join(self.test_data_path, "merger",
"add_conflicting_stream.yaml"), True))
merger = Modulemd.ModuleIndexMerger()
merger.associate_index(base_idx, 0)
merger.associate_index(add_only_idx, 0)
merged_idx = merger.resolve()
self.assertIsNotNone(merged_idx)
psql = merged_idx.get_module('postgresql')
self.assertIsNotNone(psql)
psql_defs = psql.get_defaults()
self.assertIsNotNone(psql_defs)
self.assertIsNone(psql_defs.get_default_stream())
expected_profile_defs = {
'8.1': set(['client', 'server', 'foo']),
'8.2': set(['client', 'server', 'foo']),
}
for stream in expected_profile_defs.keys():
self.assertEqual(
set(psql_defs.get_default_profiles_for_stream(stream)),
expected_profile_defs[stream])
def test_xmd_issue_290(self):
if '_overrides_module' in dir(Modulemd):
stream = Modulemd.ModuleStream.read_file(
"%s/290.yaml" % (os.getenv('TEST_DATA_PATH')), True, '', '')
self.assertIsNotNone(stream)
xmd = stream.get_xmd()
xmd["something"] = ["foo", "bar"]
stream.set_xmd(xmd)
index = Modulemd.ModuleIndex()
index.add_module_stream(stream)
self.maxDiff = None
output_yaml = index.dump_to_string()
self.assertIsNotNone(output_yaml)
pass
def test_merger(self):
# Get a set of objects in a ModuleIndex
base_index = Modulemd.ModuleIndex()
base_index.update_from_file(
path.join(self.test_data_path, "merging-base.yaml"), True)
# Baseline
httpd_defaults = base_index.get_module("httpd").get_defaults()
self.assertIsNotNone(httpd_defaults)
self.assertEqual(httpd_defaults.get_default_stream(), "2.2")
httpd_profile_streams = (
httpd_defaults.get_streams_with_default_profiles())
self.assertEqual(len(httpd_profile_streams), 2)
self.assertTrue("2.2" in httpd_profile_streams)
self.assertTrue("2.8" in httpd_profile_streams)
self.assertEqual(
len(httpd_defaults.get_default_profiles_for_stream("2.2")), 2)
self.assertTrue(
"client" in httpd_defaults.get_default_profiles_for_stream("2.2"))
self.assertTrue(
"server" in httpd_defaults.get_default_profiles_for_stream("2.2"))
self.assertTrue(
"notreal" in httpd_defaults.get_default_profiles_for_stream("2.8"))
self.assertEqual(httpd_defaults.get_default_stream("workstation"),
"2.4")
httpd_profile_streams = httpd_defaults.get_streams_with_default_profiles(
"workstation")
self.assertEqual(len(httpd_profile_streams), 2)
self.assertTrue("2.4" in httpd_profile_streams)
self.assertTrue("2.6" in httpd_profile_streams)
self.assertEqual(
len(
httpd_defaults.get_default_profiles_for_stream(
"2.4", "workstation")),
1,
)
self.assertEqual(
len(
httpd_defaults.get_default_profiles_for_stream(
"2.6", "workstation")),
3,
)
# Get another set of objects that will override the default stream for
# nodejs
override_nodejs_index = Modulemd.ModuleIndex()
override_nodejs_index.update_from_file(
path.join(self.test_data_path, "overriding-nodejs.yaml"), True)
# Test that adding both of these at the same priority level results in
# the no default stream
merger = Modulemd.ModuleIndexMerger()
merger.associate_index(base_index, 0)
merger.associate_index(override_nodejs_index, 0)
merged_index = merger.resolve()
self.assertIsNotNone(merged_index)
nodejs = merged_index.get_module("nodejs")
self.assertIsNotNone(nodejs)
nodejs_defaults = nodejs.get_defaults()
self.assertIsNotNone(nodejs_defaults)
self.assertIsNone(nodejs_defaults.get_default_stream())
# Get another set of objects that will override the above
override_index = Modulemd.ModuleIndex()
override_index.update_from_file(
path.join(self.test_data_path, "overriding.yaml"), True)
# Test that override_index at a higher priority level succeeds
# Test that adding both of these at the same priority level fails
# with a merge conflict.
# Use randomly-selected high and low values to make sure we don't have
# sorting issues.
merger = Modulemd.ModuleIndexMerger()
random_low = random.randint(1, 100)
random_high = random.randint(101, 999)
print("Low priority: %d, High priority: %d" %
(random_low, random_high))
merger.associate_index(base_index, random_low)
merger.associate_index(override_index, random_high)
merged_index = merger.resolve()
self.assertIsNotNone(merged_index)
# Validate merged results
# HTTPD
httpd_defaults = merged_index.get_module("httpd").get_defaults()
self.assertIsNotNone(httpd_defaults)
self.assertEqual(httpd_defaults.get_default_stream(), "2.4")
httpd_profile_streams = (
httpd_defaults.get_streams_with_default_profiles())
self.assertEqual(len(httpd_profile_streams), 2)
self.assertTrue("2.2" in httpd_profile_streams)
self.assertTrue("2.4" in httpd_profile_streams)
self.assertEqual(
len(httpd_defaults.get_default_profiles_for_stream("2.2")), 2)
self.assertTrue(
"client" in httpd_defaults.get_default_profiles_for_stream("2.2"))
self.assertTrue(
"server" in httpd_defaults.get_default_profiles_for_stream("2.2"))
self.assertTrue(
"client" in httpd_defaults.get_default_profiles_for_stream("2.4"))
self.assertTrue(
"server" in httpd_defaults.get_default_profiles_for_stream("2.4"))
self.assertEqual(httpd_defaults.get_default_stream("workstation"),
"2.8")
httpd_profile_streams = httpd_defaults.get_streams_with_default_profiles(
"workstation")
self.assertEqual(len(httpd_profile_streams), 3)
self.assertTrue("2.4" in httpd_profile_streams)
self.assertTrue("2.6" in httpd_profile_streams)
self.assertTrue("2.8" in httpd_profile_streams)
self.assertEqual(
len(
httpd_defaults.get_default_profiles_for_stream(
"2.4", "workstation")),
1,
)
self.assertEqual(
len(
httpd_defaults.get_default_profiles_for_stream(
"2.6", "workstation")),
3,
)
self.assertEqual(
len(
httpd_defaults.get_default_profiles_for_stream(
"2.8", "workstation")),
4,
)
rpms_with_epoch = []
for i in rpms:
n, v, ra = i.rsplit("-", 2)
nevra = "%s-0:%s-%s" % (n, v, ra)
rpms_with_epoch.append(nevra)
rpms = rpms_with_epoch
module_stream = Modulemd.ModuleStreamV2.new(name, stream)
module_stream.set_version(int(version))
module_stream.add_module_license("LGPLv2")
module_stream.set_summary("Fake module")
module_stream.set_description(module_stream.get_summary())
for rpm in rpms:
module_stream.add_rpm_artifact(rpm[:-4])
for profile_name in profiles:
profile = Modulemd.Profile.new(profile_name)
profile.set_description("Description for profile %s." %
profile_name)
for profile_rpm in profiles[profile_name]["rpms"]:
profile.add_rpm(profile_rpm)
module_stream.add_profile(profile)
module_index = Modulemd.ModuleIndex()
module_index.add_module_stream(module_stream)
with open(os.path.join(module_dir, "%s.%s.yaml" % (module_id, arch)),
'w') as f:
f.write(module_index.dump_to_string())