def test_detect_package_manangers(self, is_file: Mock, is_dir: Mock):
runtime = MagicMock(uuid="123456")
runtime.group_config.cachito.enabled = True
meta = ImageMetadata(
runtime,
model.Model({
"key": "foo",
'data': {
'name': 'openshift/foo',
'distgit': {
'branch': 'fake-branch-rhel-8'
},
"additional_tags": ["tag_a", "tag_b"]
},
}))
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.logger = logging.getLogger()
dg.distgit_dir = "/path/to/distgit/containers/foo"
dg.dg_path = Path(dg.distgit_dir)
is_dir.return_value = True
is_file.side_effect = lambda f: f.name in [
"go.mod", "package-lock.json"
]
actual = dg._detect_package_manangers()
self.assertEqual(set(actual), {"gomod", "npm"})
def test_push_with_io_error(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
# preventing tests from executing real commands
flexmock(
distgit.exectools).should_receive("cmd_gather").and_return(None)
# pretending cmd_assert raised an IOError
flexmock(distgit.exectools).should_receive("cmd_assert").and_raise(
IOError("io-error"))
metadata = flexmock(
runtime=self.mock_runtime(
global_opts={"rhpkg_push_timeout": "_irrelevant_"},
get_named_semaphore=lambda *_, **__: Lock()),
config=flexmock(distgit=flexmock(branch="_irrelevant_")),
name="_irrelevant_",
logger=flexmock(info=lambda *_: None))
expected = (metadata, repr(IOError("io-error")))
actual = distgit.ImageDistGitRepo(metadata, autoclone=False).push()
self.assertEqual(expected, actual)
def test_generate_osbs_image_config_with_addtional_tags(self):
runtime = MagicMock(uuid="123456")
runtime.group_config.cachito.enabled = False
meta = ImageMetadata(
runtime,
model.Model({
"key": "foo",
'data': {
'name': 'openshift/foo',
'distgit': {
'branch': 'fake-branch-rhel-8'
},
"additional_tags": ["tag_a", "tag_b"]
},
}))
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.logger = logging.getLogger()
# assembly is not enabled
runtime.assembly = None
container_yaml = dg._generate_osbs_image_config("v4.10.0")
self.assertEqual(
sorted(container_yaml["tags"]),
sorted(['v4.10.0.123456', 'v4.10', 'v4.10.0', 'tag_a', 'tag_b']))
# assembly is enabled
runtime.assembly = "art3109"
container_yaml = dg._generate_osbs_image_config("v4.10.0")
self.assertEqual(
sorted(container_yaml["tags"]),
sorted([
'assembly.art3109', 'v4.10.0.123456', 'v4.10', 'v4.10.0',
'tag_a', 'tag_b'
]))
def test_add_missing_pkgs_succeed(self):
md = MockMetadata(MockRuntime(self.logger))
d = distgit.ImageDistGitRepo(md, autoclone=False)
d._add_missing_pkgs("haproxy")
self.assertEqual(1, len(d.runtime.missing_pkgs))
self.assertIn("distgit_key image is missing package haproxy", d.runtime.missing_pkgs)
def test_generate_osbs_image_config_with_cachito_enabled(
self, is_dir: Mock):
runtime = MagicMock(uuid="123456",
assembly="test",
assembly_basis_event=None,
profile=None,
odcs_mode=False)
runtime.group_config = model.Model()
meta = ImageMetadata(
runtime,
model.Model({
"key": "foo",
'data': {
'name': 'openshift/foo',
'distgit': {
'branch': 'fake-branch-rhel-8'
},
"additional_tags": ["tag_a", "tag_b"],
"content": {
"source": {
"git": {
"url":
"[email protected]:openshift-priv/foo.git",
"branch": {
"target": "release-4.10"
}
}
}
},
"cachito": {
"enabled": True,
"flags": ["gomod-vendor-check"]
}
},
}))
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.logger = logging.getLogger()
dg.distgit_dir = "/path/to/distgit/containers/foo"
dg.dg_path = Path(dg.distgit_dir)
dg.actual_source_url = "[email protected]:openshift-priv/foo.git"
dg.source_full_sha = "deadbeef"
dg._detect_package_manangers = MagicMock(return_value=["gomod"])
actual = dg._generate_osbs_image_config("v4.10.0")
self.assertEqual(actual["remote_sources"][0]["remote_source"]["repo"],
"https://example.com/openshift-priv/foo")
self.assertEqual(actual["remote_sources"][0]["remote_source"]["ref"],
"deadbeef")
self.assertEqual(
actual["remote_sources"][0]["remote_source"]["pkg_managers"],
["gomod"])
self.assertEqual(actual["remote_sources"][0]["remote_source"]["flags"],
["gomod-vendor-check"])
def test_image_build_method_from_config(self):
metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(
default_image_build_method="default-method")),
config=flexmock(
distgit=flexmock(branch=distgit.Missing),
image_build_method="config-method",
get=lambda _, d: d),
name="_irrelevant_",
logger="_irrelevant_")
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
self.assertEqual("config-method", repo.image_build_method)
def test_push_image_is_late_push(self):
metadata = flexmock(config=flexmock(
push=flexmock(late=True), distgit=flexmock(branch="_irrelevant_")),
distgit_key="distgit_key",
runtime=self.mock_runtime(),
name="_irrelevant_",
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
expected = ("distgit_key", True)
actual = repo.push_image([], "push_to_defaults")
self.assertEqual(expected, actual)
def test_image_build_method_imagebuilder(self):
get = lambda key, default: dict({"builder": "..."}
) if key == "from" else default
metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(
default_image_build_method="default-method")),
config=flexmock(
distgit=flexmock(branch=distgit.Missing),
image_build_method=distgit.Missing,
get=get),
name="_irrelevant_",
logger="_irrelevant_")
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
self.assertEqual("imagebuilder", repo.image_build_method)
def test_push_image_to_defaults_fail_mirroring(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(
distgit.os).should_receive("mkdir").replace_with(lambda _: None)
flexmock(distgit.os.path).should_receive("isdir").and_return(True)
flexmock(distgit.os.path).should_receive("isfile").and_return(True)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(
sys.modules["__builtin__"]).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
# simulating an error while executing cmd_gather
(flexmock(distgit.exectools).should_receive("cmd_gather").and_return(
(1, "", "stderr")))
# a failed cmd_gather will try again in X seconds, we don't want to wait
flexmock(
distgit.time).should_receive("sleep").replace_with(lambda *_: None)
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
namespace="my-namespace",
get_default_push_names=lambda *_: ["my-default-name"],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = ["tag-a", "tag-b"]
push_to_defaults = True
version_release_tuple = ("version", "release")
try:
repo.push_image(tag_list,
push_to_defaults,
version_release_tuple=version_release_tuple)
self.fail("Should have raised an exception, but didn't")
except IOError as e:
expected_msg = "Error pushing image: stderr"
self.assertEqual(expected_msg, e.message)
def test_push_image_to_defaults_with_lstate(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(
distgit.os).should_receive("mkdir").replace_with(lambda _: None)
flexmock(distgit.os.path).should_receive("isdir").and_return(True)
flexmock(distgit.os.path).should_receive("isfile").and_return(True)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(
sys.modules["__builtin__"]).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
# preventing tests from executing real commands
flexmock(distgit.exectools).should_receive("cmd_gather").and_return(
(0, "", ""))
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
get_default_push_names=lambda *_: ["my-default-name"],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None),
namespace="_irrelevant_")
metadata.runtime.state = {"images:push": "my-runtime-state"}
metadata.runtime.command = "images:push"
(flexmock(
distgit.state).should_receive("record_image_success").with_args(
"my-runtime-state", metadata).once().and_return(None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = ["tag-a", "tag-b"]
push_to_defaults = True
expected = ("my-distgit-key", True)
actual = repo.push_image(tag_list,
push_to_defaults,
version_release_tuple=("version", "release"))
self.assertEqual(expected, actual)
def test_push_image_nothing_to_push(self):
metadata = flexmock(config=flexmock(
push=flexmock(late=distgit.Missing),
distgit=flexmock(branch="_irrelevant_")),
distgit_key="distgit_key",
get_default_push_names=lambda *_: [],
get_additional_push_names=lambda *_: [],
runtime=self.mock_runtime(),
name="_irrelevant_",
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
push_to_defaults = False
expected = ("distgit_key", True)
actual = repo.push_image([], push_to_defaults)
self.assertEqual(expected, actual)
def test_push_image_registry_config(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(
distgit.os).should_receive("mkdir").replace_with(lambda _: None)
flexmock(distgit.os.path).should_receive("isdir").and_return(True)
flexmock(distgit.os.path).should_receive("isfile").and_return(True)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(io).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
flexmock(distgit.util).should_receive(
"get_docker_config_json").and_return('/auth/config.json')
expected_cmd = "oc image mirror --insecure=true --filename=some-workdir/push/my-distgit-key --registry-config=/auth/config.json"
(flexmock(distgit.exectools).should_receive("cmd_gather").with_args(
expected_cmd).once().and_return((0, "", "")))
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
get_default_push_names=lambda *_: ["my-default-name"],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None),
namespace="_irrelevant_")
metadata.runtime.working_dir = "some-workdir"
metadata.runtime.group_config.insecure_source = True
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = ["tag-a", "tag-b"]
push_to_defaults = True
expected = ("my-distgit-key", True)
actual = repo.push_image(tag_list,
push_to_defaults,
version_release_tuple=("version", "release"),
registry_config_dir='/auth')
self.assertEqual(expected, actual)
def test_push_image_push_config_dir_already_created_by_another_thread(
self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(distgit.os.path).should_receive("isdir").and_return(
False).and_return(True)
flexmock(distgit.os.path).should_receive("isfile").and_return(False)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(
sys.modules["__builtin__"]).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
# simulating an "already exists" error on mkdir attempt
(flexmock(distgit.os).should_receive("mkdir").and_raise(
OSError(errno.EEXIST, os.strerror(errno.EEXIST))))
# preventing tests from executing real commands
flexmock(distgit.exectools).should_receive("cmd_gather").and_return(
(0, "", ""))
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
namespace="my-namespace",
get_default_push_names=lambda *_: ["my-default-name"],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = ["tag-a", "tag-b"]
push_to_defaults = True
expected = ("my-distgit-key", True)
actual = repo.push_image(tag_list,
push_to_defaults,
version_release_tuple=("version", "release"))
self.assertEqual(expected, actual)
def test_inject_yum_update_commands_without_repos(self):
runtime = MagicMock()
meta = ImageMetadata(
runtime,
model.Model({
"key": "foo",
'data': {
'name': 'openshift/foo',
'distgit': {
'branch': 'fake-branch-rhel-8'
},
"enabled_repos": []
},
}))
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.logger = logging.getLogger()
dg.dg_path = MagicMock()
dockerfile = """
FROM some-base-image:some-tag AS builder
# __doozer=yum-update
USER 0
# __doozer=yum-update
RUN yum update -y && yum clean all # set final_stage_user in ART metadata if this fails
LABEL name=value
RUN some-command
FROM another-base-image:some-tag
# __doozer=yum-update
USER 0
# __doozer=yum-update
RUN yum update -y && yum clean all # set final_stage_user in ART metadata if this fails
# __doozer=yum-update
USER 1001
COPY --from=builder /some/path/a /some/path/b
""".strip()
actual = dg._update_yum_update_commands(
True, io.StringIO(dockerfile)).getvalue().strip().splitlines()
expected = """
FROM some-base-image:some-tag AS builder
LABEL name=value
RUN some-command
FROM another-base-image:some-tag
COPY --from=builder /some/path/a /some/path/b
""".strip().splitlines()
self.assertListEqual(actual, expected)
def test_clone_invokes_read_master_data(self):
"""
Mocking `clone` method of parent class, since we are only interested
in validating that `_read_master_data` is called in the child class.
"""
(flexmock(distgit.DistGitRepo).should_receive("clone").replace_with(
lambda *_: None))
(flexmock(distgit.ImageDistGitRepo).should_receive(
"_read_master_data").once())
metadata = flexmock(
runtime=self.mock_runtime(),
config=flexmock(distgit=flexmock(branch=distgit.Missing)),
name="_irrelevant_",
logger="_irrelevant_")
(distgit.ImageDistGitRepo(metadata, autoclone=False).clone(
"distgits_root_dir", "distgit_branch"))
def test_wait_for_build_with_build_status_false(self):
metadata = flexmock(
qualified_name="my-qualified-name",
config=flexmock(distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
name="_irrelevant_",
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
repo.build_lock = flexmock() # we don't want tests to be blocked
repo.build_status = False
try:
repo.wait_for_build("i-am-waiting")
self.fail("Should have raised IOError")
except IOError as e:
expected = "Error building image: my-qualified-name (i-am-waiting was waiting)"
actual = str(e)
self.assertEqual(expected, actual)
def test_push_image_no_push_tags(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(
distgit.os).should_receive("mkdir").replace_with(lambda _: None)
flexmock(distgit.os.path).should_receive("isdir").and_return(True)
flexmock(distgit.os.path).should_receive("isfile").and_return(True)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(io).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
# preventing tests from executing real commands
flexmock(distgit.exectools).should_receive("cmd_gather").and_return(
(0, "", ""))
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
namespace="my-namespace",
get_latest_build_info=lambda *_: ("_", "my-version", "my-release"),
get_default_push_names=lambda *_: ["my/default-name"],
get_default_push_tags=lambda *_: [],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = []
push_to_defaults = True
expected = ("my-distgit-key", True)
actual = repo.push_image(tag_list,
push_to_defaults,
version_release_tuple=("version", "release"),
dry_run=True)
self.assertEqual(expected, actual)
def test_detect_package_manangers_without_git_clone(self):
runtime = MagicMock(uuid="123456")
runtime.group_config.cachito.enabled = True
meta = ImageMetadata(
runtime,
model.Model({
"key": "foo",
'data': {
'name': 'openshift/foo',
'distgit': {
'branch': 'fake-branch-rhel-8'
},
"additional_tags": ["tag_a", "tag_b"]
},
}))
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.logger = logging.getLogger()
with self.assertRaises(FileNotFoundError):
dg._detect_package_manangers()
def test_push_image_fail_to_create_a_push_config_dir(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
flexmock(distgit.os.path).should_receive("isdir").and_return(False)
flexmock(distgit.os.path).should_receive("isfile").and_return(True)
flexmock(
distgit.os).should_receive("remove").replace_with(lambda _: None)
(flexmock(io).should_receive("open").and_return(
flexmock(write=lambda *_: None, readlines=lambda *_: [])))
# simulating an error on mkdir attempt
flexmock(distgit.os).should_receive("mkdir").and_raise(OSError)
# preventing tests from executing real commands
flexmock(distgit.exectools).should_receive("cmd_gather").and_return(
(0, "", ""))
metadata = flexmock(
config=flexmock(push=flexmock(late=distgit.Missing),
name="my-name",
namespace="my-namespace",
distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
distgit_key="my-distgit-key",
name="my-name",
namespace="my-namespace",
get_default_push_names=lambda *_: ["my-default-name"],
get_additional_push_names=lambda *_: [],
logger=flexmock(info=lambda *_: None))
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
tag_list = ["tag-a", "tag-b"]
push_to_defaults = True
version_release_tuple = ("version", "release")
self.assertRaises(OSError,
repo.push_image,
tag_list,
push_to_defaults,
version_release_tuple=version_release_tuple)
def test_wait_for_build_with_build_status_true(self):
logger = flexmock()
(logger.should_receive("info").with_args(
"Member waiting for me to build: i-am-waiting").once().ordered())
(logger.should_receive("info").with_args(
"Member successfully waited for me to build: i-am-waiting").once().
ordered())
metadata = flexmock(
logger=logger,
config=flexmock(distgit=flexmock(branch="_irrelevant_")),
runtime=self.mock_runtime(),
name="_irrelevant_")
repo = distgit.ImageDistGitRepo(metadata, autoclone=False)
repo.build_lock = flexmock() # we don't want tests to be blocked
repo.build_status = True
repo.wait_for_build("i-am-waiting")
def test_cgit_file_available(self, mocked_head):
meta = MockMetadata(MockRuntime(self.logger))
cgit_url = "http://distgit.example.com/cgit/containers/foo/plain/some_path/some_file.txt?h=some-branch&id=abcdefg"
meta.cgit_file_url = lambda *args, **kwargs: cgit_url
dg = distgit.ImageDistGitRepo(meta, autoclone=False)
dg.sha = "abcdefg"
mocked_head.return_value.status_code = 404
existence, url = dg.cgit_file_available("some_path/some_file.txt")
self.assertEqual(url, cgit_url)
self.assertFalse(existence)
mocked_head.return_value.status_code = 200
existence, url = dg.cgit_file_available("some_path/some_file.txt")
self.assertEqual(url, cgit_url)
self.assertTrue(existence)
mocked_head.return_value.status_code = 500
mocked_head.return_value.raise_for_status.side_effect = IOError
with self.assertRaises(IOError):
dg.cgit_file_available("some_path/some_file.txt")
def test_push(self):
# preventing tests from interacting with the real filesystem
flexmock(distgit).should_receive("Dir").and_return(
flexmock(__exit__=None))
(flexmock(distgit.exectools).should_receive("cmd_assert").with_args(
"timeout 999 rhpkg push", retries=3).once().ordered())
(flexmock(distgit.exectools).should_receive("cmd_gather").with_args(
["timeout", "60", "git", "push", "--tags"]).once().ordered())
metadata = flexmock(
runtime=flexmock(global_opts={"rhpkg_push_timeout": 999},
branch="_irrelevant_"),
config=flexmock(distgit=flexmock(branch="_irrelevant_")),
name="_irrelevant_",
logger=flexmock(info=lambda *_: None))
expected = (metadata, True)
actual = distgit.ImageDistGitRepo(metadata, autoclone=False).push()
self.assertEqual(expected, actual)
def setUp(self):
super(TestImageDistGit, self).setUp()
self.img_dg = distgit.ImageDistGitRepo(self.md, autoclone=False)
self.img_dg.runtime.group_config = model.Model()