def test_quote_params(run_skopeo_mock, initialized_db, app):
"""
Basic test of successful mirror.
"""
mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
mirror.external_reference = "& rm -rf /;/namespace/repository"
mirror.external_registry_username = "******"
mirror.save()
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"--creds",
"`rm -rf /`",
"'docker://& rm -rf /;/namespace/repository:latest'",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--all",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"--src-creds",
"`rm -rf /`",
"'docker://& rm -rf /;/namespace/repository:latest'",
"docker://localhost:5000/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "stdout", "stderr"),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_mirror_config_server_hostname(run_skopeo_mock, initialized_db, app,
monkeypatch):
"""
Set REPO_MIRROR_SERVER_HOSTNAME to override SERVER_HOSTNAME config.
"""
mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"--debug",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"--debug",
"copy",
"--all",
"--remove-signatures",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:latest",
"docker://config_server_hostname/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "Success", ""),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
monkeypatch.setenv("DEBUGLOG", "true")
with patch.dict("data.model.config.app_config",
{"REPO_MIRROR_SERVER_HOSTNAME": "config_server_hostname"}):
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_mirror_unsigned_images(run_skopeo_mock, initialized_db, app):
"""
Test whether the insecure-policy option is added when a repository is passed with unsigned_images.
"""
mirror, repo = create_mirror_repo_robot(["latest"],
external_registry_config={
"verify_tls": False,
"unsigned_images": True
})
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=False",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"--insecure-policy",
"copy",
"--all",
"--remove-signatures",
"--src-tls-verify=False",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:latest",
"docker://localhost:5000/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "stdout", "stderr"),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_successful_mirror_verbose_logs(run_skopeo_mock, initialized_db, app,
monkeypatch):
"""
Basic test of successful mirror with verbose logs turned on.
"""
mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"--debug",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"--debug",
"copy",
"--all",
"--remove-signatures",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:latest",
"docker://localhost:5000/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "Success", ""),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
monkeypatch.setenv("DEBUGLOG", "true")
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_successful_disabled_sync_now(run_skopeo_mock, initialized_db, app):
"""
Disabled mirrors still allow "sync now".
"""
mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
mirror.is_enabled = False
mirror.sync_status = RepoMirrorStatus.SYNC_NOW
mirror.save()
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--all",
"--remove-signatures",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:latest",
"docker://localhost:5000/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "stdout", "stderr"),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_successful_mirror(run_skopeo_mock, initialized_db, app):
"""
Basic test of successful mirror.
"""
mirror, repo = create_mirror_repo_robot(
["latest", "7.1"], external_registry_config={"verify_tls": False})
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=False",
u"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', ""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--src-tls-verify=False",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
u"docker://registry.example.com/namespace/repository:latest",
u"docker://localhost:5000/mirror/repo:latest",
],
"results":
SkopeoResults(True, [], "stdout", "stderr"),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_mirror_config_server_hostname(run_skopeo_mock, initialized_db, app,
monkeypatch):
"""
Set REPO_MIRROR_SERVER_HOSTNAME to override SERVER_HOSTNAME config
"""
mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
skopeo_calls = [{
"args": [
"/usr/bin/skopeo", "--debug", "inspect", "--tls-verify=True",
u"docker://registry.example.com/namespace/repository:latest"
],
"results":
SkopeoResults(True, [], '{"RepoTags": ["latest"]}', '')
}, {
"args": [
"/usr/bin/skopeo", "--debug", "copy", "--src-tls-verify=True",
"--dest-tls-verify=True", "--dest-creds",
"%s:%s" %
(mirror.internal_robot.username, mirror.internal_robot.email),
u"docker://registry.example.com/namespace/repository:latest",
u"docker://config_server_hostname/mirror/repo:latest"
],
"results":
SkopeoResults(True, [], 'Success', '')
}]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call['args']
assert proxy == {}
return skopeo_call['results']
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
monkeypatch.setenv('DEBUGLOG', 'true')
with patch.dict('data.model.config.app_config',
{'REPO_MIRROR_SERVER_HOSTNAME': 'config_server_hostname'}):
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
def test_remove_obsolete_tags(initialized_db):
"""
As part of the mirror, the set of tags on the remote repository is compared to the local
existing tags.
Those not present on the remote are removed locally.
"""
mirror, repository = create_mirror_repo_robot(["updated", "created"], repo_name="removed")
_create_tag(repository, "oldtag")
incoming_tags = ["one", "two"]
deleted_tags = delete_obsolete_tags(mirror, incoming_tags)
assert [tag.name for tag in deleted_tags] == [tag.name]
def test_remove_obsolete_tags(initialized_db):
"""
As part of the mirror, the set of tags on the remote repository is compared to the local
existing tags.
Those not present on the remote are removed locally.
"""
mirror, repository = create_mirror_repo_robot(["updated", "created"], repo_name="removed")
manifest = Manifest.get()
image = find_create_or_link_image("removed", repository, None, {}, "local_us")
tag = create_or_update_tag_for_repo(
repository, "oldtag", image.docker_image_id, oci_manifest=manifest, reversion=True
)
incoming_tags = ["one", "two"]
deleted_tags = delete_obsolete_tags(mirror, incoming_tags)
assert [tag.name for tag in deleted_tags] == [tag.name]
def test_inspect_error_mirror(run_skopeo_mock, initialized_db, app):
"""
Test for no tag for skopeo inspect.
The mirror is processed four times, asserting that the remaining syncs decrement until next sync
is bumped to the future, confirming the fourth is never processed.
"""
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
mirror, repo = create_mirror_repo_robot(["7.1"])
# Call number 1
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:7.1",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
]
worker._process_mirrors()
mirror = RepoMirrorConfig.get_by_id(mirror.id)
assert [] == skopeo_calls
assert 2 == mirror.sync_retries_remaining
# Call number 2
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:7.1",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
]
worker._process_mirrors()
mirror = RepoMirrorConfig.get_by_id(mirror.id)
assert [] == skopeo_calls
assert 1 == mirror.sync_retries_remaining
# Call number 3
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:7.1",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
]
worker._process_mirrors()
mirror = RepoMirrorConfig.get_by_id(mirror.id)
assert [] == skopeo_calls
assert 3 == mirror.sync_retries_remaining
# Call number 4
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:7.1",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:latest",
],
"results":
SkopeoResults(
False,
[],
"",
'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"',
),
},
]
worker._process_mirrors()
mirror = RepoMirrorConfig.get_by_id(mirror.id)
assert 2 == len(skopeo_calls)
assert 3 == mirror.sync_retries_remaining
def test_rollback(run_skopeo_mock, initialized_db, app):
"""
Tags in the repo:
"updated" - this tag will be updated during the mirror
"removed" - this tag will be removed during the mirror
"created" - this tag will be created during the mirror
"""
mirror, repo = create_mirror_repo_robot(["updated", "created", "zzerror"])
_create_tag(repo, "updated")
_create_tag(repo, "deleted")
skopeo_calls = [
{
"args": [
"/usr/bin/skopeo",
"inspect",
"--tls-verify=True",
"docker://registry.example.com/namespace/repository:updated",
],
"results":
SkopeoResults(
True, [],
'{"RepoTags": ["latest", "updated", "created", "zzerror"]}',
""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--all",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:created",
"docker://localhost:5000/mirror/repo:created",
],
"results":
SkopeoResults(True, [], "Success", ""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--all",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:updated",
"docker://localhost:5000/mirror/repo:updated",
],
"results":
SkopeoResults(True, [], "Success", ""),
},
{
"args": [
"/usr/bin/skopeo",
"copy",
"--all",
"--src-tls-verify=True",
"--dest-tls-verify=True",
"--dest-creds",
"%s:%s" % (mirror.internal_robot.username,
retrieve_robot_token(mirror.internal_robot)),
"docker://registry.example.com/namespace/repository:zzerror",
"docker://localhost:5000/mirror/repo:zzerror",
],
"results":
SkopeoResults(False, [], "", "ERROR"),
},
]
def skopeo_test(args, proxy):
try:
skopeo_call = skopeo_calls.pop(0)
assert args == skopeo_call["args"]
assert proxy == {}
if args[1] == "copy" and args[7].endswith(":updated"):
_create_tag(repo, "updated")
elif args[1] == "copy" and args[7].endswith(":created"):
_create_tag(repo, "created")
return skopeo_call["results"]
except Exception as e:
skopeo_calls.append(skopeo_call)
raise e
run_skopeo_mock.side_effect = skopeo_test
worker = RepoMirrorWorker()
worker._process_mirrors()
assert [] == skopeo_calls
