def test_search_distributor_with_relative_url():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
relative_url="relative/path",
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
relative_url="relative/path",
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
dist3 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo2",
relative_url="another/path",
)
repo2 = Repository(id="repo2", distributors=(dist3, ))
controller.insert_repository(repo1)
controller.insert_repository(repo2)
client = controller.client
crit = Criteria.with_field("relative_url", Matcher.regex("relative/path"))
found = client.search_distributor(crit).result().data
assert sorted(found) == [dist2, dist1]
def test_publish_absent_raises():
"""repo.publish() of a nonexistent repo raises."""
controller = FakeController()
controller.insert_repository(
YumRepository(
id="repo1",
distributors=[
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="cdn_distributor",
type_id="rpm_rsync_distributor"),
],
))
client = controller.client
repo_copy1 = client.get_repository("repo1")
repo_copy2 = client.get_repository("repo1")
# If I delete the repo through one handle...
assert repo_copy1.delete().result()
# ...then publish through the other handle becomes impossible
exception = repo_copy2.publish().exception()
assert isinstance(exception, PulpException)
assert "repo1 not found" in str(exception)
def test_search_mapped_field_less_than():
controller = FakeController()
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 23, 2, 5, 0, tzinfo=None),
)
dist2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
last_publish=datetime.datetime(2019, 8, 27, 2, 5, 0, tzinfo=None),
)
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.with_field(
"last_publish",
Matcher.less_than(datetime.datetime(2019, 8, 24, 0, 0, 0)))
found = client.search_distributor(crit).result().data
assert found == [dist1]
def test_publish_retries(fast_poller, requests_mocker, client, caplog):
"""publish retries distributors as they fail"""
caplog.set_level(logging.WARNING)
repo = YumRepository(
id="some-repo",
distributors=(
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="cdn_distributor", type_id="cdn_distributor"),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[
{"json": {"spawned_tasks": [{"task_id": "task1"}]}},
{"json": {"spawned_tasks": [{"task_id": "task2"}]}},
{"json": {"spawned_tasks": [{"task_id": "task3"}]}},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{"json": [{"task_id": "task1", "state": "finished"}]},
{"json": [{"task_id": "task2", "state": "error"}]},
{"json": [{"task_id": "task3", "state": "finished"}]},
],
)
publish_f = repo.publish()
# It should succeed
tasks = publish_f.result()
# It should return only the *successful* tasks
assert sorted([t.id for t in tasks]) == ["task1", "task3"]
# Pick out the HTTP requests triggering distributors
publish_reqs = [
req
for req in requests_mocker.request_history
if req.url
== "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/"
]
publish_distributors = [req.json()["id"] for req in publish_reqs]
# It should have triggered cdn_distributor twice, since the first attempt failed
assert publish_distributors == [
"yum_distributor",
"cdn_distributor",
"cdn_distributor",
]
# The retry should have been logged
messages = caplog.messages
assert (
messages[-1].splitlines()[0] == "Retrying due to error: Task task2 failed [1/6]"
)
def test_can_publish():
"""repo.publish() succeeds with fake client and populates publish_history."""
controller = FakeController()
controller.insert_repository(
YumRepository(
id="repo1",
distributors=[
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="cdn_distributor",
type_id="rpm_rsync_distributor"),
],
))
controller.insert_repository(YumRepository(id="repo2"))
client = controller.client
repo1 = client.get_repository("repo1")
# Call to publish should succeed
publish_f = repo1.publish()
# The future should resolve successfully
tasks = publish_f.result()
# It should have returned at least one successful task.
assert tasks
for task in tasks:
assert task.succeeded
# The change should be reflected in the controller's publish history
history = controller.publish_history
assert len(history) == 1
assert history[0].repository.id == "repo1"
assert history[0].tasks == tasks
def _add_repo(controller):
# test repos added to the controller
dt1 = datetime(2019, 9, 10, 0, 0, 0)
r1_d1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1",
last_publish=dt1,
relative_url="content/unit/1/client",
)
r1_d2 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1",
last_publish=dt1,
relative_url="content/unit/1/client",
)
repo1 = Repository(
id="repo1",
eng_product_id=101,
distributors=[r1_d1, r1_d2],
relative_url="content/unit/1/client",
mutable_urls=["mutable1", "mutable2"],
)
dt2 = datetime(2019, 9, 12, 0, 0, 0)
d2 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo2",
last_publish=dt2,
relative_url="content/unit/2/client",
)
repo2 = Repository(
id="repo2",
eng_product_id=102,
distributors=[d2],
relative_url="content/unit/2/client",
)
dt3 = datetime(2019, 9, 7, 0, 0, 0)
d3 = Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo3",
last_publish=dt3,
relative_url="content/unit/3/client",
)
repo3 = Repository(
id="repo3",
eng_product_id=103,
distributors=[d3],
relative_url="content/unit/3/client",
)
controller.insert_repository(repo1)
controller.insert_repository(repo2)
controller.insert_repository(repo3)
def test_publish_cancel(fast_poller, requests_mocker, client, caplog):
"""Cancelling a publish future will cancel running Pulp task(s)."""
repo = YumRepository(
id="some-repo",
distributors=(
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="cdn_distributor", type_id="cdn_distributor"),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[{
"json": {
"spawned_tasks": [{
"task_id": "task1"
}]
}
}],
)
task_search_url = "https://pulp.example.com/pulp/api/v2/tasks/search/"
requests_mocker.post(task_search_url, [{
"json": [{
"task_id": "task1",
"state": "running"
}]
}])
requests_mocker.delete("https://pulp.example.com/pulp/api/v2/tasks/task1/")
# Start the publish
publish_f = repo.publish()
# Wait until we're sure poll thread has seen this task
for _ in range(0, 1000):
if (requests_mocker.last_request
and requests_mocker.last_request.url == task_search_url):
break
time.sleep(0.001)
assert requests_mocker.last_request.url == task_search_url
# We should be able to cancel it
assert publish_f.cancel()
# It should have cancelled the underlying Pulp task
task_req = [
r for r in requests_mocker.request_history
if r.url == "https://pulp.example.com/pulp/api/v2/tasks/task1/"
]
assert len(task_req) == 1
assert task_req[0].method == "DELETE"
def test_publish_with_options(requests_mocker, client):
"""publish passes expected config into distributors based on publish options"""
repo = YumRepository(
id="some-repo",
distributors=(
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="cdn_distributor", type_id="rpm_rsync_distributor"),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[
{"json": {"spawned_tasks": [{"task_id": "task1"}]}},
{"json": {"spawned_tasks": [{"task_id": "task2"}]}},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{"json": [{"task_id": "task1", "state": "finished"}]},
{"json": [{"task_id": "task2", "state": "finished"}]},
],
)
options = PublishOptions(
clean=True, force=True, origin_only=True, rsync_extra_args=["-a"]
)
# It should have succeeded, with the tasks as retrieved from Pulp
assert sorted(repo.publish(options)) == [
Task(id="task1", succeeded=True, completed=True),
Task(id="task2", succeeded=True, completed=True),
]
req = requests_mocker.request_history
# The yum_distributor request should have set force_full, but not
# delete since it's not recognized by that distributor
assert req[0].json()["override_config"] == {"force_full": True}
# The cdn_distributor request should have set force_full, delete
# and content_units_only
assert req[2].json()["override_config"] == {
"force_full": True,
"delete": True,
"content_units_only": True,
"rsync_extra_args": ["-a"],
}
def test_publish_order(requests_mocker, client):
"""publish runs docker/rsync distributors in correct order"""
repo = ContainerImageRepository(
id="some-repo",
distributors=(
Distributor(
id="docker_web_distributor_name_cli", type_id="docker_distributor_web"
),
Distributor(id="cdn_distributor", type_id="docker_rsync_distributor"),
Distributor(
id="cdn_distributor_unprotected", type_id="docker_rsync_distributor"
),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[
{"json": {"spawned_tasks": [{"task_id": "task1"}]}},
{"json": {"spawned_tasks": [{"task_id": "task2"}]}},
{"json": {"spawned_tasks": [{"task_id": "task3"}]}},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{"json": [{"task_id": "task1", "state": "finished"}]},
{"json": [{"task_id": "task2", "state": "finished"}]},
{"json": [{"task_id": "task3", "state": "finished"}]},
],
)
# It should have succeeded, with the tasks as retrieved from Pulp
assert sorted(repo.publish().result()) == [
Task(id="task1", succeeded=True, completed=True),
Task(id="task2", succeeded=True, completed=True),
Task(id="task3", succeeded=True, completed=True),
]
req = requests_mocker.request_history
ids = [r.json()["id"] for r in req if r.url.endswith("/publish/")]
# It should have triggered these distributors in this order
assert ids == [
"cdn_distributor",
"cdn_distributor_unprotected",
"docker_web_distributor_name_cli",
]
def test_delete_missing_repo_fails():
"""dist.delete() of distributor with absent repo fails."""
controller = FakeController()
controller.insert_repository(
Repository(
id="repo",
distributors=[
Distributor(id="somedist", type_id="sometype", repo_id="repo")
],
))
client = controller.client
# Get two handles to the repo, to set up the situation that
# the distributor is not detached despite repo deletion
repo1 = client.get_repository("repo")
repo2 = client.get_repository("repo")
dist = repo2.distributors[0]
# We can delete the repo
assert repo1.delete().result()
# The distributor (obtained via repo2) is not detached, but nevertheless
# deletion should fail as repo no longer exists
assert "Repository id=repo not found" in str(dist.delete().exception())
def test_publish_fail(fast_poller, requests_mocker, client):
"""publish raises TaskFailedException if publish task fails"""
repo = YumRepository(
id="some-repo",
distributors=(Distributor(id="yum_distributor",
type_id="yum_distributor"), ),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
json={"spawned_tasks": [{
"task_id": "task1"
}]},
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
json=[{
"task_id": "task1",
"state": "error"
}],
)
publish_f = repo.publish()
# It should raise this exception
with pytest.raises(TaskFailedException) as error:
publish_f.result()
# The exception should have a reference to the task which failed
assert error.value.task.id == "task1"
assert "Task task1 failed" in str(error.value)
def test_maintenance_on_with_regex(command_tester):
"""Test set maintenance by using regex"""
repo1 = Repository(id="repo1")
dist1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
relative_url="rhel/7",
repo_id="repo2",
)
repo2 = Repository(id="repo2", distributors=(dist1, ))
with get_task_instance(True, repo1, repo2) as task_instance:
command_tester.test(
task_instance.main,
[
"test-maintenance-on",
"--pulp-url",
"http://some.url",
"--repo-url-regex",
"rhel",
],
)
assert_expected_report(["repo2"], task_instance.pulp_client)
def get_task_instance(on, *repos):
iso_distributor = Distributor(
id="iso_distributor",
type_id="iso_distributor",
relative_url="root",
repo_id="redhat-maintenance",
)
maint_repo = FileRepository(id="redhat-maintenance",
distributors=[iso_distributor])
if on:
task_instance = FakeSetMaintenanceOn()
task_instance.pulp_client_controller.insert_repository(maint_repo)
else:
task_instance = FakeSetMaintenanceOff()
task_instance.pulp_client_controller.insert_repository(maint_repo)
# if unset maintenance mode, we need to pre-set maintenance first
report = task_instance.pulp_client.get_maintenance_report().result()
report = report.add([repo.id for repo in repos])
task_instance.pulp_client.set_maintenance(report)
for repo in list(repos):
task_instance.pulp_client_controller.insert_repository(repo)
return task_instance
def test_distributors_created():
"""from_data sets distributors attribute appropriately"""
repo = Repository.from_data(
{
"id": "some-repo",
"distributors": [
{"id": "dist1", "distributor_type_id": "type1"},
{"id": "dist2", "distributor_type_id": "type1"},
],
}
)
assert repo.distributors == (
Distributor(id="dist1", type_id="type1"),
Distributor(id="dist2", type_id="type1"),
)
def test_publish_out_repos(mock_ubipop_runner):
dt = datetime(2019, 9, 12, 0, 0, 0)
d1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo",
last_publish=dt,
relative_url="content/unit/2/client",
)
repo = YumRepository(
id="repo",
eng_product_id=102,
distributors=[d1],
relative_url="content/unit/2/client",
)
fake_pulp = FakeController()
repo.__dict__["_client"] = fake_pulp.client
fake_pulp.insert_repository(repo)
# Setup output repos, leave only binary repo as the actual one
mock_ubipop_runner.repos.out_repos = RepoSet(
f_proxy(f_return(repo)), f_proxy(f_return()), f_proxy(f_return())
)
fts = mock_ubipop_runner._publish_out_repos()
# we should publish only one repository with one distributor
assert len(fts) == 1
assert [hist.repository.id for hist in fake_pulp.publish_history] == ["repo"]
def test_delete_missing_distributor_succeeds():
"""dist.delete() of absent distributor with fake client succeeds.
Deleting a distributor succeeds with the fake client since it also succeeds
with the real client.
"""
controller = FakeController()
controller.insert_repository(
Repository(
id="repo",
distributors=[
Distributor(id="somedist", type_id="sometype", repo_id="repo")
],
))
client = controller.client
# Get two handles to the repos/distributors so we can delete via one handle
# and try to delete again via the other
repo_copy1 = client.get_repository("repo")
repo_copy2 = client.get_repository("repo")
dist_copy1 = repo_copy1.distributors[0]
dist_copy2 = repo_copy2.distributors[0]
# First delete succeeds, with some tasks
assert dist_copy1.delete().result()
# Second delete also succeeds, but there are no tasks since distributor
# already doesn't exist
assert dist_copy2.delete().result() == []
def test_invalid_distributor_repo_id():
"""distributor's repo id being different from repo it's attached is invalid"""
dist = Distributor(id="dist", type_id="yum_distributor", repo_id="repo")
with pytest.raises(ValueError) as ex:
repo = Repository(id="test_repo", distributors=[dist])
assert "repo_id doesn't match for dist" in str(ex.value)
def test_delete_success(fast_poller, requests_mocker, client):
"""delete succeeds and returns spawned tasks"""
repo = Repository(
id="some-repo",
distributors=[
Distributor(id="dist1", type_id="type1", repo_id="some-repo"),
Distributor(id="dist2", type_id="type2", repo_id="some-repo"),
],
)
repo._set_client(client)
requests_mocker.delete(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/distributors/dist1/",
json={"spawned_tasks": [{
"task_id": "task1"
}, {
"task_id": "task2"
}]},
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
json=[
{
"task_id": "task1",
"state": "finished"
},
{
"task_id": "task2",
"state": "skipped"
},
],
)
# It should succeed, with the tasks as retrieved from Pulp
dist = repo.distributors[0]
delete_dist = dist.delete()
assert sorted(delete_dist) == [
Task(id="task1", succeeded=True, completed=True),
Task(id="task2", succeeded=True, completed=True),
]
# And should now be detached
with pytest.raises(DetachedException):
dist.delete()
def test_can_search_distributors_with_relative_url(client, requests_mocker):
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/distributors/search/",
json=[
{
"id": "yum_distributor",
"distributor_type_id": "yum_distributor",
"repo_id": "test_rpm",
"config": {
"relative_url": "relative/path"
},
},
{
"id": "cdn_distributor",
"distributor_type_id": "rpm_rsync_distributor",
"config": {
"relative_url": "relative/path"
},
},
],
)
crit = Criteria.with_field("relative_url", Matcher.regex("relative/path"))
distributors_f = client.search_distributor(crit)
distributors = [dist for dist in distributors_f.result()]
# distributor objects are returned
assert sorted(distributors) == [
Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
relative_url="relative/path",
),
Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="test_rpm",
relative_url="relative/path",
),
]
# api is called once
assert requests_mocker.call_count == 1
def test_search_distributor():
controller = FakeController()
dist1 = Distributor(id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1")
dist2 = Distributor(id="cdn_distributor",
type_id="rpm_rsync_distributor",
repo_id="repo1")
repo1 = Repository(id="repo1", distributors=(dist1, dist2))
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.true()
found = client.search_distributor(crit).result().data
assert sorted(found) == [dist2, dist1]
def test_can_search_distributor(client, requests_mocker):
"""search_distributor issues distributors/search POST request as expected."""
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/distributors/search/",
json=[
{
"id": "yum_distributor",
"distributor_type_id": "yum_distributor",
"repo_id": "test_rpm",
"config": {
"relative_url": "relative/path"
},
},
{
"id": "cdn_distributor",
"distributor_type_id": "rpm_rsync_distributor",
"config": {
"relative_url": "relative/path"
},
},
],
)
distributors_f = client.search_distributor()
distributors = [dist for dist in distributors_f.result().as_iter()]
# distributor objects are returned
assert sorted(distributors) == [
Distributor(
id="cdn_distributor",
type_id="rpm_rsync_distributor",
relative_url="relative/path",
),
Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="test_rpm",
relative_url="relative/path",
),
]
# api is called once
assert requests_mocker.call_count == 1
def test_can_delete():
"""distributor.delete() with fake client removes distributor from fake data."""
controller = FakeController()
repo1 = Repository(
id="repo1",
distributors=[
Distributor(id="dist1", type_id="type1", repo_id="repo1"),
Distributor(id="dist2", type_id="type2", repo_id="repo1"),
],
)
controller.insert_repository(repo1)
client = controller.client
repo1 = client.get_repository("repo1")
# Call to delete should succeed
delete_f = repo1.distributors[0].delete()
# The future should resolve successfully
tasks = delete_f.result()
# It should have returned at least one successful task.
assert tasks
for task in tasks:
assert task.succeeded
# The change should be reflected in the controller, with the
# deleted distributor no longer present
assert controller.repositories == [
Repository(
id="repo1",
distributors=[
Distributor(id="dist2", type_id="type2", repo_id="repo1")
],
)
]
def test_search_null_and():
"""Search with an empty AND gives an error."""
controller = FakeController()
dist1 = Distributor(id="yum_distributor",
type_id="yum_distributor",
repo_id="repo1")
repo1 = Repository(id="repo1", distributors=[dist1])
controller.insert_repository(repo1)
client = controller.client
crit = Criteria.and_()
assert "Invalid AND in search query" in str(
client.search_repository(crit).exception())
assert "Invalid AND in search query" in str(
client.search_distributor(crit).exception())
def test_publish_broken_response(fast_poller, requests_mocker, client):
"""publish raises an exception if Pulp /publish/ responded with parseable
JSON, but not of the expected structure
"""
repo = YumRepository(
id="some-repo",
distributors=(Distributor(id="yum_distributor", type_id="yum_distributor"),),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
json={"spawned_tasks": ["oops, not a valid response"]},
)
publish_f = repo.publish()
# It should raise some kind of exception due to the invalid spawned_tasks structure
assert publish_f.exception()
def test_detached_noclient():
"""delete raises if called on a distributor without client"""
with pytest.raises(DetachedException):
Distributor(id="some-dist",
type_id="some-dist-type",
repo_id="some-repo").delete()
def test_detached_norepo(client):
"""delete raises if called on a distributor without repo"""
with pytest.raises(DetachedException):
dist = Distributor(id="some-dist", type_id="some-dist-type")
dist._set_client(client)
dist.delete()
def test_publish_update_mappings(fast_poller, requests_mocker, client, caplog):
"""Publish on FileRepository will generate ud_file_release_mappings_2."""
caplog.set_level(logging.INFO, "pubtools.pulplib")
repo = FileRepository(
id="some-repo",
distributors=[
Distributor(id="iso_distributor", type_id="iso_distributor")
],
)
repo.__dict__["_client"] = client
# Force client to use a small page size so that we're able to verify
# all pages end up handled.
client._PAGE_SIZE = 3
# Arrange for the repo to currently exist with some mappings.
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/",
json={
"id": "some-repo",
"notes": {
"ud_file_release_mappings_2":
json.dumps(
# Define some mappings so there's a mix of data needing
# an update and data needing no update.
{
"1.0": [{
"filename": "file1",
"order": 3.0
}],
"3.0": [{
"filename": "file3",
"order": 1234
}],
"4.0": [{
"filename": "file4",
"order": 0
}],
"some-other-version": [{
"filename": "whatever-file",
"order": "abc",
"foo": "bar"
}],
})
},
},
)
# Make the repo have some units with the mapping-relevant fields. We use
# two pages here to ensure that the code implements pagination.
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/",
[
{
# page 1
"json": [
{
"metadata": {
"_content_type_id": "iso",
"name": "file1",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "1.0",
"display_order": 3.0,
},
}
},
{
"metadata": {
"_content_type_id":
"iso",
"name":
"file2",
"size":
1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
}
},
{
"metadata": {
"_content_type_id": "iso",
"name": "file3",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "3.0"
},
}
},
]
},
{
# page 2
"json": [
{
"metadata": {
"_content_type_id": "iso",
"name": "file4",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "4.0",
"display_order": -2,
},
}
},
{
"metadata": {
"_content_type_id": "iso",
"name": "file5",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "1.0",
"display_order": 4.5,
},
}
},
{
"metadata": {
"_content_type_id": "iso",
"name": "file6",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "6.0"
},
}
},
]
},
{
# page 3: nothing more
"json": []
},
],
)
# Allow for an update to the repository.
requests_mocker.put(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/",
json={})
# It should publish as usual.
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
json={"spawned_tasks": [{
"task_id": "publish-task"
}]},
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
json=[{
"task_id": "publish-task",
"state": "finished"
}],
)
# It should have succeeded, with the publish task as retrieved from pulp
assert sorted(repo.publish()) == [
Task(id="publish-task", succeeded=True, completed=True)
]
# Now let's have a look at what it updated.
# The last operations are (update repo, start publish, await tasks),
# so the update request should always be the 3rd last.
req = requests_mocker.request_history[-3]
# Should have been a PUT to the repo.
assert req.url == "https://pulp.example.com/pulp/api/v2/repositories/some-repo/"
assert req.method == "PUT"
# It should have included this field in the delta...
mapping_json = req.json()["delta"]["notes"]["ud_file_release_mappings_2"]
# It should have been valid JSON
mapping = json.loads(mapping_json)
# It should be equal to this:
assert mapping == {
"1.0": [
# This file wasn't changed
{
"filename": "file1",
"order": 3.0
},
# This was added
{
"filename": "file5",
"order": 4.5
},
],
# This wasn't touched because, although the repo has the file, it has no
# defined order
"3.0": [{
"filename": "file3",
"order": 1234
}],
# This was updated
"4.0": [{
"filename": "file4",
"order": -2.0
}],
# This was added, with the entire version not previously defined.
# Also, order field is omitted because none was in the data, but
# the version and file still must be written.
"6.0": [{
"filename": "file6"
}],
# This is some unrelated junk which should be left alone during the
# update process.
"some-other-version": [{
"filename": "whatever-file",
"foo": "bar",
"order": "abc"
}],
}
# It should have logged that this happened
assert "Updated ud_file_release_mappings_2 in some-repo" in caplog.messages
def test_publish_update_mappings_noop(fast_poller, requests_mocker, client):
"""Publish on FileRepository does not do unnecessary repo updates."""
repo = FileRepository(
id="some-repo",
distributors=[
Distributor(id="iso_distributor", type_id="iso_distributor")
],
)
repo.__dict__["_client"] = client
# Arrange for the repo to currently exist with up-to-date mappings.
requests_mocker.get(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/",
json={
"id": "some-repo",
"notes": {
"ud_file_release_mappings_2":
json.dumps({
"1.0": [
{
"filename": "file1",
"order": 3.0
},
{
"filename": "file2",
"order": 6.0
},
],
"other": ["whatever"],
})
},
},
)
# Make the repo have some units with the mapping-relevant fields.
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/",
json=[
{
"metadata": {
"_content_type_id": "iso",
"name": "file1",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "1.0",
"display_order": 3.0
},
}
},
{
"metadata": {
"_content_type_id": "iso",
"name": "file2",
"size": 1,
"checksum":
"cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411",
"pulp_user_metadata": {
"version": "1.0"
},
}
},
],
)
# It should NOT update the repository, so we don't register any PUT.
# It should publish as usual.
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
json={"spawned_tasks": [{
"task_id": "publish-task"
}]},
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
json=[{
"task_id": "publish-task",
"state": "finished"
}],
)
# It should have succeeded, with the tasks as retrieved from Pulp
assert sorted(repo.publish()) == [
Task(id="publish-task", succeeded=True, completed=True)
]
def test_publish_distributors(fast_poller, requests_mocker, client):
"""publish succeeds and returns tasks from each applicable distributor"""
repo = YumRepository(
id="some-repo",
distributors=(
Distributor(id="yum_distributor", type_id="yum_distributor"),
Distributor(id="other_distributor", type_id="other_distributor"),
Distributor(id="cdn_distributor", type_id="rpm_rsync_distributor"),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[
{
"json": {
"spawned_tasks": [{
"task_id": "task1"
}, {
"task_id": "task2"
}]
}
},
{
"json": {
"spawned_tasks": [{
"task_id": "task3"
}]
}
},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [
{
"task_id": "task1",
"state": "finished"
},
{
"task_id": "task2",
"state": "skipped"
},
]
},
{
"json": [{
"task_id": "task3",
"state": "finished"
}]
},
],
)
# It should have succeeded, with the tasks as retrieved from Pulp
assert sorted(repo.publish()) == [
Task(id="task1", succeeded=True, completed=True),
Task(id="task2", succeeded=True, completed=True),
Task(id="task3", succeeded=True, completed=True),
]
# It should have first issued a request to publish yum_distributor
req = requests_mocker.request_history
assert (
req[0].url ==
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/"
)
assert req[0].json() == {"id": "yum_distributor", "override_config": {}}
# Then polled for resulting tasks to succeed
assert req[1].url == "https://pulp.example.com/pulp/api/v2/tasks/search/"
assert req[1].json() == {
"criteria": {
"filters": {
"task_id": {
"$in": ["task1", "task2"]
}
}
}
}
# Then published the next distributor
assert (
req[2].url ==
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/"
)
assert req[2].json() == {"id": "cdn_distributor", "override_config": {}}
# Then waited for those tasks to finish too
assert req[3].url == "https://pulp.example.com/pulp/api/v2/tasks/search/"
assert req[3].json() == {
"criteria": {
"filters": {
"task_id": {
"$in": ["task3"]
}
}
}
}
# And there should have been no more requests
assert len(req) == 4
def test_publish_origin_only(requests_mocker, client):
"""publish skips docker distributor if origin_only=True"""
repo = ContainerImageRepository(
id="some-repo",
distributors=(
Distributor(id="docker_web_distributor_name_cli",
type_id="docker_distributor_web"),
Distributor(id="cdn_distributor",
type_id="docker_rsync_distributor"),
Distributor(id="cdn_distributor_unprotected",
type_id="docker_rsync_distributor"),
),
)
repo.__dict__["_client"] = client
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/publish/",
[
{
"json": {
"spawned_tasks": [{
"task_id": "task1"
}]
}
},
{
"json": {
"spawned_tasks": [{
"task_id": "task2"
}]
}
},
],
)
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/tasks/search/",
[
{
"json": [{
"task_id": "task1",
"state": "finished"
}]
},
{
"json": [{
"task_id": "task2",
"state": "finished"
}]
},
],
)
# It should have succeeded, with the tasks as retrieved from Pulp
assert sorted(repo.publish(PublishOptions(origin_only=True))) == [
Task(id="task1", succeeded=True, completed=True),
Task(id="task2", succeeded=True, completed=True),
]
req = requests_mocker.request_history
publish_req = [r for r in req if r.url.endswith("/publish/")]
# It should have triggered these distributors in this order
ids = [r.json()["id"] for r in publish_req]
assert ids == ["cdn_distributor", "cdn_distributor_unprotected"]
# It should have triggered with content_units_only=True
assert publish_req[-1].json()["override_config"]["content_units_only"]
