def test_publish_out_repos(mock_ubipop_runner):
dt = datetime(2019, 9, 12, 0, 0, 0)
d1 = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id="repo",
last_publish=dt,
relative_url="content/unit/2/client",
)
repo = YumRepository(
id="repo",
eng_product_id=102,
distributors=[d1],
relative_url="content/unit/2/client",
)
fake_pulp = FakeController()
repo.__dict__["_client"] = fake_pulp.client
fake_pulp.insert_repository(repo)
# Setup output repos, leave only binary repo as the actual one
mock_ubipop_runner.repos.out_repos = RepoSet(
f_proxy(f_return(repo)), f_proxy(f_return()), f_proxy(f_return())
)
fts = mock_ubipop_runner._publish_out_repos()
# we should publish only one repository with one distributor
assert len(fts) == 1
assert [hist.repository.id for hist in fake_pulp.publish_history] == ["repo"]
def _delete_distributor(self, repo_id, distributor_id):
with self._lock:
repo_f = self.get_repository(repo_id)
if repo_f.exception():
# Repo can't be found, let that exception propagate
return repo_f
repo = repo_f.result()
new_distributors = [
dist for dist in repo.distributors if dist.id != distributor_id
]
dist_found = new_distributors != repo.distributors
if not dist_found:
# Deleting something which already doesn't exist is fine
return f_return([])
idx = self._repositories.index(repo)
self._repositories[idx] = attr.evolve(
repo, distributors=new_distributors)
return f_return([
Task(
id=self._next_task_id(),
completed=True,
succeeded=True,
tags=[
"pulp:repository:%s" % repo_id,
"pulp:repository_distributor:%s" % distributor_id,
"pulp:action:remove_distributor",
],
)
])
def test_num_types():
xint = f_proxy(f_return(123))
xfloat = f_proxy(f_return(1.23))
assert float(xint) == 123.0
assert complex(xint) == complex(123)
assert int(xfloat) == 1
def fixture_ubi_repo_set_no_debug():
ubi_binary = get_test_repo(id="ubi-foo-rpms", ubi_config_version="7")
ubi_source = get_test_repo(id="ubi-foo-source", ubi_config_version="7")
ubi_binary.get_source_repository = MagicMock()
ubi_binary.get_source_repository.return_value = ubi_source
ubi_binary.get_debug_repository = MagicMock()
ubi_binary.get_debug_repository.return_value = f_proxy(f_return())
rhel_binary = get_test_repo(id="foo-rpms", ubi_config_version="7")
rhel_source = get_test_repo(id="foo-source", ubi_config_version="7")
rhel_binary.get_source_repository = MagicMock()
rhel_binary.get_source_repository.return_value = rhel_source
rhel_binary.get_debug_repository = MagicMock()
rhel_binary.get_debug_repository.return_value = f_proxy(f_return())
yield UbiRepoSet(
RepoSet(
[rhel_binary],
[rhel_source],
f_proxy(f_return()),
),
RepoSet(
ubi_binary,
ubi_source,
f_proxy(f_return()),
),
)
def test_get_modular_srpms_criteria(ubi_config):
"""Testing creation of criteria for srpms query"""
matcher = ModularMatcher(None, ubi_config.modules)
unit_1 = UbiUnit(
RpmUnit(
name="test",
version="1.0",
release="1",
arch="x86_64",
sourcerpm="test.x86_64.src.rpm",
),
None,
)
unit_2 = UbiUnit(
RpmUnit(
name="test-debug",
version="1.0",
release="1",
arch="i386",
sourcerpm="test-debug.i386.src.rpm",
),
None,
)
# we need to set up binary and debug rpms
# the criteria are based on sourcerpm attr of those units
matcher.binary_rpms = f_proxy(f_return(set([unit_1])))
matcher.debug_rpms = f_proxy(f_return(set([unit_2])))
criteria = matcher._get_srpms_criteria()
# there should be 1 criteria created
assert len(criteria) == 2
# it should be instance of Criteria
for crit in criteria:
assert isinstance(crit, Criteria)
def test_as_completed_with_timeout_reset():
# tests correctly yielded futures from as_completed_with_timeout_reset()
futures = [f_return(1), f_return(2)]
current = set()
for ft in as_completed_with_timeout_reset(futures, timeout=1):
value = ft.result()
current.add(value)
expected = set([1, 2])
assert current == expected
def test_bool():
xtrue = f_proxy(f_return(True))
xfalse = f_proxy(f_return(False))
xempty = f_proxy(f_return([]))
# All of these count as truthy, as we want to allow testing if
# a future was returned without blocking
assert xtrue
assert xfalse
assert xempty
def fixture_mock_current_content():
rpm = get_rpm_unit(
name="rpm_current",
filename="rpm_current.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-rpms",
)
srpm = get_srpm_unit(
name="srpm_current",
filename="srpm_current.src.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-source",
)
debug_rpm = get_rpm_unit(
name="debug_rpm_current",
filename="debug_rpm_current.rpm",
version="1",
release="0",
arch="x86_64",
src_repo_id="ubi-foo-debug",
)
modulemd_unit = get_modulemd_unit(
name="md_current",
stream="foo",
version=1,
context="bar",
arch="x86_64",
src_repo_id="ubi-foo-rpms",
)
modulemd_defaults_unit = get_modulemd_defaults_unit(
name="mdd_current",
stream="rhel",
profiles={"2.5": ["common"]},
repo_id="ubi-foo-rpms",
src_repo_id="ubi-foo-rpms",
)
binary_rpms = f_proxy(f_return([rpm]))
debug_rpms = f_proxy(f_return([srpm]))
source_rpms = f_proxy(f_return([debug_rpm]))
modulemds = f_proxy(f_return([modulemd_unit]))
modulemd_defaults = f_proxy(f_return([modulemd_defaults_unit]))
repo_content = RepoContent(
binary_rpms, debug_rpms, source_rpms, modulemds, modulemd_defaults
)
yield repo_content
def test_flat_map_error():
map_in = f_return(0)
# This one should fail...
mapped = f_flat_map(map_in, div10)
# Now map it through an error handler
mapped = f_flat_map(mapped, error_fn=lambda ex: f_return(str(ex)))
result = mapped.result()
assert "division" in result
def test_len():
xlist = f_proxy(f_return([1, 2, 3]))
xint = f_proxy(f_return(42))
xstr = f_proxy(f_return("hello"))
assert len(xlist) == 3
assert len(xstr) == 5
with pytest.raises(TypeError) as exc:
len(xint)
assert "has no len" in str(exc.value)
def test_set_maintenance(client, requests_mocker):
maintenance_report = {
"last_updated": "2019-08-15T14:21:12Z",
"last_updated_by": "pubtools.pulplib",
"repos": {
"repo1": {
"message": "Maintenance Mode Enabled",
"owner": "pubtools.pulplib",
"started": "2019-08-15T14:21:12Z",
}
},
}
requests_mocker.post(
"https://pulp.example.com/pulp/api/v2/repositories/search/",
[{
"json": [{
"id": "redhat-maintenance",
"notes": {
"_repo-type": "iso-repo"
}
}]
}],
)
report = MaintenanceReport._from_data(maintenance_report)
with patch("pubtools.pulplib.FileRepository.upload_file") as mocked_upload:
with patch("pubtools.pulplib.Repository.publish") as mocked_publish:
upload_task = Task(id="upload-task",
completed=True,
succeeded=True)
publish_task = [
Task(id="publish-task", completed=True, succeeded=True)
]
mocked_upload.return_value = f_return(upload_task)
mocked_publish.return_value = f_return(publish_task)
# set_maintenance.result() should return whatever publish.result() returns
assert client.set_maintenance(report).result() is publish_task
# upload_file should be called with (file_obj, 'repos.json')
args = mocked_upload.call_args
report_file = args[0][0]
report = MaintenanceReport()._from_data(json.loads(report_file.read()))
assert len(report.entries) == 1
assert report.entries[0].repo_id == "repo1"
assert report.last_updated_by == "pubtools.pulplib"
# search repo, upload and publish should be called once each
assert requests_mocker.call_count == 1
assert mocked_publish.call_count == 1
assert mocked_upload.call_count == 1
def test_sign():
xint = f_proxy(f_return(123))
xnegint = f_proxy(f_return(-123))
assert -xint == -123
assert +xint == 123
assert abs(xnegint) == 123
assert ~xint == -124
assert ~xnegint == 122
assert ~~xint == 123
assert ~~xnegint == -123
def test_gc_error(mock_logger):
"""logs error when repo delete task returns an error reponse"""
repo = Repository(
id="rhel-test-garbage-collect-7-days-old",
created=_get_created(7),
is_temporary=True,
)
controller = _get_fake_controller(repo)
gc = GarbageCollect()
arg = ["", "--pulp-url", "http://some.url"]
with patch("sys.argv", arg):
with patch.object(controller.client,
"_delete_repository") as repo_delete:
with _patch_pulp_client(controller.client):
repo_delete.return_value = f_return([
Task(
id="12334",
completed=True,
succeeded=False,
error_summary="Error occured",
)
])
gc.main()
mock_logger.error.assert_any_call("Error occured")
def update_repository(self, repository):
self._ensure_alive()
with self._state.lock:
existing_repo = None
for candidate in self._state.repositories:
if candidate.id == repository.id:
existing_repo = candidate
break
else:
return f_return_error(
PulpException("repository not found: %s" % repository.id)
)
# We've got a repo, now update it.
update = {}
for fld in existing_repo._mutable_note_fields():
update[fld.name] = getattr(repository, fld.name)
updated_repo = attr.evolve(existing_repo, **update)
self._state.repositories = [
repo for repo in self._state.repositories if repo.id != updated_repo.id
] + [updated_repo]
return f_return()
def _do_upload_file(self, upload_id, file_obj, name="<unknown file>"): # pylint: disable=unused-argument
# We keep track of uploaded content as we may need it at import time.
buffer = six.BytesIO()
self._uploads_pending[upload_id] = buffer
is_file_obj = "close" in dir(file_obj)
if not is_file_obj:
file_obj = open(file_obj, "rb")
def do_next_upload(checksum, size):
while True:
data = file_obj.read(1024 * 1024)
if not data:
break
if isinstance(data, six.text_type):
data = data.encode("utf-8")
buffer.write(data)
checksum.update(data)
size += len(data)
return f_return(UploadResult(checksum.hexdigest(), size))
out = f_flat_map(f_return(),
lambda _: do_next_upload(hashlib.sha256(), 0))
out.add_done_callback(lambda _: file_obj.close())
return out
def update_content(self, unit):
self._ensure_alive()
if not unit.unit_id:
raise ValueError("unit_id missing on call to update_content()")
# The unit has to exist.
existing_unit = None
for candidate in self._all_units:
if (candidate.content_type_id == unit.content_type_id
and candidate.unit_id == unit.unit_id):
existing_unit = candidate
break
else:
return f_return_error(
PulpException("unit not found: %s" % unit.unit_id))
# OK, we have a unit to update. Figure out which fields we can update.
update = {}
for fld in unit._usermeta_fields():
update[fld.name] = getattr(unit, fld.name)
updated_unit = attr.evolve(existing_unit, **update)
unit_key = units.make_unit_key(updated_unit)
self._units_by_key[unit_key] = updated_unit
return f_return()
def empty_future(value):
if "add_done_callback" in dir(value):
# It's a future, map it to None
return f_map(value, lambda _: None)
# It's not a future => operation has already completed,
# return empty future to denote success
return f_return()
def get_maintenance_report(self):
if self._maintenance_report:
report = MaintenanceReport._from_data(
json.loads(self._maintenance_report))
else:
report = MaintenanceReport()
return f_proxy(f_return(report))
def _do_unassociate(self, repo_id, type_ids):
repo_f = self.get_repository(repo_id)
if repo_f.exception():
return repo_f
current = self._repo_units.get(repo_id, set())
removed = set()
kept = set()
for unit in current:
if type_ids is None or unit.content_type_id in type_ids:
removed.add(unit)
else:
kept.add(unit)
self._repo_units[repo_id] = kept
task = Task(
id=self._next_task_id(),
repo_id=repo_id,
completed=True,
succeeded=True,
units=removed,
)
return f_return([task])
def _do_import(
self, repo_id, upload_id, unit_type_id, unit_key, unit_metadata=None
):
repo_f = self.get_repository(repo_id)
if repo_f.exception():
# Repo can't be found, let that exception propagate
return repo_f
repo = repo_f.result()
with self._state.lock:
# Get the uploaded content we're about to import; though it's not
# guaranteed to be present (e.g. erratum has no file).
# If not present, we just use an empty BytesIO.
upload_content = self._state.uploads_pending.pop(upload_id, six.BytesIO())
upload_content.seek(0)
new_units = self._state.unitmaker.make_units(
unit_type_id, unit_key, unit_metadata, upload_content, repo_id
)
new_units = [
attr.evolve(u, repository_memberships=[repo.id]) for u in new_units
]
self._state.insert_repo_units(repo_id, new_units)
task = Task(id=self._state.next_task_id(), completed=True, succeeded=True)
# upload_history is a deprecated field, data is maintained for iso only.
if unit_type_id == "iso":
self._state.upload_history.append(
Upload(repo, [task], unit_key["name"], unit_key["checksum"])
)
return f_return([task])
def test_gc_error(mock_logger):
"""logs error when repo delete task returns an error reponse"""
repo = {
"id": "rhel-test-garbage-collect-7-days-old",
"notes": {
"pub_temp_repo": True,
"created": _get_time_created(7)
},
}
controller = _get_fake_controller(repo)
gc = GarbageCollect()
arg = ["", "--pulp-url", "http://some.url", "--verbose"]
with patch("sys.argv", arg):
with patch.object(controller.client,
"_delete_repository") as repo_delete:
with patch("pubtools._pulp.task.PulpTask.pulp_client",
controller.client):
repo_delete.return_value = f_return([
Task(
id="12334",
completed=True,
succeeded=False,
error_summary="Error occured",
)
])
gc.main()
mock_logger.error.assert_any_call("Error occured")
def _delete_repository(self, repo_id):
with self._lock:
found = False
for idx, repo in enumerate(self._repositories):
if repo.id == repo_id:
found = True
break
if not found:
# Deleting something which already doesn't exist is fine
return f_return([])
self._repositories.pop(idx) # pylint: disable=undefined-loop-variable
return f_return([
Task(id=self._next_task_id(), completed=True, succeeded=True)
])
def _get_current_content(self):
"""
Gather current content of output repos
"""
criteria = [Criteria.true()]
current_modulemds = f_proxy(
self._executor.submit(Matcher.search_modulemds, criteria,
[self.repos.out_repos.rpm]))
current_modulemd_defaults = f_proxy(
self._executor.submit(Matcher.search_modulemd_defaults, criteria,
[self.repos.out_repos.rpm]))
current_rpms = f_proxy(
self._executor.submit(Matcher.search_rpms, criteria,
[self.repos.out_repos.rpm]))
current_srpms = f_proxy(
self._executor.submit(Matcher.search_srpms, criteria,
[self.repos.out_repos.source]))
if self.repos.out_repos.debug.result():
current_debug_rpms = f_proxy(
self._executor.submit(Matcher.search_rpms, criteria,
[self.repos.out_repos.debug]))
else:
current_debug_rpms = f_proxy(f_return([]))
current_content = RepoContent(
current_rpms,
current_srpms,
current_debug_rpms,
current_modulemds,
current_modulemd_defaults,
)
return current_content
def _get_ubi_repo_sets(self, ubi_binary_cs):
"""
Searches for ubi repository triplet (binary rpm, srpm, debug) for
one ubi config item and tries to determine their population sources
(input repositories). Returns list UbiRepoSet objects that provides
input and output repositories that are used for population process.
"""
rpm_repos = self.pulp_client.search_repository(
Criteria.and_(
Criteria.with_field("notes.content_set", ubi_binary_cs),
Criteria.with_field("ubi_population", True),
))
ubi_repo_sets = []
for out_rpm_repo in rpm_repos:
out_source_repo = out_rpm_repo.get_source_repository()
out_debug_repo = out_rpm_repo.get_debug_repository()
in_rpm_repos = self._get_population_sources(out_rpm_repo)
in_source_repos = self._get_population_sources(out_source_repo)
in_debug_repos = self._get_population_sources(out_debug_repo)
# we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency
# that all objects in out|in_repos are futures
out_repos = (
f_proxy(f_return(out_rpm_repo)),
out_source_repo,
out_debug_repo,
)
in_repos = (in_rpm_repos, in_source_repos, in_debug_repos)
ubi_repo_sets.append(
UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos)))
return ubi_repo_sets
def test_get_modular_rpms_criteria(ubi_config):
"""Test creation of criteria for rpms query"""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="test",
stream="10",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
),
None,
)
matcher.modules = f_proxy(f_return(set([unit])))
criteria = matcher._get_modular_rpms_criteria()
# there should be 1 criterium created - srpm is skipped
assert len(criteria) == 1
# it should be instance of Criteria
for crit in criteria:
assert isinstance(crit, Criteria)
def _do_upload_file(self, upload_id, file_obj, name):
def do_next_upload(checksum, size):
data = file_obj.read(self._CHUNK_SIZE)
if data:
if isinstance(data, six.text_type):
# if it's unicode, need to encode before calculate checksum
data = data.encode("utf-8")
checksum.update(data)
return f_flat_map(
self._do_upload(data, upload_id, size),
lambda _: do_next_upload(checksum, size + len(data)),
)
# nothing more to upload, return checksum and size
return f_return((checksum.hexdigest(), size))
is_file_object = "close" in dir(file_obj)
if not is_file_object:
file_obj = open(file_obj, "rb")
LOG.info("Uploading %s to Pulp", name)
upload_f = f_flat_map(f_return(),
lambda _: do_next_upload(hashlib.sha256(), 0))
if not is_file_object:
upload_f.add_done_callback(lambda _: file_obj.close())
return upload_f
def _get_related_repository(self, repo_t):
if not self._client:
raise DetachedException()
suffixes_mapping = {
"binary": "/os",
"debug": "/debug",
"source": "/source/SRPMS",
}
regex = r"(/os|/source/SRPMS|/debug)$"
def unpack_page(page):
if len(page.data) != 1:
return None
return page.data[0]
suffix = suffixes_mapping[repo_t]
if str(self.relative_url).endswith(suffix):
return f_proxy(f_return(self))
base_url = re.sub(regex, "", self.relative_url)
relative_url = base_url + suffix
criteria = Criteria.with_field("notes.relative_url", relative_url)
page_f = self._client.search_repository(criteria)
repo_f = f_map(page_f, unpack_page)
return f_proxy(repo_f)
def handle_results(page):
for unit in page.data:
unit = UbiUnit(unit, repo.id)
units.add(unit)
if page.next:
return f_flat_map(page.next, handle_results)
return f_return(units)
def do_next_upload(checksum, size):
data = file_obj.read(1024 * 1024)
if data:
checksum.update(data)
size += len(data)
return do_next_upload(checksum, size)
return f_return((checksum.hexdigest(), size))
def test_modular_rpms_filenames_per_profiles_missing_profile(ubi_config):
"""Test getting filename from module artifacts, request for non-existing profile in modulemd"""
matcher = ModularMatcher(None, ubi_config.modules)
unit = UbiUnit(
ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=100,
context="abcd",
arch="x86_64",
artifacts=[
"perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src",
"perl-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
"bash-7:10.5-el6.x86_64",
"bash-devel-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64",
],
profiles={"another": ["bash"]},
),
None,
)
modules = f_proxy(f_return(set([unit])))
filenames = matcher._modular_rpms_filenames(modules)
# all non-src pkgs are in result
# this result is driven by ubi_config that force to use only profile called "test"
# but we don't have this profile in the testing modulemd, so we take all non-src artifacts
assert len(filenames) == 3
assert filenames == set([
"bash-10.5-el6.x86_64.rpm",
"bash-devel-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm",
"perl-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm",
])
