def test_publish_out_repos(mock_ubipop_runner): dt = datetime(2019, 9, 12, 0, 0, 0) d1 = Distributor( id="yum_distributor", type_id="yum_distributor", repo_id="repo", last_publish=dt, relative_url="content/unit/2/client", ) repo = YumRepository( id="repo", eng_product_id=102, distributors=[d1], relative_url="content/unit/2/client", ) fake_pulp = FakeController() repo.__dict__["_client"] = fake_pulp.client fake_pulp.insert_repository(repo) # Setup output repos, leave only binary repo as the actual one mock_ubipop_runner.repos.out_repos = RepoSet( f_proxy(f_return(repo)), f_proxy(f_return()), f_proxy(f_return()) ) fts = mock_ubipop_runner._publish_out_repos() # we should publish only one repository with one distributor assert len(fts) == 1 assert [hist.repository.id for hist in fake_pulp.publish_history] == ["repo"]
def _delete_distributor(self, repo_id, distributor_id): with self._lock: repo_f = self.get_repository(repo_id) if repo_f.exception(): # Repo can't be found, let that exception propagate return repo_f repo = repo_f.result() new_distributors = [ dist for dist in repo.distributors if dist.id != distributor_id ] dist_found = new_distributors != repo.distributors if not dist_found: # Deleting something which already doesn't exist is fine return f_return([]) idx = self._repositories.index(repo) self._repositories[idx] = attr.evolve( repo, distributors=new_distributors) return f_return([ Task( id=self._next_task_id(), completed=True, succeeded=True, tags=[ "pulp:repository:%s" % repo_id, "pulp:repository_distributor:%s" % distributor_id, "pulp:action:remove_distributor", ], ) ])
def test_num_types(): xint = f_proxy(f_return(123)) xfloat = f_proxy(f_return(1.23)) assert float(xint) == 123.0 assert complex(xint) == complex(123) assert int(xfloat) == 1
def fixture_ubi_repo_set_no_debug(): ubi_binary = get_test_repo(id="ubi-foo-rpms", ubi_config_version="7") ubi_source = get_test_repo(id="ubi-foo-source", ubi_config_version="7") ubi_binary.get_source_repository = MagicMock() ubi_binary.get_source_repository.return_value = ubi_source ubi_binary.get_debug_repository = MagicMock() ubi_binary.get_debug_repository.return_value = f_proxy(f_return()) rhel_binary = get_test_repo(id="foo-rpms", ubi_config_version="7") rhel_source = get_test_repo(id="foo-source", ubi_config_version="7") rhel_binary.get_source_repository = MagicMock() rhel_binary.get_source_repository.return_value = rhel_source rhel_binary.get_debug_repository = MagicMock() rhel_binary.get_debug_repository.return_value = f_proxy(f_return()) yield UbiRepoSet( RepoSet( [rhel_binary], [rhel_source], f_proxy(f_return()), ), RepoSet( ubi_binary, ubi_source, f_proxy(f_return()), ), )
def test_get_modular_srpms_criteria(ubi_config): """Testing creation of criteria for srpms query""" matcher = ModularMatcher(None, ubi_config.modules) unit_1 = UbiUnit( RpmUnit( name="test", version="1.0", release="1", arch="x86_64", sourcerpm="test.x86_64.src.rpm", ), None, ) unit_2 = UbiUnit( RpmUnit( name="test-debug", version="1.0", release="1", arch="i386", sourcerpm="test-debug.i386.src.rpm", ), None, ) # we need to set up binary and debug rpms # the criteria are based on sourcerpm attr of those units matcher.binary_rpms = f_proxy(f_return(set([unit_1]))) matcher.debug_rpms = f_proxy(f_return(set([unit_2]))) criteria = matcher._get_srpms_criteria() # there should be 1 criteria created assert len(criteria) == 2 # it should be instance of Criteria for crit in criteria: assert isinstance(crit, Criteria)
def test_as_completed_with_timeout_reset(): # tests correctly yielded futures from as_completed_with_timeout_reset() futures = [f_return(1), f_return(2)] current = set() for ft in as_completed_with_timeout_reset(futures, timeout=1): value = ft.result() current.add(value) expected = set([1, 2]) assert current == expected
def test_bool(): xtrue = f_proxy(f_return(True)) xfalse = f_proxy(f_return(False)) xempty = f_proxy(f_return([])) # All of these count as truthy, as we want to allow testing if # a future was returned without blocking assert xtrue assert xfalse assert xempty
def fixture_mock_current_content(): rpm = get_rpm_unit( name="rpm_current", filename="rpm_current.rpm", version="1", release="0", arch="x86_64", src_repo_id="ubi-foo-rpms", ) srpm = get_srpm_unit( name="srpm_current", filename="srpm_current.src.rpm", version="1", release="0", arch="x86_64", src_repo_id="ubi-foo-source", ) debug_rpm = get_rpm_unit( name="debug_rpm_current", filename="debug_rpm_current.rpm", version="1", release="0", arch="x86_64", src_repo_id="ubi-foo-debug", ) modulemd_unit = get_modulemd_unit( name="md_current", stream="foo", version=1, context="bar", arch="x86_64", src_repo_id="ubi-foo-rpms", ) modulemd_defaults_unit = get_modulemd_defaults_unit( name="mdd_current", stream="rhel", profiles={"2.5": ["common"]}, repo_id="ubi-foo-rpms", src_repo_id="ubi-foo-rpms", ) binary_rpms = f_proxy(f_return([rpm])) debug_rpms = f_proxy(f_return([srpm])) source_rpms = f_proxy(f_return([debug_rpm])) modulemds = f_proxy(f_return([modulemd_unit])) modulemd_defaults = f_proxy(f_return([modulemd_defaults_unit])) repo_content = RepoContent( binary_rpms, debug_rpms, source_rpms, modulemds, modulemd_defaults ) yield repo_content
def test_flat_map_error(): map_in = f_return(0) # This one should fail... mapped = f_flat_map(map_in, div10) # Now map it through an error handler mapped = f_flat_map(mapped, error_fn=lambda ex: f_return(str(ex))) result = mapped.result() assert "division" in result
def test_len(): xlist = f_proxy(f_return([1, 2, 3])) xint = f_proxy(f_return(42)) xstr = f_proxy(f_return("hello")) assert len(xlist) == 3 assert len(xstr) == 5 with pytest.raises(TypeError) as exc: len(xint) assert "has no len" in str(exc.value)
def test_set_maintenance(client, requests_mocker): maintenance_report = { "last_updated": "2019-08-15T14:21:12Z", "last_updated_by": "pubtools.pulplib", "repos": { "repo1": { "message": "Maintenance Mode Enabled", "owner": "pubtools.pulplib", "started": "2019-08-15T14:21:12Z", } }, } requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/search/", [{ "json": [{ "id": "redhat-maintenance", "notes": { "_repo-type": "iso-repo" } }] }], ) report = MaintenanceReport._from_data(maintenance_report) with patch("pubtools.pulplib.FileRepository.upload_file") as mocked_upload: with patch("pubtools.pulplib.Repository.publish") as mocked_publish: upload_task = Task(id="upload-task", completed=True, succeeded=True) publish_task = [ Task(id="publish-task", completed=True, succeeded=True) ] mocked_upload.return_value = f_return(upload_task) mocked_publish.return_value = f_return(publish_task) # set_maintenance.result() should return whatever publish.result() returns assert client.set_maintenance(report).result() is publish_task # upload_file should be called with (file_obj, 'repos.json') args = mocked_upload.call_args report_file = args[0][0] report = MaintenanceReport()._from_data(json.loads(report_file.read())) assert len(report.entries) == 1 assert report.entries[0].repo_id == "repo1" assert report.last_updated_by == "pubtools.pulplib" # search repo, upload and publish should be called once each assert requests_mocker.call_count == 1 assert mocked_publish.call_count == 1 assert mocked_upload.call_count == 1
def test_sign(): xint = f_proxy(f_return(123)) xnegint = f_proxy(f_return(-123)) assert -xint == -123 assert +xint == 123 assert abs(xnegint) == 123 assert ~xint == -124 assert ~xnegint == 122 assert ~~xint == 123 assert ~~xnegint == -123
def test_gc_error(mock_logger): """logs error when repo delete task returns an error reponse""" repo = Repository( id="rhel-test-garbage-collect-7-days-old", created=_get_created(7), is_temporary=True, ) controller = _get_fake_controller(repo) gc = GarbageCollect() arg = ["", "--pulp-url", "http://some.url"] with patch("sys.argv", arg): with patch.object(controller.client, "_delete_repository") as repo_delete: with _patch_pulp_client(controller.client): repo_delete.return_value = f_return([ Task( id="12334", completed=True, succeeded=False, error_summary="Error occured", ) ]) gc.main() mock_logger.error.assert_any_call("Error occured")
def update_repository(self, repository): self._ensure_alive() with self._state.lock: existing_repo = None for candidate in self._state.repositories: if candidate.id == repository.id: existing_repo = candidate break else: return f_return_error( PulpException("repository not found: %s" % repository.id) ) # We've got a repo, now update it. update = {} for fld in existing_repo._mutable_note_fields(): update[fld.name] = getattr(repository, fld.name) updated_repo = attr.evolve(existing_repo, **update) self._state.repositories = [ repo for repo in self._state.repositories if repo.id != updated_repo.id ] + [updated_repo] return f_return()
def _do_upload_file(self, upload_id, file_obj, name="<unknown file>"): # pylint: disable=unused-argument # We keep track of uploaded content as we may need it at import time. buffer = six.BytesIO() self._uploads_pending[upload_id] = buffer is_file_obj = "close" in dir(file_obj) if not is_file_obj: file_obj = open(file_obj, "rb") def do_next_upload(checksum, size): while True: data = file_obj.read(1024 * 1024) if not data: break if isinstance(data, six.text_type): data = data.encode("utf-8") buffer.write(data) checksum.update(data) size += len(data) return f_return(UploadResult(checksum.hexdigest(), size)) out = f_flat_map(f_return(), lambda _: do_next_upload(hashlib.sha256(), 0)) out.add_done_callback(lambda _: file_obj.close()) return out
def update_content(self, unit): self._ensure_alive() if not unit.unit_id: raise ValueError("unit_id missing on call to update_content()") # The unit has to exist. existing_unit = None for candidate in self._all_units: if (candidate.content_type_id == unit.content_type_id and candidate.unit_id == unit.unit_id): existing_unit = candidate break else: return f_return_error( PulpException("unit not found: %s" % unit.unit_id)) # OK, we have a unit to update. Figure out which fields we can update. update = {} for fld in unit._usermeta_fields(): update[fld.name] = getattr(unit, fld.name) updated_unit = attr.evolve(existing_unit, **update) unit_key = units.make_unit_key(updated_unit) self._units_by_key[unit_key] = updated_unit return f_return()
def empty_future(value): if "add_done_callback" in dir(value): # It's a future, map it to None return f_map(value, lambda _: None) # It's not a future => operation has already completed, # return empty future to denote success return f_return()
def get_maintenance_report(self): if self._maintenance_report: report = MaintenanceReport._from_data( json.loads(self._maintenance_report)) else: report = MaintenanceReport() return f_proxy(f_return(report))
def _do_unassociate(self, repo_id, type_ids): repo_f = self.get_repository(repo_id) if repo_f.exception(): return repo_f current = self._repo_units.get(repo_id, set()) removed = set() kept = set() for unit in current: if type_ids is None or unit.content_type_id in type_ids: removed.add(unit) else: kept.add(unit) self._repo_units[repo_id] = kept task = Task( id=self._next_task_id(), repo_id=repo_id, completed=True, succeeded=True, units=removed, ) return f_return([task])
def _do_import( self, repo_id, upload_id, unit_type_id, unit_key, unit_metadata=None ): repo_f = self.get_repository(repo_id) if repo_f.exception(): # Repo can't be found, let that exception propagate return repo_f repo = repo_f.result() with self._state.lock: # Get the uploaded content we're about to import; though it's not # guaranteed to be present (e.g. erratum has no file). # If not present, we just use an empty BytesIO. upload_content = self._state.uploads_pending.pop(upload_id, six.BytesIO()) upload_content.seek(0) new_units = self._state.unitmaker.make_units( unit_type_id, unit_key, unit_metadata, upload_content, repo_id ) new_units = [ attr.evolve(u, repository_memberships=[repo.id]) for u in new_units ] self._state.insert_repo_units(repo_id, new_units) task = Task(id=self._state.next_task_id(), completed=True, succeeded=True) # upload_history is a deprecated field, data is maintained for iso only. if unit_type_id == "iso": self._state.upload_history.append( Upload(repo, [task], unit_key["name"], unit_key["checksum"]) ) return f_return([task])
def test_gc_error(mock_logger): """logs error when repo delete task returns an error reponse""" repo = { "id": "rhel-test-garbage-collect-7-days-old", "notes": { "pub_temp_repo": True, "created": _get_time_created(7) }, } controller = _get_fake_controller(repo) gc = GarbageCollect() arg = ["", "--pulp-url", "http://some.url", "--verbose"] with patch("sys.argv", arg): with patch.object(controller.client, "_delete_repository") as repo_delete: with patch("pubtools._pulp.task.PulpTask.pulp_client", controller.client): repo_delete.return_value = f_return([ Task( id="12334", completed=True, succeeded=False, error_summary="Error occured", ) ]) gc.main() mock_logger.error.assert_any_call("Error occured")
def _delete_repository(self, repo_id): with self._lock: found = False for idx, repo in enumerate(self._repositories): if repo.id == repo_id: found = True break if not found: # Deleting something which already doesn't exist is fine return f_return([]) self._repositories.pop(idx) # pylint: disable=undefined-loop-variable return f_return([ Task(id=self._next_task_id(), completed=True, succeeded=True) ])
def _get_current_content(self): """ Gather current content of output repos """ criteria = [Criteria.true()] current_modulemds = f_proxy( self._executor.submit(Matcher.search_modulemds, criteria, [self.repos.out_repos.rpm])) current_modulemd_defaults = f_proxy( self._executor.submit(Matcher.search_modulemd_defaults, criteria, [self.repos.out_repos.rpm])) current_rpms = f_proxy( self._executor.submit(Matcher.search_rpms, criteria, [self.repos.out_repos.rpm])) current_srpms = f_proxy( self._executor.submit(Matcher.search_srpms, criteria, [self.repos.out_repos.source])) if self.repos.out_repos.debug.result(): current_debug_rpms = f_proxy( self._executor.submit(Matcher.search_rpms, criteria, [self.repos.out_repos.debug])) else: current_debug_rpms = f_proxy(f_return([])) current_content = RepoContent( current_rpms, current_srpms, current_debug_rpms, current_modulemds, current_modulemd_defaults, ) return current_content
def _get_ubi_repo_sets(self, ubi_binary_cs): """ Searches for ubi repository triplet (binary rpm, srpm, debug) for one ubi config item and tries to determine their population sources (input repositories). Returns list UbiRepoSet objects that provides input and output repositories that are used for population process. """ rpm_repos = self.pulp_client.search_repository( Criteria.and_( Criteria.with_field("notes.content_set", ubi_binary_cs), Criteria.with_field("ubi_population", True), )) ubi_repo_sets = [] for out_rpm_repo in rpm_repos: out_source_repo = out_rpm_repo.get_source_repository() out_debug_repo = out_rpm_repo.get_debug_repository() in_rpm_repos = self._get_population_sources(out_rpm_repo) in_source_repos = self._get_population_sources(out_source_repo) in_debug_repos = self._get_population_sources(out_debug_repo) # we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency # that all objects in out|in_repos are futures out_repos = ( f_proxy(f_return(out_rpm_repo)), out_source_repo, out_debug_repo, ) in_repos = (in_rpm_repos, in_source_repos, in_debug_repos) ubi_repo_sets.append( UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos))) return ubi_repo_sets
def test_get_modular_rpms_criteria(ubi_config): """Test creation of criteria for rpms query""" matcher = ModularMatcher(None, ubi_config.modules) unit = UbiUnit( ModulemdUnit( name="test", stream="10", version=100, context="abcd", arch="x86_64", artifacts=[ "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src", "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64", ], ), None, ) matcher.modules = f_proxy(f_return(set([unit]))) criteria = matcher._get_modular_rpms_criteria() # there should be 1 criterium created - srpm is skipped assert len(criteria) == 1 # it should be instance of Criteria for crit in criteria: assert isinstance(crit, Criteria)
def _do_upload_file(self, upload_id, file_obj, name): def do_next_upload(checksum, size): data = file_obj.read(self._CHUNK_SIZE) if data: if isinstance(data, six.text_type): # if it's unicode, need to encode before calculate checksum data = data.encode("utf-8") checksum.update(data) return f_flat_map( self._do_upload(data, upload_id, size), lambda _: do_next_upload(checksum, size + len(data)), ) # nothing more to upload, return checksum and size return f_return((checksum.hexdigest(), size)) is_file_object = "close" in dir(file_obj) if not is_file_object: file_obj = open(file_obj, "rb") LOG.info("Uploading %s to Pulp", name) upload_f = f_flat_map(f_return(), lambda _: do_next_upload(hashlib.sha256(), 0)) if not is_file_object: upload_f.add_done_callback(lambda _: file_obj.close()) return upload_f
def _get_related_repository(self, repo_t): if not self._client: raise DetachedException() suffixes_mapping = { "binary": "/os", "debug": "/debug", "source": "/source/SRPMS", } regex = r"(/os|/source/SRPMS|/debug)$" def unpack_page(page): if len(page.data) != 1: return None return page.data[0] suffix = suffixes_mapping[repo_t] if str(self.relative_url).endswith(suffix): return f_proxy(f_return(self)) base_url = re.sub(regex, "", self.relative_url) relative_url = base_url + suffix criteria = Criteria.with_field("notes.relative_url", relative_url) page_f = self._client.search_repository(criteria) repo_f = f_map(page_f, unpack_page) return f_proxy(repo_f)
def handle_results(page): for unit in page.data: unit = UbiUnit(unit, repo.id) units.add(unit) if page.next: return f_flat_map(page.next, handle_results) return f_return(units)
def do_next_upload(checksum, size): data = file_obj.read(1024 * 1024) if data: checksum.update(data) size += len(data) return do_next_upload(checksum, size) return f_return((checksum.hexdigest(), size))
def test_modular_rpms_filenames_per_profiles_missing_profile(ubi_config): """Test getting filename from module artifacts, request for non-existing profile in modulemd""" matcher = ModularMatcher(None, ubi_config.modules) unit = UbiUnit( ModulemdUnit( name="fake_name", stream="fake_stream", version=100, context="abcd", arch="x86_64", artifacts=[ "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src", "perl-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64", "bash-7:10.5-el6.x86_64", "bash-devel-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64", ], profiles={"another": ["bash"]}, ), None, ) modules = f_proxy(f_return(set([unit]))) filenames = matcher._modular_rpms_filenames(modules) # all non-src pkgs are in result # this result is driven by ubi_config that force to use only profile called "test" # but we don't have this profile in the testing modulemd, so we take all non-src artifacts assert len(filenames) == 3 assert filenames == set([ "bash-10.5-el6.x86_64.rpm", "bash-devel-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm", "perl-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm", ])