def test_field_or_criteria(): """or is translated to a mongo fragment as expected.""" c1 = Criteria.with_field("f1", "v1") c2 = Criteria.with_field("f2", "v2") assert filters_for_criteria(Criteria.or_(c1, c2)) == { "$or": [{ "f1": { "$eq": "v1" } }, { "f2": { "$eq": "v2" } }] }
def test_search_content_type_id_in_or(client, requests_mocker): """Searching with a content_type_id within $or fails as unsupported""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client crit = Criteria.or_( Criteria.with_field("name", "hello.txt"), Criteria.with_field_in("content_type_id", ["rpm", "iso"]), ) with pytest.raises(ValueError) as e: repo.search_content(crit).result() assert "Can't serialize criteria for Pulp query; too complicated" in str( e.value)
def test_mixed_search(client, requests_mocker): """Searching with a criteria mixing several fields works correctly""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", json=[{ "metadata": { "_content_type_id": "iso", "name": "hello.txt", "size": 23, "checksum": "a" * 64, } }], ) crit = Criteria.and_( Criteria.with_field_in("content_type_id", ["rpm", "iso"]), Criteria.with_field("name", "hello.txt"), ) files = list(repo.search_content(crit)) assert files == [FileUnit(path="hello.txt", size=23, sha256sum="a" * 64)] history = requests_mocker.request_history # There should have been just one request assert len(history) == 1 request = history[0] body = request.json() # This should have been the request body. assert body == { "criteria": { "type_ids": ["rpm", "iso"], "skip": 0, "limit": 2000, "filters": { "unit": { "name": { "$eq": "hello.txt" } } }, } }
def test_search_mapped_field_less_than(): controller = FakeController() dist1 = Distributor( id="yum_distributor", type_id="yum_distributor", repo_id="repo1", last_publish=datetime.datetime(2019, 8, 23, 2, 5, 0, tzinfo=None), ) dist2 = Distributor( id="cdn_distributor", type_id="rpm_rsync_distributor", repo_id="repo1", last_publish=datetime.datetime(2019, 8, 27, 2, 5, 0, tzinfo=None), ) repo1 = Repository(id="repo1", distributors=(dist1, dist2)) controller.insert_repository(repo1) client = controller.client crit = Criteria.with_field( "last_publish", Matcher.less_than(datetime.datetime(2019, 8, 24, 0, 0, 0))) found = client.search_distributor(crit).result().data assert found == [dist1]
def test_search_content_invalid_content_type(populated_units, controller): """search_content_by_type with invalid content type""" with pytest.raises(PulpException): for x in controller.client.search_content( Criteria.with_field("content_type_id", "invalid") ).result(): pass
def _get_current_content(self): """ Gather current content of output repos """ criteria = [Criteria.true()] current_modulemds = f_proxy( self._executor.submit(Matcher.search_modulemds, criteria, [self.repos.out_repos.rpm])) current_modulemd_defaults = f_proxy( self._executor.submit(Matcher.search_modulemd_defaults, criteria, [self.repos.out_repos.rpm])) current_rpms = f_proxy( self._executor.submit(Matcher.search_rpms, criteria, [self.repos.out_repos.rpm])) current_srpms = f_proxy( self._executor.submit(Matcher.search_srpms, criteria, [self.repos.out_repos.source])) if self.repos.out_repos.debug.result(): current_debug_rpms = f_proxy( self._executor.submit(Matcher.search_rpms, criteria, [self.repos.out_repos.debug])) else: current_debug_rpms = f_proxy(f_return([])) current_content = RepoContent( current_rpms, current_srpms, current_debug_rpms, current_modulemds, current_modulemd_defaults, ) return current_content
def test_search_content_unsupported_operator(populated_repo): """search_content using unsupported operators on content_type_id raises""" with pytest.raises(ValueError) as e: populated_repo.search_content( Criteria.with_field("content_type_id", Matcher.regex("foobar"))) assert "unsupported expression for content_type_id" in str(e.value)
def test_no_update_on_same_cves(command_tester): with FakeFixCves() as fake_fix_cves: fake_pulp = fake_fix_cves.pulp_client_controller client = fake_pulp.client _setup_controller(fake_pulp) command_tester.test( fake_fix_cves.main, [ "test-fix-cves", "--pulp-url", "https://pulp.example.com", "--advisory", "RHSA-1234:56", "--cves", "CVE-123", ], ) erratum = list( client.search_content(Criteria.with_field("id", "RHSA-1234:56"))) assert len(erratum) == 1 erratum = erratum[0] assert erratum.id == "RHSA-1234:56" # no updates. erratum version not bumped. assert erratum.version == "2" cves = sorted([ref for ref in erratum.references if ref.type == "cve"]) # same CVE exists assert len(cves) == 1 assert cves[0].id == "CVE-123"
def test_fix_cves(command_tester): with FakeFixCves() as fake_fix_cves: fake_pulp = fake_fix_cves.pulp_client_controller client = fake_pulp.client _setup_controller(fake_pulp) command_tester.test( fake_fix_cves.main, [ "test-fix-cves", "--pulp-url", "https://pulp.example.com", "--advisory", "RHSA-1234:56", "--cves", "CVE-987,CVE-456", ], ) updated_erratum = list( client.search_content(Criteria.with_field("id", "RHSA-1234:56"))) # only single erratum exists on Pulp assert len(updated_erratum) == 1 updated_erratum = updated_erratum[0] assert updated_erratum.id == "RHSA-1234:56" # erratum version bumped assert updated_erratum.version == "3" cves = sorted( [ref for ref in updated_erratum.references if ref.type == "cve"]) # only new CVEs in the erratum assert len(cves) == 2 assert cves[0].id == "CVE-456" assert cves[1].id == "CVE-987"
def test_search_content_by_unit_type(populated_repo): """search_content on unit_type returns only units of that type""" crit = Criteria.with_unit_type(ModulemdUnit) units = list(populated_repo.search_content(crit)) assert sorted(units) == [ ModulemdUnit( unit_id="23a7711a-8133-2876-37eb-dcd9e87a1613", name="module1", stream="s1", version=1234, context="a1b2", arch="x86_64", repository_memberships=["repo1"], ), ModulemdUnit( unit_id="e6f4590b-9a16-4106-cf6a-659eb4862b21", name="module2", stream="s2", version=1234, context="a1b2", arch="x86_64", repository_memberships=["repo1"], ), ]
def search_content(self, criteria=None): self._ensure_alive() criteria = criteria or Criteria.true() out = [] # Pass the criteria through the code used by the real client to build # up the Pulp query. We don't actually *use* the resulting query since # we're not accessing a real Pulp server. The point is to ensure the # same validation and error behavior as used by the real client also # applies to the fake. prepared_search = search_for_criteria(criteria, Unit) available_type_ids = set(self._type_ids) missing_type_ids = set(prepared_search.type_ids or []) - available_type_ids if missing_type_ids: return f_return_error( PulpException( "following type ids are not supported on the server: %s" % ",".join(missing_type_ids))) for unit in self._all_units: if (prepared_search.type_ids and unit.content_type_id not in prepared_search.type_ids): continue if match_object(criteria, unit): out.append(unit) # callers should not make any assumption about the order of returned # values. Encourage that by returning output in unpredictable order random.shuffle(out) return self._prepare_pages(out)
def test_search_paginates(): controller = FakeController() repos = [] for i in range(0, 1000): repo = Repository(id="repo-%s" % i) repos.append(repo) controller.insert_repository(repo) client = controller.client crit = Criteria.true() page = client.search_repository(crit) found_repos = list(page) page_count = 1 while page.next: page_count += 1 page = page.next.result() # There should have been several pages (it is not defined exactly # what page size the fake client uses, but it should be relatively # small to enforce that clients think about pagination) assert page_count >= 10 # All repos should have been found assert sorted(found_repos) == sorted(repos)
def _search_repo_units(self, repo_id, criteria): criteria = criteria or Criteria.true() # Pass the criteria through the same handling as used by the real client # for serialization, to ensure we reject criteria also rejected by real client # and also accumulate unit_fields. prepared_search = search_for_criteria(criteria, Unit) repo_f = self.get_repository(repo_id) if repo_f.exception(): return repo_f with self._state.lock: repo_units = self._state.repo_units(repo_id) out = [] try: for unit in repo_units: if match_object(criteria, unit): unit = units.with_filtered_fields(unit, prepared_search.unit_fields) out.append(unit) except Exception as ex: # pylint: disable=broad-except return f_return_error(ex) random.shuffle(out) return self._prepare_pages(out)
def check_repos(self): self._sanitize_repo_ids_args() repo_ids = self.args.repo_ids found_repo_ids = [] out = [] # apply the filters and get repo_ids repo_ids = self._filter_repos(repo_ids) # get repo objects for the repo_ids searched_repos = self.pulp_client.search_repository(Criteria.with_id(repo_ids)) for repo in searched_repos: out.append(repo) found_repo_ids.append(repo.id) # Bail out if user requested repos which don't exist # or there are no repos returned to publish missing = set(repo_ids) - set(found_repo_ids) missing = sorted(list(missing)) if missing: self.fail("Requested repo(s) don't exist: %s", ", ".join(missing)) if not out: self.fail("No repo(s) found to publish") return out
def adjust_maintenance_report(self, report): """Remove entries from maintenance report by repo ids or repo url regex or both""" to_remove = [] existed_repo_ids = [entry.repo_id for entry in report.entries] if self.args.repo_url_regex: # search all repos with id existed in the report existed_repos = self.pulp_client.search_repository( Criteria.with_id(existed_repo_ids)).result() for repo in existed_repos: if repo.relative_url and re.search(self.args.repo_url_regex, repo.relative_url): to_remove.append(repo.id) if self.args.repo_ids: for repo_id in self.args.repo_ids: if repo_id in existed_repo_ids: to_remove.append(repo_id) else: LOG.warning("Repository %s is not in maintenance mode", repo_id) if to_remove: LOG.info( "Following repositories will be removed from maintenance mode:" ) for repo_id in to_remove: LOG.info(" - %s", repo_id) report = report.remove(to_remove, owner=self.args.owner) return report
def test_field_less_than_criteria(): """with_field with less_than is translated as expected for date and non-date types """ publish_date = datetime.datetime(2019, 8, 27, 0, 0, 0) c1 = Criteria.with_field("num_field", Matcher.less_than(5)) c2 = Criteria.with_field("date_field", Matcher.less_than(publish_date)) assert filters_for_criteria(c1) == {"num_field": {"$lt": 5}} assert filters_for_criteria(c2) == { "date_field": { "$lt": { "$date": "2019-08-27T00:00:00Z" } } }
def test_search_distributor_with_relative_url(): controller = FakeController() dist1 = Distributor( id="yum_distributor", type_id="yum_distributor", repo_id="repo1", relative_url="relative/path", ) dist2 = Distributor( id="cdn_distributor", type_id="rpm_rsync_distributor", repo_id="repo1", relative_url="relative/path", ) repo1 = Repository(id="repo1", distributors=(dist1, dist2)) dist3 = Distributor( id="yum_distributor", type_id="yum_distributor", repo_id="repo2", relative_url="another/path", ) repo2 = Repository(id="repo2", distributors=(dist3, )) controller.insert_repository(repo1) controller.insert_repository(repo2) client = controller.client crit = Criteria.with_field("relative_url", Matcher.regex("relative/path")) found = client.search_distributor(crit).result().data assert sorted(found) == [dist2, dist1]
def test_search_and(): controller = FakeController() repo1 = Repository(id="repo1") repo2 = Repository(id="repo2", created=datetime.datetime.utcnow()) repo3 = Repository(id="repo3", created=datetime.datetime.utcnow()) controller.insert_repository(repo1) controller.insert_repository(repo2) controller.insert_repository(repo3) client = controller.client crit = Criteria.and_(Criteria.with_field("notes.created", Criteria.exists), Criteria.with_id("repo2")) found = client.search_repository(crit).data assert sorted(found) == [repo2]
def _filtered_repo_distributors(self): published_before = self.args.published_before url_regex = self.args.repo_url_regex # define the criteria on available filters crit = [Criteria.true()] if published_before: crit.append( Criteria.with_field("last_publish", Matcher.less_than(published_before)) ) if url_regex: crit.append( Criteria.with_field("relative_url", Matcher.regex(url_regex.pattern)) ) crit = Criteria.and_(*crit) return self.pulp_client.search_distributor(crit)
def test_eng_product_in(): """eng_product is mapped correctly""" crit = Criteria.with_field_in("eng_product_id", [12, 34, 56]) assert filters_for_criteria(crit, Repository) == { "notes.eng_product": { "$in": ["12", "34", "56"] } }
def test_signing_keys(): """signing_keys are mapped correctly""" crit = Criteria.with_field("signing_keys", ["abc", "def", "123"]) assert filters_for_criteria(crit, Repository) == { "notes.signatures": { "$eq": "abc,def,123" } }
def test_type(): """type is mapped correctly""" crit = Criteria.with_field("type", Matcher.regex("foobar")) assert filters_for_criteria(crit, Repository) == { "notes._repo-type": { "$regex": "foobar" } }
def test_search_or(): controller = FakeController() repo1 = Repository(id="repo1") repo2 = Repository(id="repo2") repo3 = Repository(id="repo3") controller.insert_repository(repo1) controller.insert_repository(repo2) controller.insert_repository(repo3) client = controller.client crit = Criteria.or_(Criteria.with_id("repo3"), Criteria.with_field("id", Matcher.equals("repo1"))) found = client.search_repository(crit).data assert sorted(found) == [repo1, repo3]
def test_field_exists_criteria(): """with_field using 'exists' is translated to a mongo fragment as expected.""" assert filters_for_criteria( Criteria.with_field("some.field", Criteria.exists)) == { "some.field": { "$exists": True } }
def test_field_eq_criteria(): """with_field is translated to a mongo fragment as expected.""" assert filters_for_criteria(Criteria.with_field("some.field", "someval")) == { "some.field": { "$eq": "someval" } }
def test_field_in_criteria(): """with_field_in is translated to a mongo fragment as expected.""" assert filters_for_criteria( Criteria.with_field_in("some.field", ["val1", "val2"])) == { "some.field": { "$in": ["val1", "val2"] } }
def test_is_temporary(): """is_temporary is mapped correctly""" crit = Criteria.with_field("is_temporary", True) assert filters_for_criteria(crit, Repository) == { "notes.pub_temp_repo": { "$eq": True } }
def run_with_client(self, client): # At the time we run, it is the case that all items exist with the desired # state, in the desired repos. Now we need to publish affected repos. # # This is also a synchronization point. The idea is that publishing repo # makes content available, and there may be dependencies between the bits # of content we've handled, so we should ensure *all* of them are in correct # repos before we start publish of *any* repos to increase the chance that # all of them land at once. # # TODO: once exodus is live, consider refactoring this to not be a # synchronization point (or make it optional?) as the above motivation goes # away - the CDN origin supports near-atomic update. all_repo_ids = set() set_cdn_published = set() all_items = [] for item in self.iter_input(): all_repo_ids.update(item.publish_pulp_repos) # any unit which supports cdn_published but hasn't had it set yet should # have it set once the publish completes. unit = item.pulp_unit if hasattr(unit, "cdn_published") and unit.cdn_published is None: set_cdn_published.add(unit) all_items.append(item) # Locate all the repos for publish. repo_fs = client.search_repository( Criteria.with_id(sorted(all_repo_ids))) # Start publishing them, including cache flushes. publish_fs = self.publish_with_cache_flush(repo_fs, set_cdn_published, client) # Then wait for publishes to finish. for f in publish_fs: f.result() # At this stage we consider all items to be fully "pushed". pushed_items = [ attr.evolve(item, pushsource_item=attr.evolve(item.pushsource_item, state="PUSHED")) for item in all_items ] self.update_push_items(pushed_items) # Mark as done for accurate progress logs. # Note we don't keep track of exactly which items got published through each # repo, so this will simply show that everything moved from in progress to done # at once. for _ in pushed_items: self.in_queue.task_done() # And we know nothing more happens to the push items, so we can tell # collector that we're finished. self.update_push_items([self.FINISHED])
def test_search_content_by_type(populated_repo): """search_content for particular type returns matching content""" crit = Criteria.with_field("content_type_id", "rpm") units = list(populated_repo.search_content(crit)) assert sorted(units) == [ RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"), ]
def test_search_content_mixed_fields(populated_repo): """search_content crossing multiple fields and types returns matching units""" crit = Criteria.and_( Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]), Criteria.with_field_in("name", ["bash", "module1"]), ) units = list(populated_repo.search_content(crit)) # Note: sorting different types not natively supported, hence sorting by repr assert sorted(units, key=repr) == [ ModulemdUnit(name="module1", stream="s1", version=1234, context="a1b2", arch="x86_64"), RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), ]