def _rpm_criteria(self, filename, signing_keys=None, sha256sum=None): if signing_keys: return Criteria.and_( Criteria.with_field("filename", filename), Criteria.with_field("signing_key", Matcher.in_(signing_keys)), ) if sha256sum: return Criteria.and_( Criteria.with_field("filename", filename), Criteria.with_field("sha256sum", sha256sum), ) return Criteria.with_field("filename", filename)
def clean_all_rpm_content(self): # Clear out old all-rpm-content LOG.info("Start old all-rpm-content deletion") arc_threshold = self.args.arc_threshold criteria = Criteria.and_( Criteria.with_unit_type(RpmUnit), Criteria.with_field( "cdn_published", Matcher.less_than(datetime.utcnow() - timedelta(days=arc_threshold)), ), ) clean_repos = list( self.pulp_client.search_repository( Criteria.with_field("id", "all-rpm-content"))) if not clean_repos: LOG.info("No repos found for cleaning.") return arc_repo = clean_repos[0] deleted_arc = list(arc_repo.remove_content(criteria=criteria)) deleted_content = [] for task in deleted_arc: if task.repo_id == "all-rpm-content": for unit in task.units: LOG.info("Old all-rpm-content deleted: %s", unit.name) deleted_content.append(unit) if not deleted_content: LOG.info("No all-rpm-content found older than %s", arc_threshold)
def test_field_and_criteria(): """and is translated to a mongo fragment as expected.""" c1 = Criteria.with_field("f1", "v1") c2 = Criteria.with_field("f2", "v2") assert filters_for_criteria(Criteria.and_(c1, c2)) == { "$and": [{"f1": {"$eq": "v1"}}, {"f2": {"$eq": "v2"}}] }
def _get_ubi_repo_sets(self, ubi_binary_cs): """ Searches for ubi repository triplet (binary rpm, srpm, debug) for one ubi config item and tries to determine their population sources (input repositories). Returns list UbiRepoSet objects that provides input and output repositories that are used for population process. """ rpm_repos = self.pulp_client.search_repository( Criteria.and_( Criteria.with_field("notes.content_set", ubi_binary_cs), Criteria.with_field("ubi_population", True), )) ubi_repo_sets = [] for out_rpm_repo in rpm_repos: out_source_repo = out_rpm_repo.get_source_repository() out_debug_repo = out_rpm_repo.get_debug_repository() in_rpm_repos = self._get_population_sources(out_rpm_repo) in_source_repos = self._get_population_sources(out_source_repo) in_debug_repos = self._get_population_sources(out_debug_repo) # we need to apply f_proxy(f_return()) for out_rpm_repo for keeping consistency # that all objects in out|in_repos are futures out_repos = ( f_proxy(f_return(out_rpm_repo)), out_source_repo, out_debug_repo, ) in_repos = (in_rpm_repos, in_source_repos, in_debug_repos) ubi_repo_sets.append( UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos))) return ubi_repo_sets
def run(self): LOG.debug("Garbage collection begins") criteria = Criteria.and_( Criteria.with_field("notes.created", Matcher.exists()), Criteria.with_field("notes.pub_temp_repo", True), ) # fetch repos for the criteria repos = self.pulp_client.search_repository(criteria).result() LOG.debug("repos fetched") gc_threshold = self.args.gc_threshold deleted_repos = [] # initiate deletion task for the repos for repo in repos.as_iter(): repo_age = datetime.utcnow() - repo.created if repo_age > timedelta(days=gc_threshold): LOG.info("Deleting %s (created on %s)", repo.id, repo.created) deleted_repos.append(repo.delete()) if not deleted_repos: LOG.info("No repo(s) found older than %s day(s)", gc_threshold) # log for error during deletion for task in deleted_repos: out = task.result()[0] if out.error_details or out.error_summary: LOG.error(out.error_details or out.error_summary) LOG.info("Temporary repo(s) deletion completed")
def _search_units( cls, repo, criteria_list, content_type_id, batch_size_override=None ): """ Search for units of one content type associated with given repository by criteria. """ units = set() batch_size = batch_size_override or BATCH_SIZE def handle_results(page): for unit in page.data: unit = UbiUnit(unit, repo.id) units.add(unit) if page.next: return f_flat_map(page.next, handle_results) return f_return(units) criteria_split = [] for start in range(0, len(criteria_list), batch_size): criteria_split.append(criteria_list[start : start + batch_size]) fts = [] for criteria_batch in criteria_split: _criteria = Criteria.and_( Criteria.with_field("content_type_id", content_type_id), Criteria.or_(*criteria_batch), ) page_f = repo.search_content(_criteria) handled_f = f_flat_map(page_f, handle_results) fts.append(handled_f) return f_flat_map(f_sequence(fts), flatten_list_of_sets)
def test_search_content_mixed_fields(populated_repo): """search_content crossing multiple fields and types returns matching units""" crit = Criteria.and_( Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]), Criteria.with_field_in("name", ["bash", "module1"]), ) units = list(populated_repo.search_content(crit)) # Note: sorting different types not natively supported, hence sorting by repr assert sorted(units, key=repr) == [ ModulemdUnit( unit_id="23a7711a-8133-2876-37eb-dcd9e87a1613", name="module1", stream="s1", version=1234, context="a1b2", arch="x86_64", repository_memberships=["repo1"], ), RpmUnit( unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd", name="bash", version="4.0", release="1", arch="x86_64", repository_memberships=["repo1"], ), ]
def get_advisories(self, advisory_ids): criteria = criteria = Criteria.and_( Criteria.with_unit_type(ErratumUnit), Criteria.with_id(advisory_ids) ) adv_f = self.search_content(criteria) advisories = [a for a in adv_f.result()] if not advisories: self.fail("Advisory(ies) not found: %s", ", ".join(sorted(advisory_ids))) return advisories
def test_remove_fail_without_type_id(fast_poller, client): """Remove fails when a critria is provided without unit type""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client criteria = Criteria.and_( Criteria.with_field("filename", "some.rpm"), Criteria.with_field("signing_key", Matcher.in_(["key123"])), ) with pytest.raises(ValueError): repo.remove_content(criteria=criteria)
def test_search_null_and(): """Search with an empty AND gives an error.""" controller = FakeController() repo1 = Repository(id="repo1") controller.insert_repository(repo1) client = controller.client crit = Criteria.and_() assert "Invalid AND in search query" in str( client.search_repository(crit).exception())
def test_copy_with_criteria(fast_poller, requests_mocker, client): """Copy with criteria succeeds, and serializes criteria correctly.""" src = Repository(id="src-repo") dest = Repository(id="dest-repo") src.__dict__["_client"] = client dest.__dict__["_client"] = client requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/", [{"json": {"spawned_tasks": [{"task_id": "task1"}, {"task_id": "task2"}]}}], ) requests_mocker.post( "https://pulp.example.com/pulp/api/v2/tasks/search/", [ { "json": [ {"task_id": "task1", "state": "finished"}, {"task_id": "task2", "state": "skipped"}, ] } ], ) crit = Criteria.and_( Criteria.with_unit_type(RpmUnit), Criteria.with_field("name", Matcher.in_(["bash", "glibc"])), ) # Copy should succeed, and return the tasks (in this case with no matches) assert sorted(client.copy_content(src, dest, crit), key=lambda t: t.id) == [ Task(id="task1", completed=True, succeeded=True), Task(id="task2", completed=True, succeeded=True), ] hist = requests_mocker.request_history # First request should have been the associate. assert ( hist[0].url == "https://pulp.example.com/pulp/api/v2/repositories/dest-repo/actions/associate/" ) # It should have encoded our criteria object as needed by the Pulp API. assert hist[0].json() == { "criteria": { "filters": {"unit": {"name": {"$in": ["bash", "glibc"]}}}, "type_ids": ["rpm", "srpm"], }, "source_repo_id": "src-repo", }
def test_mixed_search(client, requests_mocker): """Searching with a criteria mixing several fields works correctly""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", json=[{ "metadata": { "_content_type_id": "iso", "name": "hello.txt", "size": 23, "checksum": "a" * 64, } }], ) crit = Criteria.and_( Criteria.with_field_in("content_type_id", ["rpm", "iso"]), Criteria.with_field("name", "hello.txt"), ) files = list(repo.search_content(crit)) assert files == [FileUnit(path="hello.txt", size=23, sha256sum="a" * 64)] history = requests_mocker.request_history # There should have been just one request assert len(history) == 1 request = history[0] body = request.json() # This should have been the request body. assert body == { "criteria": { "type_ids": ["rpm", "iso"], "skip": 0, "limit": 2000, "filters": { "unit": { "name": { "$eq": "hello.txt" } } }, } }
def test_stringify_complex_criteria(): crit = Criteria.and_( Criteria.with_field("must-exist", Matcher.exists()), Criteria.with_field("foo", Matcher.equals("bar")), Criteria.true(), Criteria.or_( Criteria.with_field("foo", Matcher.regex("quux")), Criteria.with_field("other", Matcher.in_(["x", "y", "z"])), Criteria.with_field("num", Matcher.less_than(9000)), ), Criteria.with_unit_type(FileUnit), ) assert (str(crit) == "((must-exist EXISTS) AND foo=='bar' AND TRUE " "AND (foo=~/quux/ OR (other IN ['x', 'y', 'z']) OR num<9000) " "AND (content_type_id IN ['iso']))")
def test_search_null_and(): """Search with an empty AND gives an error.""" controller = FakeController() dist1 = Distributor(id="yum_distributor", type_id="yum_distributor", repo_id="repo1") repo1 = Repository(id="repo1", distributors=[dist1]) controller.insert_repository(repo1) client = controller.client crit = Criteria.and_() assert "Invalid AND in search query" in str( client.search_repository(crit).exception()) assert "Invalid AND in search query" in str( client.search_distributor(crit).exception())
def _filtered_repo_distributors(self): published_before = self.args.published_before url_regex = self.args.repo_url_regex # define the criteria on available filters crit = [Criteria.true()] if published_before: crit.append( Criteria.with_field("last_publish", Matcher.less_than(published_before)) ) if url_regex: crit.append( Criteria.with_field("relative_url", Matcher.regex(url_regex.pattern)) ) crit = Criteria.and_(*crit) return self.pulp_client.search_distributor(crit)
def test_search_and(): controller = FakeController() repo1 = Repository(id="repo1") repo2 = Repository(id="repo2", created=datetime.datetime.utcnow()) repo3 = Repository(id="repo3", created=datetime.datetime.utcnow()) controller.insert_repository(repo1) controller.insert_repository(repo2) controller.insert_repository(repo3) client = controller.client crit = Criteria.and_(Criteria.with_field("notes.created", Criteria.exists), Criteria.with_id("repo2")) found = client.search_repository(crit).data assert sorted(found) == [repo2]
def test_remove_fails_without_type_id(): """repo.remove_content() fails when a criteria is provided without unit type""" controller = FakeController() client = controller.client rpm_units = [ RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"), ] repo = YumRepository(id="repo1") controller.insert_repository(repo) controller.insert_units(repo, rpm_units) criteria = Criteria.and_(Criteria.with_field("name", "bash")) with pytest.raises(ValueError): client.get_repository("repo1").remove_content(criteria=criteria)
def test_search_content_mixed_fields(populated_repo): """search_content crossing multiple fields and types returns matching units""" crit = Criteria.and_( Criteria.with_field_in("content_type_id", ["rpm", "modulemd"]), Criteria.with_field_in("name", ["bash", "module1"]), ) units = list(populated_repo.search_content(crit)) # Note: sorting different types not natively supported, hence sorting by repr assert sorted(units, key=repr) == [ ModulemdUnit(name="module1", stream="s1", version=1234, context="a1b2", arch="x86_64"), RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), ]
def _create_or_criteria(self, fields, values): # fields - list/tuple of fields [field1, field2] # values - list of tuples [(field1 value, field2 value), ...] # creates criteria for pulp query in a following way # one tuple in values uses AND logic # each criteria for one tuple are agregated by to or_criteria list or_criteria = [] for val_tuple in values: inner_and_criteria = [] if len(val_tuple) != len(fields): raise ValueError for index, field in enumerate(fields): inner_and_criteria.append(Criteria.with_field(field, val_tuple[index])) or_criteria.append(Criteria.and_(*inner_and_criteria)) return or_criteria
def test_search_fields(client, requests_mocker): """Searching with limited fields works correctly""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", json=[], ) crit = Criteria.and_( Criteria.with_unit_type(FileUnit, unit_fields=["sha256sum"]), Criteria.with_field("name", "hello.txt"), ) repo.search_content(crit).result() history = requests_mocker.request_history # There should have been just one request assert len(history) == 1 request = history[0] body = request.json() # This should have been the request body. assert body == { "criteria": { "type_ids": ["iso"], "skip": 0, "limit": 2000, "filters": { "unit": { "name": { "$eq": "hello.txt" } } }, "fields": { "unit": ["checksum", "name", "size"] }, } }
def find_related(self, repo): """Start searching for the repos which should have product_versions updated after a productid has been uploaded to 'repo'. Returns None, but influences the return value of a later call to all_results. """ arch = repo.arch eng_product_id = repo.eng_product_id platform_full_version = repo.platform_full_version search_key = (arch, eng_product_id, platform_full_version) with self.lock: self.repos.append(repo) if not self.searches.get(search_key): search_f = self.client.search_repository( Criteria.and_( Criteria.with_field("arch", arch), Criteria.with_field("eng_product_id", eng_product_id), Criteria.with_field("platform_full_version", platform_full_version), )) self.searches[search_key] = search_f
def test_copy_content_with_criteria(controller): """copy_content can filter copied units by field values""" src = YumRepository(id="src-repo") dest = YumRepository(id="dest-repo") controller.insert_repository(src) controller.insert_repository(dest) src_units = [ RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), RpmUnit(name="bash", version="4.0", release="2", arch="x86_64"), RpmUnit(name="bash", version="4.1", release="3", arch="x86_64"), RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"), ] controller.insert_units(src, src_units) client = controller.client # Repos are initially detached, re-fetch them via client src = client.get_repository(src.id).result() dest = client.get_repository(dest.id).result() # This is what we want to copy... crit = Criteria.and_( Criteria.with_field("name", "bash"), Criteria.with_field("release", Matcher.in_(["1", "3"])), ) # Copy should succeed copy_tasks = list( client.copy_content(src, dest, crit, options=CopyOptions(require_signed_rpms=False))) # It should have copied only those units matching the criteria units = sum([t.units for t in copy_tasks], []) assert sorted(units, key=repr) == [ RpmUnit(name="bash", version="4.0", release="1", arch="x86_64", epoch="0"), RpmUnit(name="bash", version="4.1", release="3", arch="x86_64", epoch="0"), ] # The copy should also impact subsequent content searches. dest_units = list(dest.search_content()) assert sorted(dest_units, key=repr) == [ RpmUnit( unit_id="e3e70682-c209-4cac-629f-6fbed82c07cd", name="bash", version="4.0", release="1", arch="x86_64", epoch="0", repository_memberships=["src-repo", "dest-repo"], ), RpmUnit( unit_id="d4713d60-c8a7-0639-eb11-67b367a9c378", name="bash", version="4.1", release="3", arch="x86_64", epoch="0", repository_memberships=["src-repo", "dest-repo"], ), ]
def criteria(self): return Criteria.and_( Criteria.with_field("sha256sum", self.pushsource_item.sha256sum), Criteria.with_field("path", self.pushsource_item.name), )
def test_remove_filtered_content(): """repo.remove_content() succeeds and removes expected units inserted via controller.""" controller = FakeController() client = controller.client rpm_units = [ RpmUnit(name="bash", version="4.0", release="1", arch="x86_64"), RpmUnit(name="glibc", version="5.0", release="1", arch="x86_64"), ] modulemd_units = [ ModulemdUnit(name="module1", stream="s1", version=1234, context="a1b2", arch="x86_64"), ModulemdUnit(name="module1", stream="s1", version=1235, context="a1b2", arch="x86_64"), ] units = rpm_units + modulemd_units repo = YumRepository(id="repo1") controller.insert_repository(repo) controller.insert_units(repo, units) criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit), Criteria.with_field("name", "bash")) remove_rpms = client.get_repository("repo1").remove_content( criteria=criteria) assert len(remove_rpms) == 1 task = remove_rpms[0] # It should have completed successfully assert task.completed assert task.succeeded # It should have removed the specific rpm assert len(task.units) == 1 assert task.units[0] == sorted(rpm_units)[0] # Try removing a module with incorrect type_ids criteria = Criteria.and_(Criteria.with_unit_type(RpmUnit), Criteria.with_field("name", "module1")) remove_rpms = client.get_repository("repo1").remove_content( criteria=criteria) assert len(remove_rpms) == 1 task = remove_rpms[0] # It should have completed successfully assert task.completed assert task.succeeded # Nothing's removed as criteria isn't fulfilled assert len(task.units) == 0 # Removing module with correct type_ids criteria = Criteria.and_(Criteria.with_unit_type(ModulemdUnit), Criteria.with_field("name", "module1")) remove_rpms = client.get_repository("repo1").remove_content( criteria=criteria) assert len(remove_rpms) == 1 task = remove_rpms[0] # It should have completed successfully assert task.completed assert task.succeeded # It should have removed both the modules as they # match the criteria assert len(task.units) == 2 assert sorted(task.units) == sorted(modulemd_units)
def get_erratum_from_pulp(self, advisory_id): crit = Criteria.and_(Criteria.with_unit_type(ErratumUnit), Criteria.with_id(advisory_id)) return self.pulp_client.search_content(criteria=crit).result()
def test_remove_with_criteria(fast_poller, requests_mocker, client): """Remove succeeds when given a critria/filter for removal""" repo = Repository(id="some-repo") repo.__dict__["_client"] = client requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/", [ { "json": { "spawned_tasks": [{ "task_id": "task1" }] } }, { "json": { "spawned_tasks": [{ "task_id": "task2" }] } }, ], ) requests_mocker.post( "https://pulp.example.com/pulp/api/v2/tasks/search/", [ { "json": [{ "task_id": "task1", "state": "finished" }] }, { "json": [{ "task_id": "task2", "state": "finished" }] }, ], ) criteria = Criteria.and_( Criteria.with_unit_type(RpmUnit), Criteria.with_field("filename", "some.rpm"), Criteria.with_field("signing_key", Matcher.in_(["key123"])), ) assert repo.remove_content(criteria=criteria).result() == [ Task(id="task1", completed=True, succeeded=True) ] # It should have passed the criteria to Pulp req = requests_mocker.request_history assert ( req[0].url == "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/" ) assert req[0].json() == { "criteria": { "filters": { "unit": { "$and": [ { "filename": { "$eq": "some.rpm" } }, { "signing_key": { "$in": ["key123"] } }, ] } }, "type_ids": ["rpm", "srpm"], } } # Providing both criteria and type_ids assert repo.remove_content(criteria=criteria, type_ids=["type1", "type2"]).result() == [ Task(id="task2", completed=True, succeeded=True) ] # It should have passed only the critera to Pulp and ignore type_ids as kwarg req = requests_mocker.request_history assert ( req[0].url == "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/unassociate/" ) assert req[0].json() == { "criteria": { "filters": { "unit": { "$and": [ { "filename": { "$eq": "some.rpm" } }, { "signing_key": { "$in": ["key123"] } }, ] } }, "type_ids": ["rpm", "srpm"], } }
def test_search_null_and(populated_repo): """Search with an empty AND gives an error.""" crit = Criteria.and_() assert "Invalid AND in search query" in str( populated_repo.search_content(crit).exception())
def _module_criteria(self, module_name): crit = [] mod_nsvca_dict = self._get_nsvca_dict(module_name) for m_part, value in mod_nsvca_dict.items(): crit.append(Criteria.with_field(m_part, value)) return Criteria.and_(*crit)
def test_stringify_simplify_and(): assert str(Criteria.and_(Criteria.with_field("x", 123))) == "x==123"
def unit_criteria(self, unit_type, partial_crit): criteria = Criteria.and_( Criteria.with_unit_type(unit_type), Criteria.or_(*partial_crit) ) return criteria