def test_search_stops_paginate(requests_mocker): """Paginated search will stop issuing requests if pages are no longer referenced.""" client = Client("https://pulp.example.com/") client._PAGE_SIZE = 10 responses = [] current_response = [] for i in range(0, 997): repo_id = "repo-%s" % i current_response.append({"id": repo_id}) if len(current_response) == client._PAGE_SIZE: responses.append({"json": current_response}) current_response = [] responses.append({"json": current_response}) # Inject a failing response for stable behavior from test: # # The expected behavior is that, after the first two requests, # pagination is cancelled since we no longer hold onto a page. # But that cancellation is racing with the scheduling of the # request, and there is generally no way to ensure our cancel # happens before the request thread starts the request, so we # can't be sure about how many requests we'd do. # # However, if we insert a failure after the first two pages, # then the client will need to retry a request (with some delay). # It should always be possible to cancel the request during that # retry delay, making the test stable. responses.insert(2, {"status_code": 500}) requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/search/", responses) # Imagine we have some function which processes two pages of results and # then stops def do_something_first_two_pages(repos_f): page = repos_f.result() return len(page.data) + len(page.next.result().data) two_pages_len = do_something_first_two_pages(client.search_repository()) # It should have seen the first two pages assert two_pages_len == 20 # Now wait a bit to ensure any futures under the hood have time to finish up time.sleep(2.0) # It should have done only two or three requests # (2 vs 3 depends on scheduling). Because we threw away all references to # the search result, it stopped searching rather than continuing through # all pages. assert requests_mocker.call_count in (2, 3)
def test_can_construct_with_throttle_arg(requests_mocker): """ A client instance can be constructed with task_throttle arg that is passed to Client._task_executor """ throttle_count = 42 client = Client("https://pulp.example.com/", task_throttle=throttle_count) assert client._task_executor._delegate._throttle() == throttle_count
def test_threads(spy_requested_threads): """Caller's requested 'threads' are passed into any internal ThreadPoolExecutors.""" with Client("https://pulp.example.com/", threads=3) as client: pass # Everything should have used the requested number of threads assert set(spy_requested_threads) == set([3])
def test_at_least_one_thread(spy_requested_threads): """Client always enforces at least one thread (per pool)""" with Client("https://pulp.example.com/", threads=-5) as client: pass # Everything should have used a single thread each assert set(spy_requested_threads) == set([1])
def test_client_lifecycle(client, requests_mocker): """Client is usable in with statement""" requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/search/", json=[{ "id": "repo1" }], ) client = Client("https://pulp.example.com") with client: # This should work OK assert client.search_repository().result() # But after end of 'with' statement, it should be shut down with pytest.raises(RuntimeError) as excinfo: client.search_repository() assert "cannot schedule new futures after shutdown" in str(excinfo.value)
def _get_pulp_client(self): auth = None # checks if pulp password is available as enviornment variable if self.args.user: if not self.args.password: self.args.password = os.environ.get("PULP_PASSWORD") if not self.args.password: LOG.warning("No password provided for %s", self.args.user) auth = (self.args.user, self.args.password) return Client(self.args.url, auth=auth, verify=False)
def _get_pulp_client(self): auth = None # checks if pulp password is available as enviornment variable if self.args.pulp_user: pulp_password = self.args.pulp_password or os.environ.get( "PULP_PASSWORD") if not pulp_password: LOG.warning("No pulp password provided for %s", self.args.pulp_user) auth = (self.args.pulp_user, pulp_password) return Client(self.args.pulp_url, auth=auth)
def test_construct_raises_on_bad_args(requests_mocker): """A client instance cannot be constructed with unexpected args.""" with pytest.raises(TypeError): client = Client("https://pulp.example.com/", whatever="foobar")
def test_can_construct_with_session_args(requests_mocker): """A client instance can be constructed with requests.Session kwargs.""" client = Client("https://pulp.example.com/", auth=("x", "y"), verify=False)
def test_can_construct(requests_mocker): """A client instance can be constructed with URL alone.""" client = Client("https://pulp.example.com/")
def _make_pulp_client(self, url, auth, insecure): return Client("https://" + url, auth=auth, verify=not insecure)
def test_search_async_cb(requests_mocker): """Paginated search works fine with an async callback on page.next. This test aims to cover a specific bug by which searches may be incorrectly cancelled even if the result was being awaited. """ with Client("https://pulp.example.com/") as client: client._PAGE_SIZE = 10 class ResponseGenerator(object): def __init__(self): self.repo_ids = ["repo-%s" % i for i in range(0, 997)] def __call__(self, request, context): # We're trying to trigger a certain bug which happens depending # on timing of GC. Simulating a non-instant HTTP request helps to # trigger it reliably. time.sleep(0.01) current_response = [] while self.repo_ids: current_response.append({"id": self.repo_ids.pop(0)}) if len(current_response) == client._PAGE_SIZE: break # GC in the middle of HTTP requests to increase chance of # hitting the error scenario. gc.collect() return current_response requests_mocker.post( "https://pulp.example.com/pulp/api/v2/repositories/search/", json=ResponseGenerator(), ) class Handler(object): def __init__(self): self._got_data = [] def __call__(self, page): self._got_data.extend(page.data) if page.next: # set ourselves up to process the next page. # It's important here that we only grab a reference to page.next # and don't keep a reference to page, as we are trying to cover # a previous bug where that scenario would wrongly cancel search. return f_flat_map(page.next, self) return f_return(self._got_data) all_data_f = f_flat_map(client.search_repository(), Handler()) # Searches are now running. Let's try doing many GCs while that's in progress # to increase the chance that we'll trigger the targeted bug. for _ in range(0, 50): gc.collect() # It should be able to complete the search. # Note: if "early GC" bug is not fixed, it would hang here, hence the # application of a timeout. all_data = all_data_f.result(10.0) # It should get all the repos successfully. ids = [repo.id for repo in all_data] assert ids == ["repo-%s" % i for i in range(0, 997)]