def test_search(self, mock_paging): mock_paging.return_value = self._search_test_data kind = "TestKind" expected_url = "https://test-domain.pushshift.io/reddit/{}/search".format( kind) api = PushshiftAPIMinimal( domain="test-domain", rate_limit_per_minute=self._rate_limit, detect_local_tz=False, ) result_gen = api._search(kind) for data_grp in self._search_test_data: for test_item in data_grp["data"]: actual_item = next(result_gen) self.assertIn(kind, str(actual_item)) self.assertEqual(test_item["created_utc"], actual_item.created) self.assertDictEqual(test_item, actual_item.d_) for key, val in test_item.items(): self.assertEqual(val, getattr(actual_item, key)) mock_paging.assert_called_once_with(expected_url, {}) # Make sure everything is complete try: next(result_gen) self.fail("Expected StopIteration") except StopIteration: pass
def test_search_stop_cond_batch(self, mock_paging): mock_paging.return_value = self._search_test_data kind = "TestKind" expected_url = "https://test-domain.pushshift.io/reddit/{}/search".format( kind) api = PushshiftAPIMinimal( domain="test-domain", rate_limit_per_minute=self._rate_limit, detect_local_tz=False, ) result_gen = api._search( kind, stop_condition=lambda x: x.created > 1530049619, return_batch=True) for data_grp in self._search_test_data: # Transform our source data to match what we expect with the stop condition expected_batch = list( filter(lambda x: x["created_utc"] <= 1530049619, data_grp["data"])) actual_batch = next(result_gen) self.assertEqual(len(expected_batch), len(actual_batch)) for idx, test_item in enumerate(expected_batch): actual_item = actual_batch[idx] self.assertIn(kind, str(actual_item)) self.assertEqual(test_item["created_utc"], actual_item.created) self.assertDictEqual(test_item, actual_item.d_) for key, val in test_item.items(): self.assertEqual(val, getattr(actual_item, key)) # Indicates that we hit the stop condition if len(expected_batch) < len(data_grp["data"]): break mock_paging.assert_called_once_with(expected_url, {}) # Make sure everything is complete try: next(result_gen) self.fail("Expected StopIteration") except StopIteration: pass