def test_search(self, mock_paging):
mock_paging.return_value = self._search_test_data
kind = "TestKind"
expected_url = "https://test-domain.pushshift.io/reddit/{}/search".format(
kind)
api = PushshiftAPIMinimal(
domain="test-domain",
rate_limit_per_minute=self._rate_limit,
detect_local_tz=False,
)
result_gen = api._search(kind)
for data_grp in self._search_test_data:
for test_item in data_grp["data"]:
actual_item = next(result_gen)
self.assertIn(kind, str(actual_item))
self.assertEqual(test_item["created_utc"], actual_item.created)
self.assertDictEqual(test_item, actual_item.d_)
for key, val in test_item.items():
self.assertEqual(val, getattr(actual_item, key))
mock_paging.assert_called_once_with(expected_url, {})
# Make sure everything is complete
try:
next(result_gen)
self.fail("Expected StopIteration")
except StopIteration:
pass
def test_search_stop_cond_batch(self, mock_paging):
mock_paging.return_value = self._search_test_data
kind = "TestKind"
expected_url = "https://test-domain.pushshift.io/reddit/{}/search".format(
kind)
api = PushshiftAPIMinimal(
domain="test-domain",
rate_limit_per_minute=self._rate_limit,
detect_local_tz=False,
)
result_gen = api._search(
kind,
stop_condition=lambda x: x.created > 1530049619,
return_batch=True)
for data_grp in self._search_test_data:
# Transform our source data to match what we expect with the stop condition
expected_batch = list(
filter(lambda x: x["created_utc"] <= 1530049619,
data_grp["data"]))
actual_batch = next(result_gen)
self.assertEqual(len(expected_batch), len(actual_batch))
for idx, test_item in enumerate(expected_batch):
actual_item = actual_batch[idx]
self.assertIn(kind, str(actual_item))
self.assertEqual(test_item["created_utc"], actual_item.created)
self.assertDictEqual(test_item, actual_item.d_)
for key, val in test_item.items():
self.assertEqual(val, getattr(actual_item, key))
# Indicates that we hit the stop condition
if len(expected_batch) < len(data_grp["data"]):
break
mock_paging.assert_called_once_with(expected_url, {})
# Make sure everything is complete
try:
next(result_gen)
self.fail("Expected StopIteration")
except StopIteration:
pass