async def test_run__passes_correct_blocklists_to_subscriptions(mock_client):
submission = MockSubmission("12322")
api = MockExportAPI().with_submission(submission)
watcher = SubscriptionWatcher(api, mock_client)
method_called = MockMethod([submission])
watcher._get_new_results = method_called.async_call
watcher.BACK_OFF = 1
watcher.blocklists = {156: {"test", "ych"}, -200: {"example"}}
sub1 = MockSubscription("deer", 156)
sub2 = MockSubscription("dog", -232)
watcher.subscriptions = [sub1, sub2]
task = asyncio.get_event_loop().create_task(watcher_killer(watcher))
# Run watcher
await watcher.run()
await task
assert submission in sub1.submissions_checked
assert len(sub1.blocklists) == 1
assert sub1.blocklists[0] in [
AndQuery([NotQuery(WordQuery("test")),
NotQuery(WordQuery("ych"))]),
AndQuery([NotQuery(WordQuery("ych")),
NotQuery(WordQuery("test"))])
]
assert submission in sub2.submissions_checked
assert len(sub2.blocklists) == 1
assert sub2.blocklists[0] == AndQuery([])
assert method_called.called
def test_matches_result__doesnt_match_blocklisted_rating():
query = "an-example"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission is just an-example",
keywords=["example", "submission", "keywords"],
rating=Rating.ADULT).build_full_submission()
blocklist = AndQuery([
NotQuery(RatingQuery(Rating.ADULT)),
NotQuery(RatingQuery(Rating.MATURE))
])
match = subscription.matches_result(submission, blocklist)
assert not match
def test_negation():
assert parse_query("-first") == NotQuery(WordQuery("first"))
assert parse_query("!first") == NotQuery(WordQuery("first"))
assert parse_query("! first") == NotQuery(WordQuery("first"))
assert parse_query("not first") == NotQuery(WordQuery("first"))
assert parse_query("NOT first") == NotQuery(WordQuery("first"))
assert parse_query("first not second") == AndQuery([WordQuery("first"), NotQuery(WordQuery("second"))])
async def run(self):
"""
This method is launched as a task, it reads the browse endpoint for new submissions, and checks if they
match existing subscriptions
"""
self.running = True
while self.running:
try:
new_results = await self._get_new_results()
except Exception as e:
logger.error("Failed to get new results", exc_info=e)
continue
count = 0
heartbeat.update_heartbeat(heartbeat_app_name)
# Check for subscription updates
for result in new_results:
count += 1
# Try and get the full data
try:
full_result = await self.api.get_full_submission(result.submission_id)
logger.debug("Got full data for submission %s", result.submission_id)
except PageNotFound:
logger.warning("Submission %s, disappeared before I could check it.", result.submission_id)
continue
except Exception as e:
logger.error("Failed to get submission %s", result.submission_id, exc_info=e)
continue
# Copy subscriptions, to avoid "changed size during iteration" issues
subscriptions = self.subscriptions.copy()
# Check which subscriptions match
matching_subscriptions = []
for subscription in subscriptions:
blocklist = self.blocklists.get(subscription.destination, set())
blocklist_query = AndQuery([NotQuery(self._get_blocklist_query(block)) for block in blocklist])
if subscription.matches_result(full_result, blocklist_query):
matching_subscriptions.append(subscription)
logger.debug(
"Submission %s matches %s subscriptions",
result.submission_id,
len(matching_subscriptions)
)
if matching_subscriptions:
await self._send_updates(matching_subscriptions, full_result)
# Update latest ids with the submission we just checked, and save config
self._update_latest_ids([result])
# If we've done ten, update heartbeat
if count % self.UPDATE_PER_HEARTBEAT == 0:
heartbeat.update_heartbeat(heartbeat_app_name)
logger.debug("Heartbeat")
# Wait
await self._wait_while_running(self.BACK_OFF)
logger.info("Subscription watcher shutting down")
def test_matches_result__doesnt_match_blocklisted_tag():
query = "an-example"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission is just an-example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission,
NotQuery(WordQuery("test")))
assert not match
def test_quotes_and_brackets():
assert parse_query("not (hello \"first :) document\")") == NotQuery(
AndQuery([WordQuery("hello"), PhraseQuery("first :) document")])
)
def test_rating_field():
assert parse_query("rating:general") == RatingQuery(Rating.GENERAL)
assert parse_query("-rating:adult") == NotQuery(RatingQuery(Rating.ADULT))
assert parse_query("ych rating:mature") == AndQuery([WordQuery("ych"), RatingQuery(Rating.MATURE)])