def test_to_json_and_back(mock_client):
test_watcher_file = "./test_subscription_watcher.json"
if os.path.exists(test_watcher_file):
os.remove(test_watcher_file)
old_filename = SubscriptionWatcher.FILENAME
SubscriptionWatcher.FILENAME = test_watcher_file
api = MockExportAPI()
latest_submissions = [
SubmissionBuilder(submission_id="123243").build_short_submission(),
SubmissionBuilder(submission_id="123242").build_short_submission(),
SubmissionBuilder(submission_id="123240").build_short_submission()
]
subscription1 = Subscription("query", 1234)
subscription2 = Subscription("example", 5678)
watcher = SubscriptionWatcher(api, mock_client)
watcher._update_latest_ids(latest_submissions)
watcher.subscriptions.add(subscription1)
watcher.subscriptions.add(subscription2)
watcher.blocklists[3452] = {"test", "example"}
watcher.blocklists[1453] = {"ych"}
try:
watcher.save_to_json()
new_watcher = SubscriptionWatcher.load_from_json(api, mock_client)
assert len(new_watcher.latest_ids) == 3
assert "123243" in new_watcher.latest_ids
assert "123242" in new_watcher.latest_ids
assert "123240" in new_watcher.latest_ids
assert list(watcher.latest_ids) == list(new_watcher.latest_ids)
assert len(new_watcher.subscriptions) == 2
list_subs = list(new_watcher.subscriptions)
if list_subs[0].query_str == "query":
assert list_subs[0].destination == 1234
assert list_subs[1].query_str == "example"
assert list_subs[1].destination == 5678
else:
assert list_subs[0].query_str == "example"
assert list_subs[0].destination == 5678
assert list_subs[1].query_str == "query"
assert list_subs[1].destination == 1234
assert len(new_watcher.blocklists) == 2
assert 3452 in new_watcher.blocklists
assert len(new_watcher.blocklists[3452]) == 2
assert isinstance(new_watcher.blocklists[3452], set)
assert "test" in new_watcher.blocklists[3452]
assert "example" in new_watcher.blocklists[3452]
assert 1453 in new_watcher.blocklists
assert len(new_watcher.blocklists[1453]) == 1
assert isinstance(new_watcher.blocklists[1453], set)
assert "ych" in new_watcher.blocklists[1453]
finally:
SubscriptionWatcher.FILENAME = old_filename
os.remove(test_watcher_file)
def test_save_to_json(mock_client):
test_watcher_file = "./test_subscription_watcher.json"
if os.path.exists(test_watcher_file):
os.remove(test_watcher_file)
api = MockExportAPI()
latest_submissions = [
SubmissionBuilder(submission_id="123243").build_short_submission(),
SubmissionBuilder(submission_id="123242").build_short_submission(),
SubmissionBuilder(submission_id="123240").build_short_submission()
]
subscription1 = Subscription("query", 1234)
subscription2 = Subscription("example", 5678)
watcher = SubscriptionWatcher(api, mock_client)
watcher._update_latest_ids(latest_submissions)
watcher.subscriptions.add(subscription1)
watcher.subscriptions.add(subscription2)
watcher.blocklists[3452] = {"test", "example"}
watcher.blocklists[1453] = {"ych"}
watcher.FILENAME = test_watcher_file
try:
watcher.save_to_json()
assert os.path.exists(test_watcher_file)
with open(test_watcher_file, "r") as f:
data = json.load(f)
assert data is not None
assert len(data['latest_ids']) == 3
assert "123240" in data['latest_ids']
assert "123242" in data['latest_ids']
assert "123243" in data['latest_ids']
assert len(data["destinations"]) == 4
assert len(data["destinations"]["1234"]["subscriptions"]) == 1
assert data["destinations"]["1234"]["subscriptions"][0][
"query"] == "query"
assert len(data["destinations"]["1234"]["blocks"]) == 0
assert len(data["destinations"]["5678"]["subscriptions"]) == 1
assert data["destinations"]["5678"]["subscriptions"][0][
"query"] == "example"
assert len(data["destinations"]["5678"]["blocks"]) == 0
assert len(data["destinations"]["3452"]["subscriptions"]) == 0
assert len(data["destinations"]["3452"]["blocks"]) == 2
assert set([
block["query"] for block in data["destinations"]["3452"]["blocks"]
]) == {"test", "example"}
assert len(data["destinations"]["1453"]["subscriptions"]) == 0
assert len(data["destinations"]["1453"]["blocks"]) == 1
assert data["destinations"]["1453"]["blocks"][0]["query"] == "ych"
finally:
os.remove(test_watcher_file)
async def test_send_updates__gathers_subscriptions(mock_client):
api = MockExportAPI()
watcher = SubscriptionWatcher(api, mock_client)
subscription1 = Subscription("test", 12345)
subscription2 = Subscription("test2", 12345)
subscription3 = Subscription("test", 54321)
submission = SubmissionBuilder().build_mock_submission()
await watcher._send_updates([subscription1, subscription2, subscription3],
submission)
assert submission._send_message.call_count == 2
call_list = submission._send_message.call_args_list
# Indifferent to call order, so figure out the order here
call1 = call_list[0]
call2 = call_list[1]
if call1[0][1] != 12345:
call1 = call_list[1]
call2 = call_list[0]
args1, kwargs1 = call1
args2, kwargs2 = call2
# Check call matching two subscriptions
assert args1[0] == mock_client
assert args2[0] == mock_client
assert args1[1] == 12345
assert args2[1] == 54321
assert "update" in kwargs1['prefix'].lower()
assert "\"test\", \"test2\"" in kwargs1['prefix']
assert "subscriptions:" in kwargs1['prefix'].lower()
# And check the one subscription call
assert "update" in kwargs2['prefix'].lower()
assert "\"test\"" in kwargs2['prefix']
assert "subscription:" in kwargs2['prefix'].lower()
def test_create_short_dict_makes_thumb_bigger_75(self):
builder = SubmissionBuilder(thumb_size=75)
big_thumb_link = builder.thumbnail_url.replace("@75-", "@1600-")
submission = FASubmission.from_short_dict(builder.build_search_json())
assert submission.thumbnail_url == big_thumb_link
async def test_gif_submission_from_cache(mock_client):
submission = SubmissionBuilder(file_ext="gif",
file_size=47453).build_full_submission()
chat = MockChat(-9327622)
message_id = 2873292
convert = MockMethod("output.mp4")
submission._convert_gif = convert.async_call
mock_open = mock.mock_open(read_data=b"data")
mock_exists = MockMethod(True)
with mock.patch("fa_search_bot.sites.fa_submission.open", mock_open):
with mock.patch("os.path.exists", mock_exists.call):
await submission.send_message(mock_client,
chat,
reply_to=message_id)
assert not convert.called
mock_client.send_message.assert_called_once()
assert mock_client.send_message.call_args[1]['entity'] == chat
assert mock_exists.called
assert mock_exists.args[
0] == f"{submission.GIF_CACHE_DIR}/{submission.submission_id}.mp4"
assert mock_open.call_args[0][
0] == f"{submission.GIF_CACHE_DIR}/{submission.submission_id}.mp4"
assert mock_open.call_args[0][1] == "rb"
assert mock_client.send_message.call_args[1][
'file'] == mock_open.return_value
assert mock_client.send_message.call_args[1]['message'] == submission.link
assert mock_client.send_message.call_args[1]['reply_to'] == message_id
async def test_auto_doc_just_over_size_limit(mock_client):
title = "Example title"
author = FAUser("A writer", "awriter")
submission = SubmissionBuilder(file_ext="pdf",
file_size=FASubmission.SIZE_LIMIT_DOCUMENT +
1,
title=title,
author=author).build_full_submission()
chat = MockChat(-9327622)
message_id = 2873292
await submission.send_message(mock_client, chat, reply_to=message_id)
mock_client.send_message.assert_called_once()
assert mock_client.send_message.call_args[1]['entity'] == chat
assert mock_client.send_message.call_args[1][
'file'] == submission.full_image_url
assert mock_client.send_message.call_args[1]['reply_to'] == message_id
assert mock_client.send_message.call_args[1]['parse_mode'] == 'html'
sent_message = mock_client.send_message.call_args[1]['message']
assert sent_message.endswith(
f"{submission.link}\n<a href=\"{submission.download_url}\">Direct download</a>"
)
assert f"\"{title}\"" in sent_message
assert author.name in sent_message
assert author.link in sent_message
async def test_convert_gif_two_pass():
submission = SubmissionBuilder(file_ext="gif",
file_size=47453).build_full_submission()
mock_run = MockMultiMethod(["Test docker", "27.5", "ffmpeg1", "ffmpeg2"])
mock_filesize = MockMethod(submission.SIZE_LIMIT_GIF + 10)
submission._run_docker = mock_run.async_call
with mock.patch("os.path.getsize", mock_filesize.call):
output_path = await submission._convert_gif(submission.download_url)
assert output_path is not None
assert output_path.endswith(".mp4")
assert mock_run.calls == 4
# Initial ffmpeg call
assert mock_run.args[0][1].startswith(f"-i {submission.download_url} ")
# ffprobe call
assert mock_run.args[1][1].startswith("-show_entries format=duration ")
assert mock_run.kwargs[1]["entrypoint"] == "ffprobe"
# First ffmpeg two pass call
assert mock_run.args[2][1].startswith(f"-i {submission.download_url} ")
assert " -pass 1 -f mp4 " in mock_run.args[2][1]
assert mock_run.args[2][1].endswith(" /dev/null -y")
# Second ffmpeg two pass call
assert mock_run.args[3][1].startswith(f"-i {submission.download_url} ")
assert " -pass 2 " in mock_run.args[3][1]
assert mock_run.args[3][1].endswith(f" {output_path} -y")
async def test_get_status_before_submission(requests_mock):
builder = SubmissionBuilder(thumb_size=300)
api = FAExportAPI("http://example.com/")
requests_mock.get("http://example.com/status.json",
json={
"online": {
"guests": 17,
"registered": api.STATUS_LIMIT_REGISTERED - 1,
"other": 12,
"total": api.STATUS_LIMIT_REGISTERED + 28
},
"fa_server_time_at": "2020-09-08T00:13:14Z"
})
requests_mock.get(
f"http://example.com/submission/{builder.submission_id}.json",
json=builder.build_submission_json())
submission = await api.get_full_submission(builder.submission_id)
assert api.last_status_check is not None
assert api.slow_down_status is False
assert isinstance(submission, FASubmissionFull)
assert submission.submission_id == builder.submission_id
assert submission.link == builder.link
assert submission.thumbnail_url == builder.thumbnail_url.replace(
"@300-", "@1600-")
assert submission.download_url == builder.download_url
assert submission.full_image_url == builder.download_url
def test_matches_result__regex_matches_case_insensitive():
query = "d*taur"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="DeerTaur plays in woods").build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
async def test_send_updates__updates_latest(mock_client):
api = MockExportAPI()
watcher = SubscriptionWatcher(api, mock_client)
subscription = Subscription("test", 12345)
submission = SubmissionBuilder().build_mock_submission()
await watcher._send_updates([subscription], submission)
assert subscription.latest_update is not None
def test_matches_result__suffix_doesnt_match_term():
query = "*taur"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="taur plays in woods").build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__prefix_matches():
query = "deer*"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="deertaur plays in woods").build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_not_negative_rating():
query = "deer -rating:general"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Deer plays in woods",
rating=Rating.GENERAL).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
async def test_swf_submission(mock_client):
submission = SubmissionBuilder(file_ext="swf",
file_size=47453).build_full_submission()
chat = MockChat(-9327622)
message_id = 2873292
try:
await submission.send_message(mock_client, chat, reply_to=message_id)
assert False, "Should have thrown exception."
except CantSendFileType as e:
assert str(e) == "I'm sorry, I can't neaten \".swf\" files."
async def test_get_browse_page_specify_page(requests_mock):
builder = SubmissionBuilder()
api = FAExportAPI("http://example.com/", ignore_status=True)
requests_mock.get(f"http://example.com/browse.json?page=5",
json=[builder.build_search_json()])
results = await api.get_browse_page(5)
assert len(results) == 1
assert isinstance(results[0], FASubmissionShort)
assert results[0].submission_id == builder.submission_id
def test_matches_result__keywords_not_phrase():
query = "\"hello world\""
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission says hello to the world",
keywords=["example", "hello", "world"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__two_words_in_description_matches():
query = "example submission"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="test",
description="this submission is just an example",
keywords=["example", "keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_result__doesnt_match_except_quote():
query = "taur except \"no taur\""
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission contains no taur",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
async def test_unknown_type_submission(mock_client):
submission = SubmissionBuilder(file_ext="zzz",
file_size=47453).build_full_submission()
chat = MockChat(-9327622)
message_id = 2873292
try:
await submission.send_message(mock_client, chat, reply_to=message_id)
assert False, "Should have thrown exception."
except CantSendFileType as e:
assert str(
e) == "I'm sorry, I don't understand that file extension (zzz)."
async def test_send_updates__deleted_pauses_subs(mock_client):
api = MockExportAPI()
watcher = SubscriptionWatcher(api, mock_client)
subscription = Subscription("test", 12345)
watcher.subscriptions.add(subscription)
submission = SubmissionBuilder().build_mock_submission()
submission.send_message = lambda *args, **kwargs: (_ for _ in ()).throw(
InputUserDeactivatedError(None))
await watcher._send_updates([subscription], submission)
assert subscription.paused
def test_matches_result__keyword_field():
query = "keywords:deer"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission has no deer and will not be tagged deer",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__matches_query_with_hyphen():
query = "an-example"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission is just an-example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_word_in_tag():
query = "deer"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission is <b>deer</b>",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_result__case_insensitive_query():
query = "SUBMISSION"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="test submission",
description="this submission is just an example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_result__substring_in_keywords_no_match():
query = "keyword"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="test submission",
description="this submission is just an example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__does_not_match_query_with_applicable_negated_query():
query = "test -example"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description="this submission is just an example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__doesnt_match_except_bracket_clause():
query = "multi* except (multicol* or multitude)"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description=
"this submission is just an example of a multitude of multicoloured things",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__doesnt_match_except_field():
query = "keywords:(multi* except (multitude multiple multicol*))"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description=
"this submission is about multiplication but not tagged like that",
keywords=["multitude", "multiple", "multicoloured",
"multicolors"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match
def test_matches_result__except_matches_other_match():
query = "multi* except multitude"
subscription = Subscription(query, 12432)
submission = SubmissionBuilder(
title="Test submission",
description=
"this submission is just an example of a multitude of multiple things",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert match
def test_matches_result__not_when_paused():
query = "test"
subscription = Subscription(query, 12432)
subscription.paused = True
submission = SubmissionBuilder(
title="test submission",
description="this submission is just an example",
keywords=["example", "submission",
"keywords"]).build_full_submission()
match = subscription.matches_result(submission, AndQuery([]))
assert not match