async def test_retrieve_beatmapset_events_cache(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME][3] == [
qual_event, nom_event
]
db_module.clear_cache(SCRAPER_TEST_DB_NAME)
assert not db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME]
def get_reply_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("reply",
from_string("2020-01-01 01:04:00"),
beatmapset,
user=User(2, "sometwo"),
content="thanks")
yield Event("reply",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(1, "someone"),
content="hi")
yield Event("reply",
from_string("2020-01-01 00:31:00"),
beatmapset,
discussion,
user=User(2, "sometwo"),
content="say hi back")
if page == 2:
yield Event("reply",
from_string("2020-01-01 00:30:00"),
beatmapset,
discussion,
user=User(1, "someone"),
content="yes?")
yield Event("reply",
from_string("2020-01-01 00:00:00"),
beatmapset,
discussion,
user=User(2, "sometwo"),
content="please reply")
async def test_on_events(reader):
event1 = Event(_type="hello",
time=timestamp.from_string("2020-01-01 05:00:00"))
event2 = Event(_type="there",
time=timestamp.from_string("2020-01-01 07:00:00"))
event3 = Event(_type="hi",
time=timestamp.from_string("2020-01-01 11:00:00"))
event4 = Event(_type="yes",
time=timestamp.from_string("2020-01-01 13:00:00"))
event5 = Event(_type="no",
time=timestamp.from_string("2020-01-01 15:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
reader.database.insert_event(event3)
reader.database.insert_event(event4)
reader.database.insert_event(event5)
start = timestamp.from_string("2020-01-01 00:00:00")
middle = timestamp.from_string("2020-01-01 10:00:00")
end = timestamp.from_string("2020-01-01 18:00:00")
scope = Scope("any")
await reader._Reader__push_events_between(start, middle, scope)
await reader._Reader__push_events_between(middle, end, scope)
assert len(received_event_batches) == 2
async def test_push_all_new_events(reader):
event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 01:00:00"))
event2 = Event(_type="news",
time=timestamp.from_string("2020-01-01 02:00:00"))
event3 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 03:00:00"))
event4 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 04:00:00"))
event5 = Event(_type="news",
time=timestamp.from_string("2020-01-01 05:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
reader.database.insert_event(event3)
reader.database.insert_event(event4)
reader.database.insert_event(event5)
mapset_scope = Scope("mapset", None)
news_scope = Scope("news", None)
timestamp.set_last(
new_datetime=timestamp.from_string("2020-01-01 00:00:00"),
_id=reader._Reader__time_id(mapset_scope))
timestamp.set_last(
new_datetime=timestamp.from_string("2020-01-01 00:00:00"),
_id=reader._Reader__time_id(news_scope))
await reader._Reader__push_all_new_events()
assert received_events == [event1, event3, event4, event2, event5]
assert timestamp.get_last(reader._Reader__time_id(
mapset_scope)) == timestamp.from_string("2020-01-01 04:00:00")
assert timestamp.get_last(reader._Reader__time_id(
news_scope)) == timestamp.from_string("2020-01-01 05:00:00")
def get_discussion_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("problem",
from_string("2020-01-01 03:00:00"),
beatmapset,
discussion_dq,
user=User(2, "sometwo"),
content="no wait")
yield Event("hype",
from_string("2020-01-01 02:30:00"),
beatmapset,
user=User(2, "sometwo"),
content="hype")
yield Event("praise",
from_string("2020-01-01 02:00:00"),
beatmapset,
user=User(2, "sometwo"),
content="amazing")
if page == 2:
yield Event("issue_resolve",
from_string("2020-01-01 01:00:00"),
beatmapset,
discussion,
user=User(1, "someone"))
yield Event("praise",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(2, "sometwo"),
content="wow")
async def test_forward():
sub_both = Subscription(guild_id=1,
channel_id=2,
_filter="type:test1 or type:test2")
sub_one = Subscription(guild_id=1, channel_id=1, _filter="type:test1")
subscriber.add_subscription(sub_both)
subscriber.add_subscription(sub_one)
event1 = Event(_type="test1", time=datetime.utcnow())
event2 = Event(_type="test2", time=datetime.utcnow())
bot = MockBot()
with patch("bot.subscriber.format_embed", return_value=None):
with patch("bot.subscriber.send_event", new=bot.send_event):
await subscriber.forward(event1, bot)
await subscriber.forward(event2, bot)
await asyncio.sleep(2)
assert (sub_both, event1) in bot.event_sub_pairs
assert (sub_both, event2) in bot.event_sub_pairs
assert (sub_one, event1) in bot.event_sub_pairs
assert (
sub_one, event2
) not in bot.event_sub_pairs, "A subscription was forwarded an event it was supposed to filter."
async def test_events_between(reader):
event1 = Event(_type="test",
time=timestamp.from_string("2020-01-01 05:00:00"))
event2 = Event(_type="test",
time=timestamp.from_string("2020-01-01 07:00:00"))
event3 = Event(_type="test",
time=timestamp.from_string("2020-01-02 03:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
reader.database.insert_event(event3)
all_events = await reader.events_between(
timestamp.from_string("2020-01-01 00:00:00"),
timestamp.from_string("2020-01-03 00:00:00"))
assert await anext(all_events, None) == event1
assert await anext(all_events, None) == event2
assert await anext(all_events, None) == event3
assert await anext(all_events, None) is None
latter_events = await reader.events_between(
timestamp.from_string("2020-01-01 06:00:00"),
timestamp.from_string("2020-01-03 00:00:00"))
assert await anext(latter_events, None) == event2
assert await anext(latter_events, None) == event3
assert await anext(latter_events, None) is None
def parse_users_json(group_id: int, users_json: object, last_checked_at: datetime) -> Generator[Event, None, None]:
"""Returns a generator of group addition and removal events from the given users json and group id."""
missing_user_ids = get_group_user_ids(group_id)
new_users = []
for user_json in users_json:
user_id = user_json["id"]
if user_id in missing_user_ids:
missing_user_ids.remove(user_id)
else:
new_users.append(User(_id=user_id, name=user_json["username"]))
content = None
time = last_checked_at
for missing_user_id in missing_user_ids:
yield Event(
_type = types.REMOVE,
time = time,
user = get_group_user(group_id=group_id, user_id=missing_user_id),
group = Usergroup(_id=group_id),
content = content
)
for new_user in new_users:
yield Event(
_type = types.ADD,
time = time,
user = new_user,
group = Usergroup(_id=group_id),
content = content
)
async def test_history_filtering(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
test_database.insert_event(nom_event)
test_database.insert_event(qual_event)
test_database.insert_event(suggestion_event)
# The suggestion event should not appear in the history.
history = await format_history(beatmapset, database=test_database)
assert history == "\n:thought_balloon: [someone](https://osu.ppy.sh/users/1)\u2000:heart: [sometwo](https://osu.ppy.sh/users/2)"
async def test_history_truncated(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
for _ in range(20):
test_database.insert_event(nom_event)
nom_event.time += timedelta(seconds=15)
test_database.insert_event(qual_event)
history = await format_history(beatmapset,
length_limit=200,
database=test_database)
expected_history = "\n..."
for _ in range(10):
expected_history += ":thought_balloon: "
assert history == expected_history + ":heart:"
assert len(history) <= 200
async def test_retrieve_beatmapset_events(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
scraper_test_database.insert_event(suggestion_event)
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
events = await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert nom_event in events
assert qual_event in events
assert suggestion_event in events
def test_merge_concurrent_different_times():
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 11:00:00"), user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:00"))
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
def test_merge_concurrent_1_second_off():
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"))
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 1
assert merged_events[0].type == event2.type
assert merged_events[0].user == event1.user
def test_merge_concurrent():
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:00"))
event3 = Event(_type="something else", time=timestamp.from_string("2020-01-01 07:00:00"))
merged_events = merge_concurrent([event1, event2, event3])
assert len(merged_events) == 2
assert merged_events[0].type == event2.type
assert merged_events[0].user == event1.user
assert merged_events[1] == event3
def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, "artist", "title", User(1, "someone"), modes=["osu"], genre="g", language="l")
beatmapset2 = Beatmapset(2, "artist", "title", User(2, "sometwo"), modes=["osu"], genre="g", language="l")
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), beatmapset=beatmapset1, user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"), beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
async def test_events_between_greater_than(reader):
event1 = Event(_type="test", time=timestamp.from_string("2020-01-01 05:00:00"))
event2 = Event(_type="test", time=timestamp.from_string("2020-01-01 07:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
# If we resume at 05:00:00, the event at exactly 05:00:00 should be ignored.
events = await reader.events_between(timestamp.from_string("2020-01-01 05:00:00"), timestamp.from_string("2020-01-01 07:00:00"))
assert await anext(events, None) == event2
assert await anext(events, None) is None
def test_merge_concurrent_almost_same_time_nominates():
nom_event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:01"), user=User(1, "someone"))
nom_event2 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(2, "sometwo"))
qual_event1 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:01"))
merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1])
assert len(merged_events) == 2
assert merged_events[0].type == nom_event1.type
assert merged_events[0].user == nom_event2.user
assert merged_events[1].type == qual_event1.type
assert merged_events[1].user == nom_event1.user
def test_merge_concurrent_duplicates():
nom_event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone"))
nom_event2 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone"))
nom_event3 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone"))
qual_event1 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:00"))
qual_event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:00"))
merged_events = merge_concurrent([nom_event1, nom_event2, nom_event3, qual_event1, qual_event2])
assert len(merged_events) == 1
assert merged_events[0].type == qual_event1.type
assert merged_events[0].user == nom_event1.user
def test_merge_concurrent_same_time_nominates():
# Should pick the last event in the list before the qualify as the new qualify event, so the one by sometwo in this case.
nom_event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone"))
nom_event2 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(2, "sometwo"))
qual_event1 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:00"))
merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1])
assert len(merged_events) == 2
assert merged_events[0].type == nom_event1.type
assert merged_events[0].user == nom_event1.user
assert merged_events[1].type == qual_event1.type
assert merged_events[1].user == nom_event2.user
async def test_on_event(reader):
event1 = Event(_type="hello", time=timestamp.from_string("2020-01-01 05:00:00"))
event2 = Event(_type="there", time=timestamp.from_string("2020-01-01 07:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
_from = timestamp.from_string("2020-01-01 00:00:00")
to = timestamp.from_string("2020-01-01 10:00:00")
scope = Scope("any")
await reader._Reader__push_events_between(_from, to, scope)
assert received_events == [event1, event2]
assert reader.latest_event_time == timestamp.from_string("2020-01-01 07:00:00")
def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, User(1, "someone"), allow_api=False)
beatmapset2 = Beatmapset(2, User(2, "sometwo"), allow_api=False)
event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 13:00:00"),
beatmapset=beatmapset1,
user=User(1, "someone"))
event2 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 13:00:01"),
beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
async def test_on_event_scope(reader):
event1 = Event(_type="hello", time=timestamp.from_string("2020-01-01 05:00:00"))
event2 = Event(_type="there", time=timestamp.from_string("2020-01-01 07:00:00"))
reader.database.insert_event(event1)
reader.database.insert_event(event2)
_from = timestamp.from_string("2020-01-01 00:00:00")
to = timestamp.from_string("2020-01-01 10:00:00")
scope = Scope("greet", sql_target="type=\"hello\"")
await reader._Reader__push_events_between(_from, to, scope)
assert received_events == [event1]
assert timestamp.get_last(reader._Reader__time_id(scope)) == timestamp.from_string("2020-01-01 05:00:00")
assert reader.latest_event_time == timestamp.from_string("2020-01-01 05:00:00")
def test_dissect_aliases():
event = Event(_type="nominate", time=datetime.utcnow())
assert "type:nominate" in filter_context.dissect(event)
assert "type:nomination" in filter_context.dissect(event)
assert "type:nominated" in filter_context.dissect(event)
assert "type:bubbled" in filter_context.dissect(event)
def test_parse_fetch_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
# This event basically does: 1/2 -> 0/2
event = sev_parser.parse(discussion_id=4,
obv=0,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
expected_event = Event(_type="sev",
time=from_string("2020-07-22T21:00:00+00:00"),
beatmapset=beatmapset,
discussion=discussion,
content="0/2")
assert event.type == expected_event.type
assert event.time == expected_event.time
assert event.beatmapset == expected_event.beatmapset
assert event.discussion == expected_event.discussion
assert event.content == expected_event.content
assert event == expected_event
def test_format_link_no_beatmapset():
event = Event("test", from_string("2020-04-11 20:00:00"))
with pytest.raises(ValueError) as err:
format_link(event)
assert "missing a beatmapset" in str(err)
def convert_to_event(json: object) -> Event:
beatmapset_id = float(
json["beatmapsetId"]
) if "beatmapsetId" in json and json["beatmapsetId"] else None
discussion_id = float(
json["discussionId"]
) if "discussionId" in json and json["discussionId"] else None
user_id = float(
json["userId"]) if "userId" in json and json["userId"] else None
_type = json["type"]
time = timestamp.from_string(json["timestamp"])
beatmapset = database.retrieve_beatmapset(
"id=%s", (beatmapset_id, )) if beatmapset_id else None
discussion = database.retrieve_discussion(
"id=%s", (discussion_id, )) if discussion_id else None
user = database.retrieve_user("id=%s", (user_id, )) if user_id else None
content = json["content"]
return Event(_type=_type,
time=time,
beatmapset=beatmapset,
discussion=discussion,
user=user,
content=content)
async def test_nom_comment_from_praise():
beatmapset = Beatmapset(1147354, artist="Jashin-chan (CV: Suzuki Aina)", title="Jinbouchou Aika", creator=User(9590557, "Firika"), modes=["osu"], genre="g", language="l")
nominate_event = Event(types.NOMINATE, from_string("2020-07-03T12:14:13+00:00"), beatmapset, user=User(4, "mock user"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "nice"
async def test_delete_incomplete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset) # Missing user and content.
event = Event("test", from_string("2020-01-01 00:00:00"), discussion=discussion)
# The discussions json should not be checked, so we simply set it as None.
await __populate_additional_details(event, discussions_json=None, db_name=SCRAPER_TEST_DB_NAME)
assert event.marked_for_deletion
def test_dissect_user():
user = User(2, "some two")
event = Event(_type="test", time=datetime.utcnow(), user=user)
assert filter_context.dissect(event) == [
"user:\"some two\"",
"user-id:2",
"type:test"
]
def get_beatmapset_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("disqualify",
from_string("2020-01-01 03:00:00"),
beatmapset,
discussion_dq,
user=User(2, "sometwo"))
yield Event("qualify", from_string("2020-01-01 02:31:00"), beatmapset)
yield Event("nominate",
from_string("2020-01-01 02:31:00"),
beatmapset,
user=User(2, "sometwo"))
if page == 2:
yield Event("nominate",
from_string("2020-01-01 02:30:30"),
beatmapset,
user=User(1, "someone"))