async def test_retrieve_beatmapset_events_cache(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME][3] == [
qual_event, nom_event
]
db_module.clear_cache(SCRAPER_TEST_DB_NAME)
assert not db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME]
def get_reply_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("reply",
from_string("2020-01-01 01:04:00"),
beatmapset,
user=User(2, "sometwo"),
content="thanks")
yield Event("reply",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(1, "someone"),
content="hi")
yield Event("reply",
from_string("2020-01-01 00:31:00"),
beatmapset,
discussion,
user=User(2, "sometwo"),
content="say hi back")
if page == 2:
yield Event("reply",
from_string("2020-01-01 00:30:00"),
beatmapset,
discussion,
user=User(1, "someone"),
content="yes?")
yield Event("reply",
from_string("2020-01-01 00:00:00"),
beatmapset,
discussion,
user=User(2, "sometwo"),
content="please reply")
async def test_history_filtering(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
test_database.insert_event(nom_event)
test_database.insert_event(qual_event)
test_database.insert_event(suggestion_event)
# The suggestion event should not appear in the history.
history = await format_history(beatmapset, database=test_database)
assert history == "\n:thought_balloon: [someone](https://osu.ppy.sh/users/1)\u2000:heart: [sometwo](https://osu.ppy.sh/users/2)"
async def test_history_truncated(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
for _ in range(20):
test_database.insert_event(nom_event)
nom_event.time += timedelta(seconds=15)
test_database.insert_event(qual_event)
history = await format_history(beatmapset,
length_limit=200,
database=test_database)
expected_history = "\n..."
for _ in range(10):
expected_history += ":thought_balloon: "
assert history == expected_history + ":heart:"
assert len(history) <= 200
def test_parse_timing():
# Test both additions and removals.
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(1, "one"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(2, "two"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(3, "three"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(4, "four"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(5, "five"))
start_time = datetime.utcnow()
events = []
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
for event in group_parser.parse(
group_id=7,
group_page=mock_groups.soup,
last_checked_at=from_string("2020-07-22T21:00:00+00:00")):
events.append(event)
end_time = datetime.utcnow()
# We should not be using the api to fill in user names and such, as this data is available within the users json.
assert (end_time - start_time).total_seconds() < 3
def get_discussion_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("problem",
from_string("2020-01-01 03:00:00"),
beatmapset,
discussion_dq,
user=User(2, "sometwo"),
content="no wait")
yield Event("hype",
from_string("2020-01-01 02:30:00"),
beatmapset,
user=User(2, "sometwo"),
content="hype")
yield Event("praise",
from_string("2020-01-01 02:00:00"),
beatmapset,
user=User(2, "sometwo"),
content="amazing")
if page == 2:
yield Event("issue_resolve",
from_string("2020-01-01 01:00:00"),
beatmapset,
discussion,
user=User(1, "someone"))
yield Event("praise",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(2, "sometwo"),
content="wow")
async def test_retrieve_beatmapset_events(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
scraper_test_database.insert_event(suggestion_event)
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
events = await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert nom_event in events
assert qual_event in events
assert suggestion_event in events
def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, "artist", "title", User(1, "someone"), modes=["osu"], genre="g", language="l")
beatmapset2 = Beatmapset(2, "artist", "title", User(2, "sometwo"), modes=["osu"], genre="g", language="l")
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), beatmapset=beatmapset1, user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"), beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
async def test_additional_details_resolve():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1234956, beatmapset, user=User(4967662, "greenhue"),
content="since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms.",
tab="tab", difficulty="diff"
)
resolve_event = Event(types.RESOLVE, from_string("2019-10-27T09:00:00+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(resolve_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert resolve_event.user == User(7342798, "_Epreus")
async def test_additional_details_kudosu():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1182017, beatmapset, user=User(9590557, "Firika"),
content="00:08:232 (5) - need fix too ;-;",
tab="tab", difficulty="diff"
)
kudosu_event = Event(types.KUDOSU_GAIN, from_string("2019-10-04T11:50:40+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(kudosu_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert kudosu_event.user == User(7342798, "_Epreus")
def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, User(1, "someone"), allow_api=False)
beatmapset2 = Beatmapset(2, User(2, "sometwo"), allow_api=False)
event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 13:00:00"),
beatmapset=beatmapset1,
user=User(1, "someone"))
event2 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 13:00:01"),
beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
def test_merge_concurrent_almost_same_time_nominates():
nom_event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 05:00:01"),
user=User(1, "someone"))
nom_event2 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 05:00:00"),
user=User(2, "sometwo"))
qual_event1 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 05:00:01"))
merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1])
assert len(merged_events) == 2
assert merged_events[0].type == nom_event1.type
assert merged_events[0].user == nom_event2.user
assert merged_events[1].type == qual_event1.type
assert merged_events[1].user == nom_event1.user
def parse_users_json(group_id: int, users_json: object, last_checked_at: datetime) -> Generator[Event, None, None]:
"""Returns a generator of group addition and removal events from the given users json and group id."""
missing_user_ids = get_group_user_ids(group_id)
new_users = []
for user_json in users_json:
user_id = user_json["id"]
if user_id in missing_user_ids:
missing_user_ids.remove(user_id)
else:
new_users.append(User(_id=user_id, name=user_json["username"]))
content = None
time = last_checked_at
for missing_user_id in missing_user_ids:
yield Event(
_type = types.REMOVE,
time = time,
user = get_group_user(group_id=group_id, user_id=missing_user_id),
group = Usergroup(_id=group_id),
content = content
)
for new_user in new_users:
yield Event(
_type = types.ADD,
time = time,
user = new_user,
group = Usergroup(_id=group_id),
content = content
)
def test_parse_fetch_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
# This event basically does: 1/2 -> 0/2
event = sev_parser.parse(discussion_id=4,
obv=0,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
expected_event = Event(_type="sev",
time=from_string("2020-07-22T21:00:00+00:00"),
beatmapset=beatmapset,
discussion=discussion,
content="0/2")
assert event.type == expected_event.type
assert event.time == expected_event.time
assert event.beatmapset == expected_event.beatmapset
assert event.discussion == expected_event.discussion
assert event.content == expected_event.content
assert event == expected_event
def test_merge_concurrent_same_time_nominates():
# Should pick the last event in the list before the qualify as the new qualify event, so the one by sometwo in this case.
nom_event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 05:00:00"),
user=User(1, "someone"))
nom_event2 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 05:00:00"),
user=User(2, "sometwo"))
qual_event1 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 05:00:00"))
merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1])
assert len(merged_events) == 2
assert merged_events[0].type == nom_event1.type
assert merged_events[0].user == nom_event1.user
assert merged_events[1].type == qual_event1.type
assert merged_events[1].user == nom_event2.user
def get_beatmapset_events(page: int = 1, limit: int = 50):
if page == 1:
yield Event("disqualify",
from_string("2020-01-01 03:00:00"),
beatmapset,
discussion_dq,
user=User(2, "sometwo"))
yield Event("qualify", from_string("2020-01-01 02:31:00"), beatmapset)
yield Event("nominate",
from_string("2020-01-01 02:31:00"),
beatmapset,
user=User(2, "sometwo"))
if page == 2:
yield Event("nominate",
from_string("2020-01-01 02:30:30"),
beatmapset,
user=User(1, "someone"))
def test_dissect_discussion():
user = User(1, "some one")
creator = User(2, "some two")
beatmapset = Beatmapset(4, artist="yes", title="no", creator=creator, modes=["osu", "catch"], genre="g", language="l")
discussion = Discussion(3, beatmapset=beatmapset, user=user, content="hello")
event = Event(_type="test", time=datetime.utcnow(), beatmapset=beatmapset, discussion=discussion, user=user, content="hello")
event_dissection = filter_context.dissect(event)
for pair in [
"type:test",
"content:hello",
"discussion-id:3",
"author:\"some one\"",
"author-id:1",
"discussion-content:hello"
]:
assert pair in event_dissection
def test_dissect_user():
user = User(2, "some two")
event = Event(_type="test", time=datetime.utcnow(), user=user)
assert filter_context.dissect(event) == [
"user:\"some two\"",
"user-id:2",
"type:test"
]
def test_incomplete_context_from_db():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there") # Missing tab and difficulty.
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert not __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
def beatmapset():
return Beatmapset(_id=1001546,
artist="Carpool Tunnel",
title="Afterlight",
creator=User(_id=7342798, name="_Epreus"),
modes=["osu"],
genre="g",
language="l")
async def test_nom_comment_from_praise():
beatmapset = Beatmapset(1147354,
artist="Jashin-chan (CV: Suzuki Aina)",
title="Jinbouchou Aika",
creator=User(9590557, "Firika"),
allow_api=False)
nominate_event = Event(types.NOMINATE,
from_string("2020-07-03T12:14:13+00:00"),
beatmapset,
user=User(4, "mock user"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event,
discussion_json,
db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "nice"
async def test_nom_comment_from_note():
beatmapset = Beatmapset(1147354,
artist="Jashin-chan (CV: Suzuki Aina)",
title="Jinbouchou Aika",
creator=User(9590557, "Firika"),
allow_api=False)
nominate_event = Event(types.NOMINATE,
from_string("2020-07-03T12:14:13+00:00"),
beatmapset,
user=User(5312547, "Lafayla"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event,
discussion_json,
db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "02:31:783 - 02:34:783 - should be fine being snapped to 1/16, the piano does weird pick ups for these, same thing applies to 09:51:845 - i don't think its problematic"
async def test_nom_comment_none():
beatmapset = Beatmapset(1112303,
artist="Fox Stevenson",
title="Take You Down",
creator=User(5745865, "Altai"),
allow_api=False)
nominate_event = Event(types.NOMINATE,
from_string("2020-07-01T20:48:47+00:00"),
beatmapset,
user=User(8623835, "Peter"))
discussion_json = json.loads(mock_discussion_json_nom_comment_1)
await __populate_additional_details(nominate_event,
discussion_json,
db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content is None
def test_passes_filter_event_object():
beatmapset = Beatmapset(3, allow_api=False)
event = Event(_type="nominate",
time=datetime.utcnow(),
beatmapset=beatmapset,
user=User(2, "sometwo"))
assert filter_context.test("type:nominate and user:sometwo", event)
assert not filter_context.test("type:reply", event)
def kudosu_gain_event():
mapper = User(2, "sometwo")
beatmapset = Beatmapset(3,
"artist",
"title",
mapper, ["osu"],
allow_api=False)
user = User(1, "_someone_")
discussion = Discussion(5,
beatmapset,
user,
content="hi*",
tab="generalAll",
difficulty=None)
event = Event("kudosu_gain", from_string("2020-04-11 20:00:00"),
beatmapset, discussion, mapper)
return event
def qualify_event():
beatmapset = Beatmapset(3,
"artist_with*strange~symbols`",
"title",
User(2, "_sometwo_"), ["osu"],
allow_api=False)
event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset)
return event
def test_parse_removals():
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(1, "someone"))
events = []
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
for event in group_parser.parse(
group_id=7,
group_page=mock_groups.soup,
last_checked_at=from_string("2020-07-22T21:00:00+00:00")):
events.append(event)
assert len(events) == 18
assert events[0] == Event(_type="remove",
time=from_string("2020-07-22T21:00:00+00:00"),
group=Usergroup(7),
user=User(_id=1, name="someone"))
def test_parse_additions():
events = []
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
for event in group_parser.parse(
group_id=7,
group_page=mock_groups.soup,
last_checked_at=from_string("2020-07-22T21:00:00+00:00")):
events.append(event)
assert len(events) == 17
assert events[0] == Event(_type="add",
time=from_string("2020-07-22T21:00:00+00:00"),
group=Usergroup(7),
user=User(_id=2202163))
assert events[1] == Event(_type="add",
time=from_string("2020-07-22T21:00:00+00:00"),
group=Usergroup(7),
user=User(_id=3621552))
def qualify_event():
beatmapset = Beatmapset(3,
"artist_with*strange~symbols`",
"title",
User(2, "_sometwo_"), ["osu"],
genre="g",
language="l")
event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset)
return event
def dq_event():
disqualifier = User(1, "someone")
creator = User(2, "sometwo")
beatmapset = Beatmapset(_id=4,
artist="artist",
title="title",
creator=creator,
modes=["osu", "catch"],
allow_api=False)
discussion = Discussion(_id=3,
beatmapset=beatmapset,
user=disqualifier,
content="dqed")
return Event(_type=types.DISQUALIFY,
time=from_string("2020-01-01 03:00:00"),
beatmapset=beatmapset,
discussion=discussion,
user=disqualifier,
content="dqed")
