async def test_retrieve_beatmapset_events_cache(scraper_test_database): beatmapset = Beatmapset(3, "artist", "title", User(4, "creator"), ["osu"], genre="g", language="l") nom_event = Event("nominate", from_string("2020-01-01 00:00:00"), beatmapset, user=User(1, "someone")) qual_event = Event("qualify", from_string("2020-01-01 05:00:00"), beatmapset, user=User(2, "sometwo")) scraper_test_database.insert_event(nom_event) scraper_test_database.insert_event(qual_event) await scraper_test_database.retrieve_beatmapset_events(beatmapset) assert db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME][3] == [ qual_event, nom_event ] db_module.clear_cache(SCRAPER_TEST_DB_NAME) assert not db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME]
def get_reply_events(page: int = 1, limit: int = 50): if page == 1: yield Event("reply", from_string("2020-01-01 01:04:00"), beatmapset, user=User(2, "sometwo"), content="thanks") yield Event("reply", from_string("2020-01-01 01:00:00"), beatmapset, user=User(1, "someone"), content="hi") yield Event("reply", from_string("2020-01-01 00:31:00"), beatmapset, discussion, user=User(2, "sometwo"), content="say hi back") if page == 2: yield Event("reply", from_string("2020-01-01 00:30:00"), beatmapset, discussion, user=User(1, "someone"), content="yes?") yield Event("reply", from_string("2020-01-01 00:00:00"), beatmapset, discussion, user=User(2, "sometwo"), content="please reply")
async def test_history_filtering(test_database): beatmapset = Beatmapset(3, "artist", "title", User(4, "mapper"), ["osu"], allow_api=False) nom_event = Event("nominate", from_string("2020-01-01 00:00:00"), beatmapset, user=User(1, "someone")) qual_event = Event("qualify", from_string("2020-01-01 05:00:00"), beatmapset, user=User(2, "sometwo")) suggestion_event = Event("suggestion", from_string("2020-01-01 01:00:00"), beatmapset, user=User(3, "somethree")) test_database.insert_event(nom_event) test_database.insert_event(qual_event) test_database.insert_event(suggestion_event) # The suggestion event should not appear in the history. history = await format_history(beatmapset, database=test_database) assert history == "\n:thought_balloon: [someone](https://osu.ppy.sh/users/1)\u2000:heart: [sometwo](https://osu.ppy.sh/users/2)"
async def test_history_truncated(test_database): beatmapset = Beatmapset(3, "artist", "title", User(4, "mapper"), ["osu"], allow_api=False) nom_event = Event("nominate", from_string("2020-01-01 00:00:00"), beatmapset, user=User(1, "someone")) qual_event = Event("qualify", from_string("2020-01-01 05:00:00"), beatmapset, user=User(2, "sometwo")) for _ in range(20): test_database.insert_event(nom_event) nom_event.time += timedelta(seconds=15) test_database.insert_event(qual_event) history = await format_history(beatmapset, length_limit=200, database=test_database) expected_history = "\n..." for _ in range(10): expected_history += ":thought_balloon: " assert history == expected_history + ":heart:" assert len(history) <= 200
def test_parse_timing(): # Test both additions and removals. Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(1, "one")) Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(2, "two")) Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(3, "three")) Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(4, "four")) Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(5, "five")) start_time = datetime.utcnow() events = [] with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME", SCRAPER_TEST_DB_NAME): for event in group_parser.parse( group_id=7, group_page=mock_groups.soup, last_checked_at=from_string("2020-07-22T21:00:00+00:00")): events.append(event) end_time = datetime.utcnow() # We should not be using the api to fill in user names and such, as this data is available within the users json. assert (end_time - start_time).total_seconds() < 3
def get_discussion_events(page: int = 1, limit: int = 50): if page == 1: yield Event("problem", from_string("2020-01-01 03:00:00"), beatmapset, discussion_dq, user=User(2, "sometwo"), content="no wait") yield Event("hype", from_string("2020-01-01 02:30:00"), beatmapset, user=User(2, "sometwo"), content="hype") yield Event("praise", from_string("2020-01-01 02:00:00"), beatmapset, user=User(2, "sometwo"), content="amazing") if page == 2: yield Event("issue_resolve", from_string("2020-01-01 01:00:00"), beatmapset, discussion, user=User(1, "someone")) yield Event("praise", from_string("2020-01-01 00:00:00"), beatmapset, user=User(2, "sometwo"), content="wow")
async def test_retrieve_beatmapset_events(scraper_test_database): beatmapset = Beatmapset(3, "artist", "title", User(4, "creator"), ["osu"], genre="g", language="l") nom_event = Event("nominate", from_string("2020-01-01 00:00:00"), beatmapset, user=User(1, "someone")) qual_event = Event("qualify", from_string("2020-01-01 05:00:00"), beatmapset, user=User(2, "sometwo")) suggestion_event = Event("suggestion", from_string("2020-01-01 01:00:00"), beatmapset, user=User(3, "somethree")) scraper_test_database.insert_event(suggestion_event) scraper_test_database.insert_event(nom_event) scraper_test_database.insert_event(qual_event) events = await scraper_test_database.retrieve_beatmapset_events(beatmapset) assert nom_event in events assert qual_event in events assert suggestion_event in events
def test_merge_concurrent_different_beatmapsets(): beatmapset1 = Beatmapset(1, "artist", "title", User(1, "someone"), modes=["osu"], genre="g", language="l") beatmapset2 = Beatmapset(2, "artist", "title", User(2, "sometwo"), modes=["osu"], genre="g", language="l") event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), beatmapset=beatmapset1, user=User(1, "someone")) event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"), beatmapset=beatmapset2) merged_events = merge_concurrent([event1, event2]) assert len(merged_events) == 2 assert merged_events[0] == event1 assert merged_events[1] == event2
async def test_additional_details_resolve(): beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON) discussion = Discussion( 1234956, beatmapset, user=User(4967662, "greenhue"), content="since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms.", tab="tab", difficulty="diff" ) resolve_event = Event(types.RESOLVE, from_string("2019-10-27T09:00:00+00:00"), beatmapset, discussion) discussion_json = json.loads(mock_discussion_json) await __populate_additional_details(resolve_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME) assert resolve_event.user == User(7342798, "_Epreus")
async def test_additional_details_kudosu(): beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON) discussion = Discussion( 1182017, beatmapset, user=User(9590557, "Firika"), content="00:08:232 (5) - need fix too ;-;", tab="tab", difficulty="diff" ) kudosu_event = Event(types.KUDOSU_GAIN, from_string("2019-10-04T11:50:40+00:00"), beatmapset, discussion) discussion_json = json.loads(mock_discussion_json) await __populate_additional_details(kudosu_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME) assert kudosu_event.user == User(7342798, "_Epreus")
def test_merge_concurrent_different_beatmapsets(): beatmapset1 = Beatmapset(1, User(1, "someone"), allow_api=False) beatmapset2 = Beatmapset(2, User(2, "sometwo"), allow_api=False) event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), beatmapset=beatmapset1, user=User(1, "someone")) event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"), beatmapset=beatmapset2) merged_events = merge_concurrent([event1, event2]) assert len(merged_events) == 2 assert merged_events[0] == event1 assert merged_events[1] == event2
def test_merge_concurrent_almost_same_time_nominates(): nom_event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:01"), user=User(1, "someone")) nom_event2 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(2, "sometwo")) qual_event1 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:01")) merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1]) assert len(merged_events) == 2 assert merged_events[0].type == nom_event1.type assert merged_events[0].user == nom_event2.user assert merged_events[1].type == qual_event1.type assert merged_events[1].user == nom_event1.user
def parse_users_json(group_id: int, users_json: object, last_checked_at: datetime) -> Generator[Event, None, None]: """Returns a generator of group addition and removal events from the given users json and group id.""" missing_user_ids = get_group_user_ids(group_id) new_users = [] for user_json in users_json: user_id = user_json["id"] if user_id in missing_user_ids: missing_user_ids.remove(user_id) else: new_users.append(User(_id=user_id, name=user_json["username"])) content = None time = last_checked_at for missing_user_id in missing_user_ids: yield Event( _type = types.REMOVE, time = time, user = get_group_user(group_id=group_id, user_id=missing_user_id), group = Usergroup(_id=group_id), content = content ) for new_user in new_users: yield Event( _type = types.ADD, time = time, user = new_user, group = Usergroup(_id=group_id), content = content )
def test_parse_fetch_unchanged(): with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME", SCRAPER_TEST_DB_NAME): user = User(_id=1, allow_api=False) beatmapset = Beatmapset(_id=2, creator=user, allow_api=False) discussion = Discussion(_id=4, beatmapset=beatmapset, user=user, content="123") Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion) Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2) # This event basically does: 1/2 -> 0/2 event = sev_parser.parse(discussion_id=4, obv=0, sev=None, time=from_string("2020-07-22T21:00:00+00:00")) expected_event = Event(_type="sev", time=from_string("2020-07-22T21:00:00+00:00"), beatmapset=beatmapset, discussion=discussion, content="0/2") assert event.type == expected_event.type assert event.time == expected_event.time assert event.beatmapset == expected_event.beatmapset assert event.discussion == expected_event.discussion assert event.content == expected_event.content assert event == expected_event
def test_merge_concurrent_same_time_nominates(): # Should pick the last event in the list before the qualify as the new qualify event, so the one by sometwo in this case. nom_event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(1, "someone")) nom_event2 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 05:00:00"), user=User(2, "sometwo")) qual_event1 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 05:00:00")) merged_events = merge_concurrent([nom_event1, nom_event2, qual_event1]) assert len(merged_events) == 2 assert merged_events[0].type == nom_event1.type assert merged_events[0].user == nom_event1.user assert merged_events[1].type == qual_event1.type assert merged_events[1].user == nom_event2.user
def get_beatmapset_events(page: int = 1, limit: int = 50): if page == 1: yield Event("disqualify", from_string("2020-01-01 03:00:00"), beatmapset, discussion_dq, user=User(2, "sometwo")) yield Event("qualify", from_string("2020-01-01 02:31:00"), beatmapset) yield Event("nominate", from_string("2020-01-01 02:31:00"), beatmapset, user=User(2, "sometwo")) if page == 2: yield Event("nominate", from_string("2020-01-01 02:30:30"), beatmapset, user=User(1, "someone"))
def test_dissect_discussion(): user = User(1, "some one") creator = User(2, "some two") beatmapset = Beatmapset(4, artist="yes", title="no", creator=creator, modes=["osu", "catch"], genre="g", language="l") discussion = Discussion(3, beatmapset=beatmapset, user=user, content="hello") event = Event(_type="test", time=datetime.utcnow(), beatmapset=beatmapset, discussion=discussion, user=user, content="hello") event_dissection = filter_context.dissect(event) for pair in [ "type:test", "content:hello", "discussion-id:3", "author:\"some one\"", "author-id:1", "discussion-content:hello" ]: assert pair in event_dissection
def test_dissect_user(): user = User(2, "some two") event = Event(_type="test", time=datetime.utcnow(), user=user) assert filter_context.dissect(event) == [ "user:\"some two\"", "user-id:2", "type:test" ]
def test_incomplete_context_from_db(): beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON) discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there") # Missing tab and difficulty. incomplete_discussion = Discussion(99, beatmapset) Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion) assert not __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
def beatmapset(): return Beatmapset(_id=1001546, artist="Carpool Tunnel", title="Afterlight", creator=User(_id=7342798, name="_Epreus"), modes=["osu"], genre="g", language="l")
async def test_nom_comment_from_praise(): beatmapset = Beatmapset(1147354, artist="Jashin-chan (CV: Suzuki Aina)", title="Jinbouchou Aika", creator=User(9590557, "Firika"), allow_api=False) nominate_event = Event(types.NOMINATE, from_string("2020-07-03T12:14:13+00:00"), beatmapset, user=User(4, "mock user")) discussion_json = json.loads(mock_discussion_json_nom_comment_2) await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME) assert nominate_event.content == "nice"
async def test_nom_comment_from_note(): beatmapset = Beatmapset(1147354, artist="Jashin-chan (CV: Suzuki Aina)", title="Jinbouchou Aika", creator=User(9590557, "Firika"), allow_api=False) nominate_event = Event(types.NOMINATE, from_string("2020-07-03T12:14:13+00:00"), beatmapset, user=User(5312547, "Lafayla")) discussion_json = json.loads(mock_discussion_json_nom_comment_2) await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME) assert nominate_event.content == "02:31:783 - 02:34:783 - should be fine being snapped to 1/16, the piano does weird pick ups for these, same thing applies to 09:51:845 - i don't think its problematic"
async def test_nom_comment_none(): beatmapset = Beatmapset(1112303, artist="Fox Stevenson", title="Take You Down", creator=User(5745865, "Altai"), allow_api=False) nominate_event = Event(types.NOMINATE, from_string("2020-07-01T20:48:47+00:00"), beatmapset, user=User(8623835, "Peter")) discussion_json = json.loads(mock_discussion_json_nom_comment_1) await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME) assert nominate_event.content is None
def test_passes_filter_event_object(): beatmapset = Beatmapset(3, allow_api=False) event = Event(_type="nominate", time=datetime.utcnow(), beatmapset=beatmapset, user=User(2, "sometwo")) assert filter_context.test("type:nominate and user:sometwo", event) assert not filter_context.test("type:reply", event)
def kudosu_gain_event(): mapper = User(2, "sometwo") beatmapset = Beatmapset(3, "artist", "title", mapper, ["osu"], allow_api=False) user = User(1, "_someone_") discussion = Discussion(5, beatmapset, user, content="hi*", tab="generalAll", difficulty=None) event = Event("kudosu_gain", from_string("2020-04-11 20:00:00"), beatmapset, discussion, mapper) return event
def qualify_event(): beatmapset = Beatmapset(3, "artist_with*strange~symbols`", "title", User(2, "_sometwo_"), ["osu"], allow_api=False) event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset) return event
def test_parse_removals(): Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7), user=User(1, "someone")) events = [] with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME", SCRAPER_TEST_DB_NAME): for event in group_parser.parse( group_id=7, group_page=mock_groups.soup, last_checked_at=from_string("2020-07-22T21:00:00+00:00")): events.append(event) assert len(events) == 18 assert events[0] == Event(_type="remove", time=from_string("2020-07-22T21:00:00+00:00"), group=Usergroup(7), user=User(_id=1, name="someone"))
def test_parse_additions(): events = [] with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME", SCRAPER_TEST_DB_NAME): for event in group_parser.parse( group_id=7, group_page=mock_groups.soup, last_checked_at=from_string("2020-07-22T21:00:00+00:00")): events.append(event) assert len(events) == 17 assert events[0] == Event(_type="add", time=from_string("2020-07-22T21:00:00+00:00"), group=Usergroup(7), user=User(_id=2202163)) assert events[1] == Event(_type="add", time=from_string("2020-07-22T21:00:00+00:00"), group=Usergroup(7), user=User(_id=3621552))
def qualify_event(): beatmapset = Beatmapset(3, "artist_with*strange~symbols`", "title", User(2, "_sometwo_"), ["osu"], genre="g", language="l") event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset) return event
def dq_event(): disqualifier = User(1, "someone") creator = User(2, "sometwo") beatmapset = Beatmapset(_id=4, artist="artist", title="title", creator=creator, modes=["osu", "catch"], allow_api=False) discussion = Discussion(_id=3, beatmapset=beatmapset, user=disqualifier, content="dqed") return Event(_type=types.DISQUALIFY, time=from_string("2020-01-01 03:00:00"), beatmapset=beatmapset, discussion=discussion, user=disqualifier, content="dqed")