def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, "artist", "title", User(1, "someone"), modes=["osu"], genre="g", language="l")
beatmapset2 = Beatmapset(2, "artist", "title", User(2, "sometwo"), modes=["osu"], genre="g", language="l")
event1 = Event(_type="nominate", time=timestamp.from_string("2020-01-01 13:00:00"), beatmapset=beatmapset1, user=User(1, "someone"))
event2 = Event(_type="qualify", time=timestamp.from_string("2020-01-01 13:00:01"), beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
def test_merge_concurrent_different_beatmapsets():
beatmapset1 = Beatmapset(1, User(1, "someone"), allow_api=False)
beatmapset2 = Beatmapset(2, User(2, "sometwo"), allow_api=False)
event1 = Event(_type="nominate",
time=timestamp.from_string("2020-01-01 13:00:00"),
beatmapset=beatmapset1,
user=User(1, "someone"))
event2 = Event(_type="qualify",
time=timestamp.from_string("2020-01-01 13:00:01"),
beatmapset=beatmapset2)
merged_events = merge_concurrent([event1, event2])
assert len(merged_events) == 2
assert merged_events[0] == event1
assert merged_events[1] == event2
def test_parse_fetch_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
# This event basically does: 1/2 -> 0/2
event = sev_parser.parse(discussion_id=4,
obv=0,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
expected_event = Event(_type="sev",
time=from_string("2020-07-22T21:00:00+00:00"),
beatmapset=beatmapset,
discussion=discussion,
content="0/2")
assert event.type == expected_event.type
assert event.time == expected_event.time
assert event.beatmapset == expected_event.beatmapset
assert event.discussion == expected_event.discussion
assert event.content == expected_event.content
assert event == expected_event
async def test_history_filtering(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
test_database.insert_event(nom_event)
test_database.insert_event(qual_event)
test_database.insert_event(suggestion_event)
# The suggestion event should not appear in the history.
history = await format_history(beatmapset, database=test_database)
assert history == "\n:thought_balloon: [someone](https://osu.ppy.sh/users/1)\u2000:heart: [sometwo](https://osu.ppy.sh/users/2)"
async def test_retrieve_beatmapset_events(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
suggestion_event = Event("suggestion",
from_string("2020-01-01 01:00:00"),
beatmapset,
user=User(3, "somethree"))
scraper_test_database.insert_event(suggestion_event)
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
events = await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert nom_event in events
assert qual_event in events
assert suggestion_event in events
async def test_history_truncated(test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "mapper"), ["osu"],
allow_api=False)
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
for _ in range(20):
test_database.insert_event(nom_event)
nom_event.time += timedelta(seconds=15)
test_database.insert_event(qual_event)
history = await format_history(beatmapset,
length_limit=200,
database=test_database)
expected_history = "\n..."
for _ in range(10):
expected_history += ":thought_balloon: "
assert history == expected_history + ":heart:"
assert len(history) <= 200
async def test_retrieve_beatmapset_events_cache(scraper_test_database):
beatmapset = Beatmapset(3,
"artist",
"title",
User(4, "creator"), ["osu"],
genre="g",
language="l")
nom_event = Event("nominate",
from_string("2020-01-01 00:00:00"),
beatmapset,
user=User(1, "someone"))
qual_event = Event("qualify",
from_string("2020-01-01 05:00:00"),
beatmapset,
user=User(2, "sometwo"))
scraper_test_database.insert_event(nom_event)
scraper_test_database.insert_event(qual_event)
await scraper_test_database.retrieve_beatmapset_events(beatmapset)
assert db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME][3] == [
qual_event, nom_event
]
db_module.clear_cache(SCRAPER_TEST_DB_NAME)
assert not db_module.beatmapset_event_cache[SCRAPER_TEST_DB_NAME]
def test_incomplete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
99, beatmapset) # Missing user, content, tab, and difficulty.
assert not __complete_discussion_context(discussion,
db_name=SCRAPER_TEST_DB_NAME)
def test_old_discussion():
beatmapset = Beatmapset(41823, beatmapset_json=mock_old_beatmap.JSON)
discussion = Discussion(1234956, beatmapset)
# Can't obtain any discussion data from a beatmapset that doesn't have a discussion interface.
with pytest.raises(ParsingError):
get_complete_discussion_info(discussion, beatmapset, db_name=SCRAPER_TEST_DB_NAME)
def beatmapset():
return Beatmapset(_id=1001546,
artist="Carpool Tunnel",
title="Afterlight",
creator=User(_id=7342798, name="_Epreus"),
modes=["osu"],
genre="g",
language="l")
async def test_nom_comment_none():
beatmapset = Beatmapset(1112303, artist="Fox Stevenson", title="Take You Down", creator=User(5745865, "Altai"), modes=["osu"], genre="g", language="l")
nominate_event = Event(types.NOMINATE, from_string("2020-07-01T20:48:47+00:00"), beatmapset, user=User(8623835, "Peter"))
discussion_json = json.loads(mock_discussion_json_nom_comment_1)
await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content is None
async def test_delete_incomplete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset) # Missing user and content.
event = Event("test", from_string("2020-01-01 00:00:00"), discussion=discussion)
# The discussions json should not be checked, so we simply set it as None.
await __populate_additional_details(event, discussions_json=None, db_name=SCRAPER_TEST_DB_NAME)
assert event.marked_for_deletion
async def test_nom_comment_from_praise():
beatmapset = Beatmapset(1147354, artist="Jashin-chan (CV: Suzuki Aina)", title="Jinbouchou Aika", creator=User(9590557, "Firika"), modes=["osu"], genre="g", language="l")
nominate_event = Event(types.NOMINATE, from_string("2020-07-03T12:14:13+00:00"), beatmapset, user=User(4, "mock user"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "nice"
async def test_nom_comment_from_note():
beatmapset = Beatmapset(1147354, artist="Jashin-chan (CV: Suzuki Aina)", title="Jinbouchou Aika", creator=User(9590557, "Firika"), modes=["osu"], genre="g", language="l")
nominate_event = Event(types.NOMINATE, from_string("2020-07-03T12:14:13+00:00"), beatmapset, user=User(5312547, "Lafayla"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "02:31:783 - 02:34:783 - should be fine being snapped to 1/16, the piano does weird pick ups for these, same thing applies to 09:51:845 - i don't think its problematic"
def test_incomplete_context_from_db():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there") # Missing tab and difficulty.
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert not __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
def qualify_event():
beatmapset = Beatmapset(3,
"artist_with*strange~symbols`",
"title",
User(2, "_sometwo_"), ["osu"],
allow_api=False)
event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset)
return event
def test_passes_filter_event_object():
beatmapset = Beatmapset(3, allow_api=False)
event = Event(_type="nominate",
time=datetime.utcnow(),
beatmapset=beatmapset,
user=User(2, "sometwo"))
assert filter_context.test("type:nominate and user:sometwo", event)
assert not filter_context.test("type:reply", event)
def qualify_event():
beatmapset = Beatmapset(3,
"artist_with*strange~symbols`",
"title",
User(2, "_sometwo_"), ["osu"],
genre="g",
language="l")
event = Event("qualify", from_string("2020-04-11 20:00:00"), beatmapset)
return event
async def test_additional_details_legacy_thread():
# E.g. beatmapset 296817.
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = None
kudosu_event = Event(types.KUDOSU_GAIN, from_string("2019-10-04T11:50:40+00:00"), beatmapset, discussion)
discussion_json = None
await __populate_additional_details(kudosu_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert kudosu_event.user == None
def test_complete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there", tab="tab", difficulty="diff")
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
assert incomplete_discussion.user
assert incomplete_discussion.content
assert incomplete_discussion.tab
assert incomplete_discussion.difficulty
def test_discussion():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(1234956, beatmapset)
# Some information will not be available until it is supplied by other sources
# (e.g. discussion jsons, prior database entires, scraping)
discussion = get_complete_discussion_info(discussion, beatmapset, db_name=SCRAPER_TEST_DB_NAME)
assert discussion.user.id == 4967662
assert discussion.user.name == "greenhue"
assert discussion.content == "since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms."
assert discussion.tab == "generalAll"
async def test_additional_details_dq():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1234956, beatmapset, user=User(4967662, "greenhue"),
content="since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms.",
tab="tab", difficulty="diff"
)
dq_event = Event(types.DISQUALIFY, from_string("2019-10-27T04:23:20+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(dq_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert dq_event.content == dq_event.discussion.content
async def test_additional_details_kudosu():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1182017, beatmapset, user=User(9590557, "Firika"),
content="00:08:232 (5) - need fix too ;-;",
tab="tab", difficulty="diff"
)
kudosu_event = Event(types.KUDOSU_GAIN, from_string("2019-10-04T11:50:40+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(kudosu_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert kudosu_event.user == User(7342798, "_Epreus")
def test_dissect_beatmapset():
user = User(2, "some two")
beatmapset = Beatmapset(4, artist="yes", title="no", creator=user, modes=["osu", "catch"], genre="g", language="l")
event = Event(_type="test", time=datetime.utcnow(), beatmapset=beatmapset)
assert filter_context.dissect(event) == [
"set-id:4",
"mapset-id:4",
"beatmapset-id:4",
"artist:yes",
"title:no",
"creator:\"some two\"",
"creator-id:2",
"mode:osu",
"mode:catch",
"type:test"
]
def test_dissect_discussion():
user = User(1, "some one")
creator = User(2, "some two")
beatmapset = Beatmapset(4, artist="yes", title="no", creator=creator, modes=["osu", "catch"], genre="g", language="l")
discussion = Discussion(3, beatmapset=beatmapset, user=user, content="hello")
event = Event(_type="test", time=datetime.utcnow(), beatmapset=beatmapset, discussion=discussion, user=user, content="hello")
event_dissection = filter_context.dissect(event)
for pair in [
"type:test",
"content:hello",
"discussion-id:3",
"author:\"some one\"",
"author-id:1",
"discussion-content:hello"
]:
assert pair in event_dissection
async def test_nom_comment_from_hype():
beatmapset = Beatmapset(1112303,
artist="Fox Stevenson",
title="Take You Down",
creator=User(5745865, "Altai"),
allow_api=False)
nominate_event = Event(types.NOMINATE,
from_string("2020-07-01T20:48:57+00:00"),
beatmapset,
user=User(2204515, "Mao"))
discussion_json = json.loads(mock_discussion_json_nom_comment_1)
await __populate_additional_details(nominate_event,
discussion_json,
db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content == "<3"
async def test_nom_comment_praise_then_suggestions():
beatmapset = Beatmapset(1147354,
artist="Jashin-chan (CV: Suzuki Aina)",
title="Jinbouchou Aika",
creator=User(9590557, "Firika"),
allow_api=False)
nominate_event = Event(types.NOMINATE,
from_string("2020-07-03T12:14:13+00:00"),
beatmapset,
user=User(896613, "Lasse"))
discussion_json = json.loads(mock_discussion_json_nom_comment_2)
await __populate_additional_details(nominate_event,
discussion_json,
db_name=SCRAPER_TEST_DB_NAME)
assert nominate_event.content is None
def kudosu_gain_event():
mapper = User(2, "sometwo")
beatmapset = Beatmapset(3,
"artist",
"title",
mapper, ["osu"],
allow_api=False)
user = User(1, "_someone_")
discussion = Discussion(5,
beatmapset,
user,
content="hi*",
tab="generalAll",
difficulty=None)
event = Event("kudosu_gain", from_string("2020-04-11 20:00:00"),
beatmapset, discussion, mapper)
return event
def dq_event():
disqualifier = User(1, "someone")
creator = User(2, "sometwo")
beatmapset = Beatmapset(_id=4,
artist="artist",
title="title",
creator=creator,
modes=["osu", "catch"],
allow_api=False)
discussion = Discussion(_id=3,
beatmapset=beatmapset,
user=disqualifier,
content="dqed")
return Event(_type=types.DISQUALIFY,
time=from_string("2020-01-01 03:00:00"),
beatmapset=beatmapset,
discussion=discussion,
user=disqualifier,
content="dqed")
def test_dissect_discussion():
user = User(1, "some one")
creator = User(2, "some two")
beatmapset = Beatmapset(4, creator=creator, allow_api=False)
discussion = Discussion(3,
beatmapset=beatmapset,
user=user,
content="hello")
event = Event(_type="test",
time=datetime.utcnow(),
beatmapset=beatmapset,
discussion=discussion,
user=user,
content="hello")
event_dissection = filter_context.dissect(event)
for pair in [
"type:test", "content:hello", "discussion-id:3",
"author:\"some one\"", "author-id:1", "discussion-content:hello"
]:
assert pair in event_dissection
