def test_incomplete_context_from_db():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there") # Missing tab and difficulty.
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert not __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
def test_complete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there", tab="tab", difficulty="diff")
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
assert incomplete_discussion.user
assert incomplete_discussion.content
assert incomplete_discussion.tab
assert incomplete_discussion.difficulty
def test_old_discussion():
beatmapset = Beatmapset(41823, beatmapset_json=mock_old_beatmap.JSON)
discussion = Discussion(1234956, beatmapset)
# Can't obtain any discussion data from a beatmapset that doesn't have a discussion interface.
with pytest.raises(ParsingError):
get_complete_discussion_info(discussion, beatmapset, db_name=SCRAPER_TEST_DB_NAME)
def test_parse_fetch_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
# This event basically does: 1/2 -> 0/2
event = sev_parser.parse(discussion_id=4,
obv=0,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
expected_event = Event(_type="sev",
time=from_string("2020-07-22T21:00:00+00:00"),
beatmapset=beatmapset,
discussion=discussion,
content="0/2")
assert event.type == expected_event.type
assert event.time == expected_event.time
assert event.beatmapset == expected_event.beatmapset
assert event.discussion == expected_event.discussion
assert event.content == expected_event.content
assert event == expected_event
def test_incomplete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
99, beatmapset) # Missing user, content, tab, and difficulty.
assert not __complete_discussion_context(discussion,
db_name=SCRAPER_TEST_DB_NAME)
async def test_delete_incomplete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset) # Missing user and content.
event = Event("test", from_string("2020-01-01 00:00:00"), discussion=discussion)
# The discussions json should not be checked, so we simply set it as None.
await __populate_additional_details(event, discussions_json=None, db_name=SCRAPER_TEST_DB_NAME)
assert event.marked_for_deletion
def parse_discussion(self, discussion_json: object,
beatmapset_json: object,
beatmapset: Beatmapset) -> Discussion:
"""Returns a discussion from the given discussion json. The beatmapset json is also included for efficient username querying."""
_id = discussion_json["id"]
user = self.parse_user(discussion_json["user_id"], beatmapset_json)
content = discussion_json["posts"][0]["message"] if discussion_json[
"posts"] else None
tab = self.parse_tab(discussion_json, beatmapset_json)
difficulty = self.parse_diff(discussion_json, beatmapset_json)
return Discussion(_id, beatmapset, user, content, tab, difficulty)
def test_discussion():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(1234956, beatmapset)
# Some information will not be available until it is supplied by other sources
# (e.g. discussion jsons, prior database entires, scraping)
discussion = get_complete_discussion_info(discussion, beatmapset, db_name=SCRAPER_TEST_DB_NAME)
assert discussion.user.id == 4967662
assert discussion.user.name == "greenhue"
assert discussion.content == "since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms."
assert discussion.tab == "generalAll"
async def test_additional_details_dq():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1234956, beatmapset, user=User(4967662, "greenhue"),
content="since it ranks soon gonna just dq for fierys discussion https://osu.ppy.sh/beatmapsets/1001546/discussion/-/generalAll#/1228459 plus thought about points i brought up privately in dms.",
tab="tab", difficulty="diff"
)
dq_event = Event(types.DISQUALIFY, from_string("2019-10-27T04:23:20+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(dq_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert dq_event.content == dq_event.discussion.content
async def test_additional_details_kudosu():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(
1182017, beatmapset, user=User(9590557, "Firika"),
content="00:08:232 (5) - need fix too ;-;",
tab="tab", difficulty="diff"
)
kudosu_event = Event(types.KUDOSU_GAIN, from_string("2019-10-04T11:50:40+00:00"), beatmapset, discussion)
discussion_json = json.loads(mock_discussion_json)
await __populate_additional_details(kudosu_event, discussion_json, db_name=SCRAPER_TEST_DB_NAME)
assert kudosu_event.user == User(7342798, "_Epreus")
def test_dissect_discussion():
user = User(1, "some one")
creator = User(2, "some two")
beatmapset = Beatmapset(4, artist="yes", title="no", creator=creator, modes=["osu", "catch"], genre="g", language="l")
discussion = Discussion(3, beatmapset=beatmapset, user=user, content="hello")
event = Event(_type="test", time=datetime.utcnow(), beatmapset=beatmapset, discussion=discussion, user=user, content="hello")
event_dissection = filter_context.dissect(event)
for pair in [
"type:test",
"content:hello",
"discussion-id:3",
"author:\"some one\"",
"author-id:1",
"discussion-content:hello"
]:
assert pair in event_dissection
def kudosu_gain_event():
mapper = User(2, "sometwo")
beatmapset = Beatmapset(3,
"artist",
"title",
mapper, ["osu"],
allow_api=False)
user = User(1, "_someone_")
discussion = Discussion(5,
beatmapset,
user,
content="hi*",
tab="generalAll",
difficulty=None)
event = Event("kudosu_gain", from_string("2020-04-11 20:00:00"),
beatmapset, discussion, mapper)
return event
def dq_event():
disqualifier = User(1, "someone")
creator = User(2, "sometwo")
beatmapset = Beatmapset(_id=4,
artist="artist",
title="title",
creator=creator,
modes=["osu", "catch"],
allow_api=False)
discussion = Discussion(_id=3,
beatmapset=beatmapset,
user=disqualifier,
content="dqed")
return Event(_type=types.DISQUALIFY,
time=from_string("2020-01-01 03:00:00"),
beatmapset=beatmapset,
discussion=discussion,
user=disqualifier,
content="dqed")
def test_dissect_discussion():
user = User(1, "some one")
creator = User(2, "some two")
beatmapset = Beatmapset(4, creator=creator, allow_api=False)
discussion = Discussion(3,
beatmapset=beatmapset,
user=user,
content="hello")
event = Event(_type="test",
time=datetime.utcnow(),
beatmapset=beatmapset,
discussion=discussion,
user=user,
content="hello")
event_dissection = filter_context.dissect(event)
for pair in [
"type:test", "content:hello", "discussion-id:3",
"author:\"some one\"", "author-id:1", "discussion-content:hello"
]:
assert pair in event_dissection
def suggestion_event():
beatmapset = Beatmapset(3,
"artist",
"title",
User(2, "sometwo"), ["osu"],
allow_api=False)
user = User(1, "someone")
discussion = Discussion(5,
beatmapset,
user,
content="hi",
tab="generalAll",
difficulty=None)
event = Event("suggestion",
from_string("2020-04-11 20:00:00"),
beatmapset,
discussion,
user,
content="hi")
return event
def test_parse_both_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
#with pytest.raises(DeletedContextError) as err1:
# sev_parser.parse(discussion_id=4, obv=1, sev=2, time=from_string("2020-07-22T21:00:00+00:00"))
#assert "changed back" in str(err1).lower()
with pytest.raises(DeletedContextError) as err2:
sev_parser.parse(discussion_id=4,
obv=None,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
assert "neither severity nor obviousness have been set" in str(
err2).lower()
sys.path.append('..')
from datetime import datetime
from aiess import Event, Beatmapset, User, Discussion
from aiess.timestamp import from_string
beatmapset = Beatmapset(4,
"artist",
"title",
creator=User(1, "someone"),
modes=["osu"],
genre="genre",
language="language")
discussion = Discussion(20,
beatmapset=beatmapset,
user=User(2, "sometwo"),
content="hi")
discussion_dq = Discussion(22,
beatmapset=beatmapset,
user=User(2, "sometwo"),
content="no wait")
# Note that all events are yielded from newest to oldest.
def get_news_events(_from: datetime, limit: int = 20):
# The actual newspost doesn't matter, we're just making sure crawling the events works properly.
if _from == from_string("2020-01-01 03:00:00"):
yield Event("news",
from_string("2020-01-01 03:00:00"),
newspost=None,