def test_parse_fetch_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
# This event basically does: 1/2 -> 0/2
event = sev_parser.parse(discussion_id=4,
obv=0,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
expected_event = Event(_type="sev",
time=from_string("2020-07-22T21:00:00+00:00"),
beatmapset=beatmapset,
discussion=discussion,
content="0/2")
assert event.type == expected_event.type
assert event.time == expected_event.time
assert event.beatmapset == expected_event.beatmapset
assert event.discussion == expected_event.discussion
assert event.content == expected_event.content
assert event == expected_event
def test_parse_timing():
# Test both additions and removals.
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(1, "one"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(2, "two"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(3, "three"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(4, "four"))
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(5, "five"))
start_time = datetime.utcnow()
events = []
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
for event in group_parser.parse(
group_id=7,
group_page=mock_groups.soup,
last_checked_at=from_string("2020-07-22T21:00:00+00:00")):
events.append(event)
end_time = datetime.utcnow()
# We should not be using the api to fill in user names and such, as this data is available within the users json.
assert (end_time - start_time).total_seconds() < 3
def test_incomplete_context_from_db():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there") # Missing tab and difficulty.
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert not __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
def test_database():
database = Database(SCRAPER_TEST_DB_NAME)
# Reset database to state before any tests ran.
database.clear_table_data("events")
database.clear_table_data("discussions")
database.clear_table_data("beatmapsets")
database.clear_table_data("beatmapset_modes")
database.clear_table_data("newsposts")
database.clear_table_data("group_users")
database.clear_table_data("users")
return database
def test_complete_context():
beatmapset = Beatmapset(1001546, beatmapset_json=mock_beatmap.JSON)
discussion = Discussion(99, beatmapset, user=User(1, "someone"), content="hello there", tab="tab", difficulty="diff")
incomplete_discussion = Discussion(99, beatmapset)
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
assert __complete_discussion_context(incomplete_discussion, db_name=SCRAPER_TEST_DB_NAME)
assert incomplete_discussion.user
assert incomplete_discussion.content
assert incomplete_discussion.tab
assert incomplete_discussion.difficulty
def test_get_group_events():
Database(SCRAPER_TEST_DB_NAME).clear_table_data("group_users")
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME", SCRAPER_TEST_DB_NAME):
events = get_group_events(_from=datetime.utcnow())
event_n = 0
for event in events:
assert event.type == types.ADD
assert event.user
assert event.group
event_n += 1
assert event_n > 100
def test_parse_both_unchanged():
with mock.patch("scraper.parsers.sev_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
user = User(_id=1, allow_api=False)
beatmapset = Beatmapset(_id=2, creator=user, allow_api=False)
discussion = Discussion(_id=4,
beatmapset=beatmapset,
user=user,
content="123")
Database(SCRAPER_TEST_DB_NAME).insert_discussion(discussion)
Database(SCRAPER_TEST_DB_NAME).insert_obv_sev(discussion, obv=1, sev=2)
#with pytest.raises(DeletedContextError) as err1:
# sev_parser.parse(discussion_id=4, obv=1, sev=2, time=from_string("2020-07-22T21:00:00+00:00"))
#assert "changed back" in str(err1).lower()
with pytest.raises(DeletedContextError) as err2:
sev_parser.parse(discussion_id=4,
obv=None,
sev=None,
time=from_string("2020-07-22T21:00:00+00:00"))
assert "neither severity nor obviousness have been set" in str(
err2).lower()
def __complete_discussion_context(discussion: Discussion,
db_name: str = SCRAPER_DB_NAME) -> bool:
"""Completes the context of the discussion from prior database entries, if present. Returns true if succeeded."""
cached_discussion = Database(db_name).retrieve_discussion(
"id=%s", (discussion.id, ))
if not cached_discussion:
return False
complete = (cached_discussion.user and cached_discussion.content
and cached_discussion.tab and cached_discussion.difficulty)
if not complete:
return False
discussion.user = cached_discussion.user
discussion.content = cached_discussion.content
discussion.tab = cached_discussion.tab
discussion.difficulty = cached_discussion.difficulty
return True
def test_parse_removals():
Database(SCRAPER_TEST_DB_NAME).insert_group_user(group=Usergroup(7),
user=User(1, "someone"))
events = []
with mock.patch("scraper.parsers.group_parser.SCRAPER_DB_NAME",
SCRAPER_TEST_DB_NAME):
for event in group_parser.parse(
group_id=7,
group_page=mock_groups.soup,
last_checked_at=from_string("2020-07-22T21:00:00+00:00")):
events.append(event)
assert len(events) == 18
assert events[0] == Event(_type="remove",
time=from_string("2020-07-22T21:00:00+00:00"),
group=Usergroup(7),
user=User(_id=1, name="someone"))
def test_correct_setup():
database = Database(SCRAPER_TEST_DB_NAME)
assert not database.retrieve_table_data("events")
assert not database.retrieve_table_data("discussions")
def setup_function():
database = Database(SCRAPER_TEST_DB_NAME)
# Reset database to state before any tests ran.
database.clear_table_data("events")
database.clear_table_data("discussions")
def get_group_user(group_id: int, user_id: int) -> List[int]:
"""Returns the last remembered user beloning to the given group id with the given user id."""
return Database(SCRAPER_DB_NAME).retrieve_group_user("group_id=%s AND user_id=%s", (group_id, user_id))[1]
def get_group_user_ids(group_id: int) -> List[int]:
"""Returns the last remembered user ids beloning to the given group id."""
group_user_relations = Database(SCRAPER_DB_NAME).retrieve_group_users("group_id=%s", (group_id,))
return [user.id for group, user in group_user_relations]
def __init__(self, reader_id: str, db_name: str):
self.reader_id = reader_id
self.database = Database(db_name)
self.running = False
self.latest_event_time = None
return "".join([
f"{fmt(event.type, colors.EVENT)}",
f" ({fmt(event.user, colors.AUTHOR)})" if event.user else "",
f" on {fmt(event.beatmapset, colors.CONTEXT)}" if event.beatmapset else "",
f" to/from {fmt(event.group, colors.CONTEXT)}" if event.group else "",
f" \"{event.content}\"" if event.content else ""
])
def insert_db(events) -> None:
"""Inserts the given event list into the database in reversed order."""
if not events:
return
events.sort(key=lambda event: event.time)
log(f"--- Inserting {len(events)} Events into the Database ---")
for event in events:
log(".", newline=False)
database.insert_event(event)
log()
def last_updated(current_time: datetime, _id: str) -> None:
"""Updates the last updated file to reflect the given time."""
log(f"--- Last Updated [{_id}] {current_time} ---")
timestamp.set_last(current_time, _id)
logger.init()
database = Database(SCRAPER_DB_NAME)
loop = asyncio.get_event_loop()
loop.run_until_complete(gather_loop())
def setup_function():
Database(SCRAPER_TEST_DB_NAME).clear_table_data("group_users")