def test_happy(self) -> None:
"""Tests the happy path."""
argv = ["", "tests/mock/access_log"]
buf = io.StringIO()
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
ctx.set_time(test_context.make_test_time())
relations_path = ctx.get_abspath("data/relations.yaml")
# 2020-05-09, so this will be recent
outputs = {
"git blame --line-porcelain " + relations_path: b"""
author-time 1588975200
\tujbuda:
"""
}
subprocess = test_context.TestSubprocess(outputs)
ctx.set_subprocess(subprocess)
parse_access_log.main(argv, buf, ctx)
buf.seek(0)
actual = buf.read()
self.assertIn("data/relation-inactiverelation.yaml: set inactive: false\n", actual)
self.assertIn("data/relation-gazdagret.yaml: set inactive: true\n", actual)
self.assertNotIn("data/relation-nosuchrelation.yaml: set inactive: ", actual)
# This is not in the output because it's considered as a recent relation.
self.assertNotIn("data/relation-ujbuda.yaml: set inactive: ", actual)
# This is not in the output as it's not a valid relation name.
self.assertNotIn("budafokxxx", actual)
# This is not in the output as it's a search bot, so such visits don't count.
# Also, if this would be not ignored, it would push 'inactiverelation' out of the active
# relation list.
self.assertNotIn("gyomaendrod", actual)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-stats.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
# Create a CSV that is definitely old enough to be removed.
old_path = ctx.get_abspath("workdir/stats/old.csv")
create_old_file(old_path)
today = time.strftime("%Y-%m-%d")
path = ctx.get_abspath("workdir/stats/%s.csv" % today)
cron.update_stats(ctx, overpass=True)
actual = util.get_content(path)
self.assertEqual(actual,
util.get_content("tests/network/overpass-stats.csv"))
# Make sure that the old CSV is removed.
self.assertFalse(os.path.exists(old_path))
with open(ctx.get_abspath("workdir/stats/ref.count"), "r") as stream:
num_ref = int(stream.read().strip())
self.assertEqual(num_ref, 300)
def test_happy(self) -> None:
"""Tests the happy path."""
time = test_context.make_test_time()
today = datetime.date.fromtimestamp(time.now())
actual = stats.get_previous_month(today, 2)
expected = datetime.date(2020, 3, 31)
self.assertEqual(actual, expected)
def test_empty_day_range(self) -> None:
"""Tests the case when the day range is empty."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_total(ctx, src_root, j, month_range=-1)
monthlytotal = j["monthlytotal"]
self.assertFalse(monthlytotal)
def test_empty_day_range(self) -> None:
"""Tests the case when the day range is empty."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_daily_new(ctx, src_root, j, day_range=-1)
daily = j["daily"]
self.assertFalse(daily)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_total(ctx, src_root, j)
monthlytotal = j["monthlytotal"]
self.assertEqual(len(monthlytotal), 1)
self.assertEqual(monthlytotal[0], ['2019-05', 203317])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_user_total(ctx, src_root, j)
usertotal = j["usertotal"]
self.assertEqual(len(usertotal), 1)
self.assertEqual(usertotal[0], ["2020-04-27", 43])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_topusers(ctx, src_root, j)
topusers = j["topusers"]
self.assertEqual(len(topusers), 20)
self.assertEqual(topusers[0], ["user1", "68885"])
def test_one_element_day_range(self) -> None:
"""Tests the case when the day range is of just one element."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_total(ctx, src_root, j, month_range=0)
monthlytotal = j["monthlytotal"]
self.assertEqual(len(monthlytotal), 2)
self.assertEqual(monthlytotal[0], ["2020-04", 253027])
self.assertEqual(monthlytotal[1], ["2020-05", 254651])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_progress(ctx, src_root, j)
progress = j["progress"]
self.assertEqual(progress["date"], "2020-05-10")
# 254651 / 300 * 100
self.assertEqual(progress["percentage"], 84883.67)
def test_new_missing(self) -> None:
"""Tests the case when the new path is missing."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
src_root = ctx.get_abspath("workdir/stats")
file_system.set_hide_paths(
[os.path.join(src_root, "2020-05-10.citycount")])
ctx.set_file_system(file_system)
ret = stats.get_topcities(ctx, src_root)
self.assertEqual(ret, [])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
# From now on, today is 2020-05-10, so this will read 2020-04-26, 2020-04-27, etc
# (till a file is missing.)
stats.handle_daily_new(ctx, src_root, j)
daily = j["daily"]
self.assertEqual(len(daily), 1)
self.assertEqual(daily[0], ["2020-04-26", 364])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_new(ctx, src_root, j)
monthly = j["monthly"]
self.assertEqual(len(monthly), 2)
# 2019-05 start -> end
self.assertEqual(monthly[0], ["2019-05", 3799])
# diff from last month end -> today
self.assertEqual(monthly[1], ["2020-05", 51334])
def test_no_sleep(self) -> None:
"""Tests the case when no sleep is needed."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
ctx.set_time(test_context.make_test_time())
cron.overpass_sleep(ctx)
test_time = cast(test_context.TestTime, ctx.get_time())
self.assertEqual(test_time.get_sleep(), 0)
def test_http_error(self) -> None:
"""Tests the case when we keep getting HTTP errors."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
old_mtime = ctx.get_file_system().getmtime(
ctx.get_abspath("workdir/stats/stats.json"))
cron.update_stats(ctx, overpass=True)
new_mtime = ctx.get_file_system().getmtime(
ctx.get_abspath("workdir/stats/stats.json"))
self.assertGreater(new_mtime, old_mtime)
def test_incomplete_last_month(self) -> None:
"""Tests the case when we have no data for the last, incomplete month."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
# This would be the data for the current state of the last, incomplete month.
hide_path = ctx.get_abspath("workdir/stats/2020-05-10.count")
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
ctx.set_file_system(file_system)
stats.handle_monthly_new(ctx, src_root, j)
monthly = j["monthly"]
# 1st element: 2019-05 start -> end
# No 2nd element, would be diff from last month end -> today
self.assertEqual(len(monthly), 1)
self.assertEqual(monthly[0], ["2019-05", 3799])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
old_citycount = b"""foo\t0
city1\t0
city2\t0
city3\t0
city4\t0
bar\t0
baz\t0
"""
old_citycount_value = io.BytesIO(old_citycount)
old_citycount_value.__setattr__("close", lambda: None)
new_citycount = b"""foo\t1000
city1\t1000
city2\t1000
city3\t1000
city4\t1000
bar\t2
baz\t2
"""
new_citycount_value = io.BytesIO(new_citycount)
new_citycount_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/2020-04-10.citycount"): old_citycount_value,
ctx.get_abspath("workdir/stats/2020-05-10.citycount"): new_citycount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
frequent_relations: Set[str] = {"foo", "bar"}
parse_access_log.check_top_edited_relations(ctx, frequent_relations)
self.assertIn("foo", frequent_relations)
self.assertIn("city1", frequent_relations)
self.assertIn("city2", frequent_relations)
self.assertIn("city3", frequent_relations)
self.assertIn("city4", frequent_relations)
self.assertNotIn("bar", frequent_relations)
self.assertNotIn("baz", frequent_relations)
def test_no_overpass(self) -> None:
"""Tests the case when we don't call overpass."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-wait.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
cron.update_stats(ctx, overpass=False)
test_time = cast(test_context.TestTime, ctx.get_time())
self.assertEqual(test_time.get_sleep(), 0)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
today_citycount = b"""budapest_01\t100
budapest_02\t200
\t42
"""
today_citycount_value = io.BytesIO(today_citycount)
today_citycount_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/2020-05-10.citycount"):
today_citycount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_topcities(ctx, src_root, j)
topcities = j["topcities"]
self.assertEqual(len(topcities), 2)
self.assertEqual(topcities[0], ("budapest_02", 190))
self.assertEqual(topcities[1], ("budapest_01", 90))
