def test_happy(self) -> None: """Tests the happy path.""" argv = ["", "tests/mock/access_log"] buf = io.StringIO() ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) ctx.set_time(test_context.make_test_time()) relations_path = ctx.get_abspath("data/relations.yaml") # 2020-05-09, so this will be recent outputs = { "git blame --line-porcelain " + relations_path: b""" author-time 1588975200 \tujbuda: """ } subprocess = test_context.TestSubprocess(outputs) ctx.set_subprocess(subprocess) parse_access_log.main(argv, buf, ctx) buf.seek(0) actual = buf.read() self.assertIn("data/relation-inactiverelation.yaml: set inactive: false\n", actual) self.assertIn("data/relation-gazdagret.yaml: set inactive: true\n", actual) self.assertNotIn("data/relation-nosuchrelation.yaml: set inactive: ", actual) # This is not in the output because it's considered as a recent relation. self.assertNotIn("data/relation-ujbuda.yaml: set inactive: ", actual) # This is not in the output as it's not a valid relation name. self.assertNotIn("budafokxxx", actual) # This is not in the output as it's a search bot, so such visits don't count. # Also, if this would be not ignored, it would push 'inactiverelation' out of the active # relation list. self.assertNotIn("gyomaendrod", actual)
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) routes: List[test_context.URLRoute] = [ test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt"), test_context.URLRoute( url="https://overpass-api.de/api/interpreter", data_path="", result_path="tests/network/overpass-stats.csv"), ] network = test_context.TestNetwork(routes) ctx.set_network(network) # Create a CSV that is definitely old enough to be removed. old_path = ctx.get_abspath("workdir/stats/old.csv") create_old_file(old_path) today = time.strftime("%Y-%m-%d") path = ctx.get_abspath("workdir/stats/%s.csv" % today) cron.update_stats(ctx, overpass=True) actual = util.get_content(path) self.assertEqual(actual, util.get_content("tests/network/overpass-stats.csv")) # Make sure that the old CSV is removed. self.assertFalse(os.path.exists(old_path)) with open(ctx.get_abspath("workdir/stats/ref.count"), "r") as stream: num_ref = int(stream.read().strip()) self.assertEqual(num_ref, 300)
def test_happy(self) -> None: """Tests the happy path.""" time = test_context.make_test_time() today = datetime.date.fromtimestamp(time.now()) actual = stats.get_previous_month(today, 2) expected = datetime.date(2020, 3, 31) self.assertEqual(actual, expected)
def test_empty_day_range(self) -> None: """Tests the case when the day range is empty.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_total(ctx, src_root, j, month_range=-1) monthlytotal = j["monthlytotal"] self.assertFalse(monthlytotal)
def test_empty_day_range(self) -> None: """Tests the case when the day range is empty.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_daily_new(ctx, src_root, j, day_range=-1) daily = j["daily"] self.assertFalse(daily)
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_total(ctx, src_root, j) monthlytotal = j["monthlytotal"] self.assertEqual(len(monthlytotal), 1) self.assertEqual(monthlytotal[0], ['2019-05', 203317])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_user_total(ctx, src_root, j) usertotal = j["usertotal"] self.assertEqual(len(usertotal), 1) self.assertEqual(usertotal[0], ["2020-04-27", 43])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_topusers(ctx, src_root, j) topusers = j["topusers"] self.assertEqual(len(topusers), 20) self.assertEqual(topusers[0], ["user1", "68885"])
def test_one_element_day_range(self) -> None: """Tests the case when the day range is of just one element.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_total(ctx, src_root, j, month_range=0) monthlytotal = j["monthlytotal"] self.assertEqual(len(monthlytotal), 2) self.assertEqual(monthlytotal[0], ["2020-04", 253027]) self.assertEqual(monthlytotal[1], ["2020-05", 254651])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_progress(ctx, src_root, j) progress = j["progress"] self.assertEqual(progress["date"], "2020-05-10") # 254651 / 300 * 100 self.assertEqual(progress["percentage"], 84883.67)
def test_new_missing(self) -> None: """Tests the case when the new path is missing.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() src_root = ctx.get_abspath("workdir/stats") file_system.set_hide_paths( [os.path.join(src_root, "2020-05-10.citycount")]) ctx.set_file_system(file_system) ret = stats.get_topcities(ctx, src_root) self.assertEqual(ret, [])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} # From now on, today is 2020-05-10, so this will read 2020-04-26, 2020-04-27, etc # (till a file is missing.) stats.handle_daily_new(ctx, src_root, j) daily = j["daily"] self.assertEqual(len(daily), 1) self.assertEqual(daily[0], ["2020-04-26", 364])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_new(ctx, src_root, j) monthly = j["monthly"] self.assertEqual(len(monthly), 2) # 2019-05 start -> end self.assertEqual(monthly[0], ["2019-05", 3799]) # diff from last month end -> today self.assertEqual(monthly[1], ["2020-05", 51334])
def test_no_sleep(self) -> None: """Tests the case when no sleep is needed.""" ctx = test_context.make_test_context() routes: List[test_context.URLRoute] = [ test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt") ] network = test_context.TestNetwork(routes) ctx.set_network(network) ctx.set_time(test_context.make_test_time()) cron.overpass_sleep(ctx) test_time = cast(test_context.TestTime, ctx.get_time()) self.assertEqual(test_time.get_sleep(), 0)
def test_http_error(self) -> None: """Tests the case when we keep getting HTTP errors.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) routes: List[test_context.URLRoute] = [ test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt"), ] network = test_context.TestNetwork(routes) ctx.set_network(network) old_mtime = ctx.get_file_system().getmtime( ctx.get_abspath("workdir/stats/stats.json")) cron.update_stats(ctx, overpass=True) new_mtime = ctx.get_file_system().getmtime( ctx.get_abspath("workdir/stats/stats.json")) self.assertGreater(new_mtime, old_mtime)
def test_incomplete_last_month(self) -> None: """Tests the case when we have no data for the last, incomplete month.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} # This would be the data for the current state of the last, incomplete month. hide_path = ctx.get_abspath("workdir/stats/2020-05-10.count") file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) ctx.set_file_system(file_system) stats.handle_monthly_new(ctx, src_root, j) monthly = j["monthly"] # 1st element: 2019-05 start -> end # No 2nd element, would be diff from last month end -> today self.assertEqual(len(monthly), 1) self.assertEqual(monthly[0], ["2019-05", 3799])
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() old_citycount = b"""foo\t0 city1\t0 city2\t0 city3\t0 city4\t0 bar\t0 baz\t0 """ old_citycount_value = io.BytesIO(old_citycount) old_citycount_value.__setattr__("close", lambda: None) new_citycount = b"""foo\t1000 city1\t1000 city2\t1000 city3\t1000 city4\t1000 bar\t2 baz\t2 """ new_citycount_value = io.BytesIO(new_citycount) new_citycount_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/2020-04-10.citycount"): old_citycount_value, ctx.get_abspath("workdir/stats/2020-05-10.citycount"): new_citycount_value, } file_system.set_files(files) ctx.set_file_system(file_system) frequent_relations: Set[str] = {"foo", "bar"} parse_access_log.check_top_edited_relations(ctx, frequent_relations) self.assertIn("foo", frequent_relations) self.assertIn("city1", frequent_relations) self.assertIn("city2", frequent_relations) self.assertIn("city3", frequent_relations) self.assertIn("city4", frequent_relations) self.assertNotIn("bar", frequent_relations) self.assertNotIn("baz", frequent_relations)
def test_no_overpass(self) -> None: """Tests the case when we don't call overpass.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) routes: List[test_context.URLRoute] = [ test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-wait.txt"), test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt") ] network = test_context.TestNetwork(routes) ctx.set_network(network) cron.update_stats(ctx, overpass=False) test_time = cast(test_context.TestTime, ctx.get_time()) self.assertEqual(test_time.get_sleep(), 0)
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() today_citycount = b"""budapest_01\t100 budapest_02\t200 \t42 """ today_citycount_value = io.BytesIO(today_citycount) today_citycount_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/2020-05-10.citycount"): today_citycount_value, } file_system.set_files(files) ctx.set_file_system(file_system) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_topcities(ctx, src_root, j) topcities = j["topcities"] self.assertEqual(len(topcities), 2) self.assertEqual(topcities[0], ("budapest_02", 190)) self.assertEqual(topcities[1], ("budapest_01", 90))