def test__query_should_call_database_query_correctly(self):
repo = DataRepository()
repo.database = MagicMock()
repo._query("2020-02-20T17:00:15", "2020-02-20T18:00:15", "orange_market")
repo.database.query.assert_called_once_with(
"2020-02-20 17:00:15", "2020-02-20 18:00:15", "orange_market"
)
def test_get_data_should_return_data_when_database_data_not_enough(self, mock_to_end_min):
repo = DataRepository()
mock_to_end_min.return_value = [
("2020-03-20T00:00:00", "2020-03-21T00:00:00", 10),
]
repo.database = MagicMock()
repo.database.query.return_value = []
repo._fetch_from_upbit = MagicMock(
return_value=[
{"content": "mango", "date_time": "2020-03-20T00:00:00"},
{"content": "banana", "date_time": "2020-03-20 00:01:00"},
]
)
result = repo.get_data("2020-02-20T17:00:15", "2020-02-20T22:00:15", "mango")
self.assertEqual(
result,
[
{"content": "mango", "date_time": "2020-03-20T00:00:00"},
{"content": "banana", "date_time": "2020-03-20T00:01:00"},
],
)
mock_to_end_min.assert_called_once_with(
start_iso="2020-02-20T17:00:15", end_iso="2020-02-20T22:00:15"
)
repo._fetch_from_upbit.assert_called_once_with(
"2020-02-20T17:00:15", "2020-02-20T22:00:15", "mango"
)
def test__fetch_from_upbit_should_call__fetch_from_upbit_up_to_200(self, mock_to_end_min):
start = "2020-03-20T00:00:00"
end = "2020-03-21T00:00:00"
repo = DataRepository()
repo._query = MagicMock(side_effect=[["mango"], ["apple"], ["orange"]])
repo._is_equal = MagicMock()
repo._update = MagicMock()
repo.verify_mode = False
repo._fetch_from_upbit_up_to_200 = MagicMock(side_effect=[["kiwi"]])
mock_to_end_min.return_value = [
("2020-03-20T00:00:00", "2020-03-21T00:10:00", 1),
("2020-03-20T00:10:00", "2020-03-21T00:20:00", 1),
("2020-03-20T00:20:00", "2020-03-21T00:30:00", 2),
]
repo._recovery_upbit_data = MagicMock(side_effect=[["orange"]])
result = repo._fetch_from_upbit(start, end, "mango_market")
mock_to_end_min.assert_called_once_with(start_iso=start, end_iso=end, max_count=200)
self.assertEqual(repo._query.call_args_list[0][0][0], "2020-03-20T00:00:00")
self.assertEqual(repo._query.call_args_list[1][0][0], "2020-03-20T00:10:00")
self.assertEqual(repo._query.call_args_list[2][0][0], "2020-03-20T00:20:00")
self.assertEqual(repo._query.call_args_list[0][0][1], "2020-03-21T00:10:00")
self.assertEqual(repo._query.call_args_list[1][0][1], "2020-03-21T00:20:00")
self.assertEqual(repo._query.call_args_list[2][0][1], "2020-03-21T00:30:00")
repo._fetch_from_upbit_up_to_200.assert_called_once_with(
"2020-03-21T00:30:00", 2, "mango_market"
)
repo._update.assert_called_once_with(["orange"])
self.assertEqual(result, ["mango", "apple", "orange"])
repo._recovery_upbit_data.assert_called_once_with(
["kiwi"], "2020-03-20T00:20:00", 2, "mango_market"
)
def test__fetch_from_upbit_up_to_200_NOT_throw_UserWarning_when_receive_invalid_data(
self, mock_get
):
end = "2020-03-20T00:00:00"
repo = DataRepository()
dummy_response = MagicMock()
dummy_response.json.side_effect = ValueError()
mock_get.return_value = dummy_response
with self.assertRaises(UserWarning):
data = repo._fetch_from_upbit_up_to_200(end, 200, "mango")
self.assertIsNone(data)
def test__update_should_call_database_update_correctly(self):
dummy_data = [
{"market": "mango", "date_time": "2020-02-20T17:00:15"},
{"market": "apple", "date_time": "2020-02-20T18:00:15"},
]
expected_data = [
{"market": "mango", "date_time": "2020-02-20 17:00:15"},
{"market": "apple", "date_time": "2020-02-20 18:00:15"},
]
repo = DataRepository()
repo.database = MagicMock()
repo._update(dummy_data)
repo.database.update.assert_called_once_with(expected_data)
def test__fetch_from_upbit_up_to_200_should_call_get_correctly(self, mock_get):
dummy_response = MagicMock()
expected_value = [
{
"market": "mango",
"candle_date_time_utc": "2020-03-10T13:52:00",
"candle_date_time_kst": "2020-03-10T22:52:00",
"opening_price": 9777000.00000000,
"high_price": 9778000.00000000,
"low_price": 9763000.00000000,
"trade_price": 9778000.00000000,
"timestamp": 1583848334534,
"candle_acc_trade_price": 11277224.71063000,
"candle_acc_trade_volume": 1.15377852,
"unit": 1,
},
{
"market": "mango",
"candle_date_time_utc": "2020-03-10T13:52:00",
"candle_date_time_kst": "2020-03-10T22:52:00",
"opening_price": 8777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"trade_price": 8778000.00000000,
"timestamp": 1583848334534,
"candle_acc_trade_price": 11277224.71063000,
"candle_acc_trade_volume": 1.15377852,
"unit": 1,
},
]
dummy_response.json.return_value = expected_value
mock_get.return_value = dummy_response
end = "2020-03-20T00:00:00"
repo = DataRepository()
data = repo._fetch_from_upbit_up_to_200(end, 41760, "mango")
mock_get.assert_called_once_with(
"https://api.upbit.com/v1/candles/minutes/1",
params={"market": "mango", "to": "2020-03-19T15:00:00Z", "count": 41760},
)
self.assertEqual(len(data), 2)
self.assertEqual(data[0]["opening_price"], 8777000.00000000)
self.assertEqual(data[0]["high_price"], 8778000.00000000)
self.assertEqual(data[0]["low_price"], 8763000.00000000)
self.assertEqual(data[0]["closing_price"], 8778000.00000000)
self.assertEqual(data[1]["opening_price"], 9777000.00000000)
self.assertEqual(data[1]["high_price"], 9778000.00000000)
self.assertEqual(data[1]["low_price"], 9763000.00000000)
self.assertEqual(data[1]["closing_price"], 9778000.00000000)
def test__fetch_from_upbit_up_to_200_NOT_throw_UserWarning_when_receive_response_error(
self, mock_get
):
end = "2020-03-20T00:00:00"
repo = DataRepository()
dummy_response = MagicMock()
dummy_response.json.return_value = [{"market": "apple"}, {"market": "banana"}]
dummy_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
"HTTPError dummy exception"
)
mock_get.return_value = dummy_response
with self.assertRaises(UserWarning):
data = repo._fetch_from_upbit_up_to_200(end, 200, "mango")
self.assertIsNone(data)
def test__is_equal_should_return_correct_judgement(self):
dummy_data_a = [
{"market": "mango", "date_time": "2020-02-20T17:00:15", "period": 30, "recovered": 0},
{"market": "apple", "date_time": "2020-02-20T18:00:15", "period": 30, "recovered": 0},
{"market": "kiwi", "date_time": "2020-02-20T19:00:15", "period": 30, "recovered": 0},
]
dummy_data_b = [
{"market": "mango", "date_time": "2020-02-20T17:00:15", "period": 30, "recovered": 0},
{"market": "apple", "date_time": "2020-02-20T18:00:15", "period": 30, "recovered": 0},
]
dummy_data_c = [
{"market": "mango", "date_time": "2020-02-20T17:00:15"},
{"market": "apple", "date_time": "2020-02-20T18:00:15"},
]
dummy_data_d = [
{"market": "mango", "date_time": "2020-02-20T17:00:15", "period": 30},
{"market": "apple", "date_time": "2020-02-20T18:00:15", "period": 30},
{"market": "kiwi", "date_time": "2020-02-20T19:00:15", "period": 30},
]
self.assertFalse(DataRepository._is_equal(dummy_data_a, dummy_data_b))
self.assertTrue(DataRepository._is_equal(dummy_data_b, dummy_data_c))
self.assertFalse(DataRepository._is_equal(dummy_data_b, dummy_data_d))
def test_ITG_data_repository_fetch_and_verify_with_random_period(self):
repo = DataRepository("test.db")
dt_list = []
base_datetime = [
"2019-01-01T00:00:00",
"2019-03-01T00:00:00",
"2019-05-01T00:00:00",
"2019-07-01T00:00:00",
"2019-09-01T00:00:00",
"2019-11-01T00:00:00",
"2020-01-01T00:00:00",
"2020-03-01T00:00:00",
"2020-05-01T00:00:00",
"2020-07-01T00:00:00",
"2020-09-01T00:00:00",
"2020-11-01T00:00:00",
"2021-01-01T00:00:00",
"2019-03-01T00:00:00",
"2020-05-01T00:00:00",
"2020-07-01T00:00:00",
"2021-01-01T00:00:00",
]
for item in base_datetime:
dt_list.append(self._get_random_period(item))
for dt in dt_list:
print(f"### Check {dt[0]} to {dt[1]}, count: {dt[2]}")
result = repo.get_data(dt[0], dt[1], "KRW-BTC")
broken_list = []
for i in result:
if "recovered" in i and i["recovered"] > 0:
broken_list.append(i)
print(f"== from repo with broken data {len(broken_list)}")
upbit = self._fetch_from_upbit_up_to_200(dt[1], dt[2], "KRW-BTC")
checked = self._check_equal(result, upbit)
self.assertTrue(checked + len(broken_list), dt[2])
time.sleep(1)
def test__recovery_upbit_data_should_return_recovered_data(self):
repo = DataRepository()
broken_data_0355_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11546000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:53:00",
"closing_price": 11546000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:54:00",
"closing_price": 11545000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:56:00",
"closing_price": 11561000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_0355_skipped, "2020-02-22T03:53:00", 4, "mango"
)
repo._report_broken_block.assert_called_once_with("2020-02-22T03:55:00", "mango")
self.assertEqual(len(recovered), 4)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:53:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[2]["date_time"], "2020-02-22T03:55:00")
self.assertEqual(recovered[3]["date_time"], "2020-02-22T03:56:00")
self.assertEqual(recovered[2]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11546000)
self.assertEqual(recovered[1]["closing_price"], 11545000)
self.assertEqual(recovered[2]["closing_price"], 11545000)
self.assertEqual(recovered[3]["closing_price"], 11561000)
broken_data_2_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:53:00",
"closing_price": 11546000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:54:00",
"closing_price": 11545000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:57:00",
"closing_price": 11561000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_2_skipped, "2020-02-22T03:54:00", 4, "mango"
)
self.assertEqual(repo._report_broken_block.call_args_list[0][0][0], "2020-02-22T03:55:00")
self.assertEqual(repo._report_broken_block.call_args_list[1][0][0], "2020-02-22T03:56:00")
self.assertEqual(len(recovered), 4)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:55:00")
self.assertEqual(recovered[2]["date_time"], "2020-02-22T03:56:00")
self.assertEqual(recovered[3]["date_time"], "2020-02-22T03:57:00")
self.assertEqual(recovered[1]["recovered"], 1)
self.assertEqual(recovered[2]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11545000)
self.assertEqual(recovered[1]["closing_price"], 11545000)
self.assertEqual(recovered[2]["closing_price"], 11545000)
self.assertEqual(recovered[3]["closing_price"], 11561000)
broken_data_2_skipped2 = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:53:00",
"closing_price": 11546000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:55:00",
"closing_price": 11545000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:57:00",
"closing_price": 11561000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_2_skipped2, "2020-02-22T03:54:00", 4, "mango"
)
self.assertEqual(repo._report_broken_block.call_args_list[0][0][0], "2020-02-22T03:54:00")
self.assertEqual(repo._report_broken_block.call_args_list[1][0][0], "2020-02-22T03:56:00")
self.assertEqual(len(recovered), 4)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:55:00")
self.assertEqual(recovered[2]["date_time"], "2020-02-22T03:56:00")
self.assertEqual(recovered[3]["date_time"], "2020-02-22T03:57:00")
self.assertEqual(recovered[0]["recovered"], 1)
self.assertEqual(recovered[2]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11546000)
self.assertEqual(recovered[1]["closing_price"], 11545000)
self.assertEqual(recovered[2]["closing_price"], 11545000)
self.assertEqual(recovered[3]["closing_price"], 11561000)
broken_data_first_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:54:00",
"closing_price": 11546000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_first_skipped, "2020-02-22T03:53:00", 2, "mango"
)
repo._report_broken_block.assert_called_with("2020-02-22T03:53:00", "mango")
self.assertEqual(len(recovered), 2)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:53:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[0]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11547000)
self.assertEqual(recovered[1]["closing_price"], 11546000)
broken_data_first_2_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:51:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11546000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_first_2_skipped, "2020-02-22T03:53:00", 2, "mango"
)
self.assertEqual(repo._report_broken_block.call_args_list[0][0][0], "2020-02-22T03:53:00")
self.assertEqual(repo._report_broken_block.call_args_list[1][0][0], "2020-02-22T03:54:00")
self.assertEqual(len(recovered), 2)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:53:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[0]["recovered"], 1)
self.assertEqual(recovered[1]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11546000)
self.assertEqual(recovered[1]["closing_price"], 11546000)
broken_data_last_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:53:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:54:00",
"closing_price": 11546000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_last_skipped, "2020-02-22T03:54:00", 2, "mango"
)
repo._report_broken_block.assert_called_with("2020-02-22T03:55:00", "mango")
self.assertEqual(len(recovered), 2)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:55:00")
self.assertEqual(recovered[1]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11546000)
self.assertEqual(recovered[1]["closing_price"], 11546000)
broken_data_last_2_skipped = [
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:52:00",
"closing_price": 11542000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:53:00",
"closing_price": 11547000.0,
},
{
"market": "KRW-BTC",
"date_time": "2020-02-22T03:54:00",
"closing_price": 11546000.0,
},
]
repo._report_broken_block = MagicMock()
recovered = repo._recovery_upbit_data(
broken_data_last_skipped, "2020-02-22T03:54:00", 3, "mango"
)
self.assertEqual(repo._report_broken_block.call_args_list[0][0][0], "2020-02-22T03:55:00")
self.assertEqual(repo._report_broken_block.call_args_list[1][0][0], "2020-02-22T03:56:00")
self.assertEqual(len(recovered), 3)
self.assertEqual(recovered[0]["date_time"], "2020-02-22T03:54:00")
self.assertEqual(recovered[1]["date_time"], "2020-02-22T03:55:00")
self.assertEqual(recovered[2]["date_time"], "2020-02-22T03:56:00")
self.assertEqual(recovered[1]["recovered"], 1)
self.assertEqual(recovered[2]["recovered"], 1)
self.assertEqual(recovered[0]["closing_price"], 11546000)
self.assertEqual(recovered[1]["closing_price"], 11546000)
self.assertEqual(recovered[2]["closing_price"], 11546000)