async def test_conomy_loader_wrong_name():
"""Исключение при неверном наименование таблицы."""
loader = conomy.ConomyLoader()
table_name = outer.TableName(outer.SECURITIES, "DSKY")
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
await loader.get(table_name)
def dohod(ticker: str) -> pd.DataFrame:
"""Информация по дивидендам с dohod.ru."""
table_name = outer.TableName(outer.DOHOD, ticker)
requests_handler = bootstrap.get_handler()
df = requests_handler.get_df(table_name)
start_date = bootstrap.get_start_date()
return df.loc[start_date:] # type: ignore
async def test_loader(mocker):
"""Группировка и сортировка полученных данных."""
mocker.patch.object(dohod.parser, "get_df_from_url", return_value=DF)
loader = dohod.DohodLoader()
table_name = outer.TableName(outer.DOHOD, "BELU")
pd.testing.assert_frame_equal(await loader.get(table_name), DF_REZ)
def cpi() -> pd.Series:
"""Потребительская инфляция."""
table_name = outer.TableName(outer.CPI, outer.CPI)
requests_handler = bootstrap.get_handler()
df = requests_handler.get_df(table_name)
start_date = bootstrap.get_start_date()
return df.loc[start_date:, col.CPI] # type: ignore
def dividends(ticker: str, force_update: bool = False) -> pd.DataFrame:
"""Дивиденды для данного тикера."""
table_name = outer.TableName(outer.DIVIDENDS, ticker)
requests_handler = bootstrap.get_handler()
df = requests_handler.get_df(table_name, force_update)
start_date = bootstrap.get_start_date()
return df.loc[start_date:] # type: ignore
def index() -> pd.Series:
"""Загрузка данных по индексу полной доходности с учетом российских налогов - MCFTRR."""
table_name = outer.TableName(outer.INDEX, outer.INDEX)
requests_handler = bootstrap.get_handler()
df = requests_handler.get_df(table_name)
start_date = bootstrap.get_start_date()
return df.loc[start_date:, col.CLOSE] # type: ignore
async def test_loader_first_load(mocker):
"""Вариант загрузки без начальной датой."""
fake_securities_loader = mocker.AsyncMock()
mocker.patch.object(moex, "_find_aliases", return_value=["a", "b"])
mocker.patch.object(moex.aiomoex,
"get_market_candles",
side_effect=[JSON1, JSON2])
loader = moex.QuotesLoader(fake_securities_loader)
table_name = outer.TableName(outer.QUOTES, "AKRN")
df_rez = await loader.get(table_name)
assert df_rez.columns.tolist() == [
col.OPEN,
col.CLOSE,
col.HIGH,
col.LOW,
col.TURNOVER,
]
assert df_rez.index.tolist() == [
pd.Timestamp("2011-09-27"),
pd.Timestamp("2011-09-28"),
pd.Timestamp("2011-09-29"),
]
assert df_rez[col.CLOSE].tolist() == [
2,
8,
3,
]
async def test_securities_loader_from_cache(mocker, caplog):
"""Проверка повторной загрузки из кэша."""
caplog.set_level(logging.INFO)
fake_get_board_securities = mocker.patch.object(
moex.aiomoex,
"get_board_securities",
return_value=DF_SEC,
)
loader = moex.SecuritiesLoader()
table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES)
df_rez1 = await loader.get(table_name)
fake_get_board_securities.assert_called_once()
await asyncio.sleep(0.01)
assert len(caplog.record_tuples) == 1
assert "Загрузка из кэша" not in caplog.record_tuples[0][-1]
df_rez2 = await loader.get(table_name)
fake_get_board_securities.assert_called_once()
pd.testing.assert_frame_equal(df_rez1, df_rez2)
await asyncio.sleep(0.01)
assert len(caplog.record_tuples) == 3
assert "Загрузка из кэша" in caplog.record_tuples[2][-1]
async def _first_load(self, http_session: aiohttp.ClientSession,
ticker: str) -> pd.DataFrame:
"""Первая загрузка - поиск старых тикеров по регистрационному номеру и объединение рядов."""
table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES)
df = await self._securities_loader.get(table_name)
reg_num = df.at[ticker, col.REG_NUMBER]
aliases = await _find_aliases(http_session, reg_num)
return await _download_many(http_session, aliases)
async def test_loader_raise_on_wrong_name():
"""Не верное название таблицы."""
table_name = outer.TableName(outer.CPI, "AKRN")
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
loader = dohod.DohodLoader()
await loader.get(table_name)
async def test_conomy_loader(mocker):
"""Группировка и сортировка полученных данных."""
mocker.patch.object(conomy, "_get_html")
mocker.patch.object(conomy, "_get_col_desc")
mocker.patch.object(conomy.parser, "get_df_from_html", return_value=DF)
loader = conomy.ConomyLoader()
table_name = outer.TableName(outer.CONOMY, "BELU")
pd.testing.assert_frame_equal(await loader.get(table_name), DF_REZ)
async def test_loader_logger_mixin_raises(caplog):
"""Исключение в случае не совпадения названия таблицы и группы."""
caplog.set_level(logging.INFO)
mixin = logger.LoaderLoggerMixin()
table_name = outer.TableName(outer.QUOTES, "YAKG")
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
mixin._log_and_validate_group(table_name, outer.SECURITIES)
def get_dfs(
self,
group: outer.GroupName,
names: Tuple[str, ...],
) -> Tuple[pd.DataFrame, ...]:
"""Возвращает несколько DataFrame из одной группы."""
table_names = [outer.TableName(group, name) for name in names]
commands: List[events.Command] = [events.GetDataFrame(table_name) for table_name in table_names]
result_dict = self._run_commands(commands)
return tuple(result_dict[name] for name in table_names)
def lot_size(tickers: Tuple[str, ...]) -> pd.Series:
"""Информация о размере лотов для тикеров.
:param tickers:
Перечень тикеров, для которых нужна информация.
:return:
Информация о размере лотов.
"""
table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES)
requests_handler = bootstrap.get_handler()
df = requests_handler.get_df(table_name)
return df.loc[list(tickers), col.LOT_SIZE]
async def test_conomy_loader(mocker, df_patch, df_res):
"""Проверка нижних ячеек и отбрасывание предварительных данных."""
mocker.patch.object(smart_lab.parser,
"get_df_from_url",
return_value=df_patch)
loader = smart_lab.SmartLabLoader()
table_name = outer.TableName(outer.SMART_LAB, outer.SMART_LAB)
if df_res is None:
with pytest.raises(outer.DataError, match="Некорректная html-таблица"):
await loader.get(table_name)
else:
pd.testing.assert_frame_equal(await loader.get(table_name), df_res)
async def test_index_loader(mocker):
"""Форматирование загруженных данных по индексу акциям."""
df = pd.DataFrame([dict(dd="2020-09-29", vv=3000.0)])
mocker.patch.object(moex.aiomoex, "get_board_history", return_value=df)
loader = moex.IndexLoader()
table_name = outer.TableName(outer.INDEX, outer.INDEX)
df_rez = await loader.get(table_name)
assert df_rez.columns.tolist() == [col.CLOSE]
assert df_rez.index.tolist() == [pd.Timestamp("2020-09-29")]
assert df_rez.values.tolist() == [[3000.0]]
async def test_securities_loader(mocker):
"""Форматирование загруженных данных по торгуемым акциям."""
mocker.patch.object(moex.aiomoex,
"get_board_securities",
return_value=DF_SEC)
loader = moex.SecuritiesLoader()
table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES)
df_rez = await loader.get(table_name)
assert df_rez.columns.tolist() == [col.REG_NUMBER, col.LOT_SIZE]
assert df_rez.index.tolist() == ["GAZP"]
assert df_rez.values.tolist() == [["abc", 12]]
async def test_loader(mocker):
"""Основной вариант работы загрузчика."""
fake_load_xlsx = mocker.patch.object(cpi, "_load_xlsx")
fake_validate = mocker.patch.object(cpi, "_validate")
fake_clean_up = mocker.patch.object(cpi, "_clean_up")
loader = cpi.CPILoader()
table_name = outer.TableName(outer.CPI, outer.CPI)
assert await loader.get(table_name) is fake_clean_up.return_value
fake_load_xlsx.assert_called_once_with()
fake_validate.assert_called_once_with(fake_load_xlsx.return_value)
fake_clean_up.assert_called_once_with(fake_load_xlsx.return_value)
async def test_loader_logger_mixin(caplog):
"""Логирование загрузки."""
caplog.set_level(logging.INFO)
mixin = logger.LoaderLoggerMixin()
table_name = outer.TableName(outer.QUOTES, "YAKG")
assert mixin._log_and_validate_group(table_name, outer.QUOTES) == "YAKG"
await asyncio.sleep(0.01)
assert caplog.record_tuples == [
(
"LoaderLoggerMixin",
20,
"Загрузка TableName(group='quotes', name='YAKG')",
),
]
"""Тесты загрузки таблицы с диапазоном торговых дат."""
import asyncio
import logging
import random
import pandas as pd
import pytest
from poptimizer.data.adapters.loaders import trading_dates
from poptimizer.data.config import resources
from poptimizer.data.ports import outer
LOGGER_CLASS_NAME = "TradingDatesLoader"
TABLE_NAME = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES)
JSON = ({"from": "1997-03-24", "till": "2020-09-22"}, )
DF = pd.DataFrame(JSON, dtype="datetime64[ns]")
NAMES_CASES = (
outer.TableName(outer.TRADING_DATES, "test"),
outer.TableName(outer.QUOTES, outer.TRADING_DATES),
)
@pytest.mark.parametrize("table_name", NAMES_CASES)
@pytest.mark.asyncio
async def test_raise_on_wrong_name(table_name):
"""Не верное название данных."""
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
loader = trading_dates.TradingDatesLoader()
await loader.get(table_name)
@pytest.mark.parametrize("now, end", POTENTIAL_TRADING_DAY_CASES)
def test_trading_day_potential_end(now, end, monkeypatch):
"""Тестирование двух краевых случаев на стыке потенциального окончания торгового дня."""
monkeypatch.setattr(services, "datetime", FakeDateTime(now))
assert services.trading_day_potential_end() == end
def test_day_real_end():
"""Тест на окончание реального торгового дня."""
df = pd.DataFrame([datetime(2020, 9, 11)], columns=["till"])
assert services.trading_day_real_end(df) == datetime(2020, 9, 11, 21, 45)
MAIN_HELPER = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES)
HELPER_NAME_CASES = (
(outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES), None),
(outer.TableName(outer.CONOMY, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.DOHOD, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.SMART_LAB, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.DIVIDENDS, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.CPI, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.SECURITIES, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.INDEX, outer.TRADING_DATES), MAIN_HELPER),
(outer.TableName(outer.QUOTES, outer.TRADING_DATES), MAIN_HELPER),
)
@pytest.mark.parametrize("name, answer", HELPER_NAME_CASES)
def test_get_helper_name(name, answer):
import pandas as pd
import pytest
from poptimizer.data.adapters import db
from poptimizer.data.config import resources
from poptimizer.data.ports import outer
DF = pd.DataFrame(
[[5, 6], [6, 7]],
columns=["s", "f"],
index=["r", "g"],
)
TIMESTAMP = datetime.utcnow()
TIMESTAMP = TIMESTAMP.replace(microsecond=(TIMESTAMP.microsecond // 1000) *
1000)
TABLE_NAME = outer.TableName(outer.SECURITIES, outer.SECURITIES)
TABLE_TUPLE = outer.TableTuple(outer.SECURITIES, outer.SECURITIES, DF,
TIMESTAMP)
NAME_CASES = (
[
outer.TableTuple(outer.QUOTES, "VTBR", DF, TIMESTAMP), outer.QUOTES,
"VTBR"
],
[outer.TableName(outer.QUOTES, "HYDR"), outer.QUOTES, "HYDR"],
[TABLE_TUPLE, db.MISC, outer.SECURITIES],
[TABLE_NAME, db.MISC, outer.SECURITIES],
)
@pytest.mark.parametrize("table_name, collection, name", NAME_CASES)
"""Тесты для загрузки данных с MOEX."""
import asyncio
import logging
from datetime import datetime
import pandas as pd
import pytest
from poptimizer.data.adapters.loaders import moex
from poptimizer.data.config import resources
from poptimizer.data.ports import col, outer
# flake8: noqa
NAMES_CASES = (
(moex.SecuritiesLoader(), outer.TableName(outer.SECURITIES, "test")),
(moex.SecuritiesLoader(), outer.TableName(outer.QUOTES, outer.SECURITIES)),
(moex.IndexLoader(), outer.TableName(outer.INDEX, "test")),
(moex.IndexLoader(), outer.TableName(outer.QUOTES, outer.INDEX)),
(moex.QuotesLoader(moex.SecuritiesLoader()),
outer.TableName(outer.SECURITIES, "AKRN")),
)
@pytest.mark.parametrize("loader, table_name", NAMES_CASES)
@pytest.mark.asyncio
async def test_loader_raise_on_wrong_name(loader, table_name):
"""Не верное название таблицы."""
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
await loader.get(table_name)
def recreate_table(table_tuple: outer.TableTuple) -> model.Table:
"""Создает таблицу на основе данных и обновляет ее."""
name = outer.TableName(table_tuple.group, table_tuple.name)
return model.Table(name, _TABLES_REGISTRY[name.group], table_tuple.df,
table_tuple.timestamp)
df_clean = cpi._clean_up(df)
assert df_clean.values.tolist() == [
[1.0],
[3.0],
[2.0],
[4.0],
]
assert df_clean.columns == [col.CPI]
assert df_clean.index[0] == pd.Timestamp("1992-01-31")
assert df_clean.index[-1] == pd.Timestamp("1992-04-30")
NAMES_CASES = (
outer.TableName(outer.CPI, "test"),
outer.TableName(outer.QUOTES, outer.CPI),
)
@pytest.mark.parametrize("table_name", NAMES_CASES)
@pytest.mark.asyncio
async def test_loader_raise_on_wrong_name(table_name):
"""Не верное название таблицы."""
with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"):
loader = cpi.CPILoader()
await loader.get(table_name)
@pytest.mark.asyncio
async def test_loader(mocker):
"""Тесты для загрузки данных с https://www.smart-lab.ru."""
import pandas as pd
import pytest
from poptimizer.data.adapters.loaders import smart_lab
from poptimizer.data.ports import outer
NAMES_CASES = (
outer.TableName(outer.SMART_LAB, "test"),
outer.TableName(outer.QUOTES, outer.SMART_LAB),
)
@pytest.mark.parametrize("table_name", NAMES_CASES)
@pytest.mark.asyncio
async def test_loader_raise_on_wrong_name(table_name):
"""Не верное название таблицы."""
with pytest.raises(outer.DataError,
match="Некорректное имя таблицы для обновления"):
loader = smart_lab.SmartLabLoader()
await loader.get(table_name)
DF = pd.DataFrame(
[[4.0], [1.0], [2.0], [None]],
index=["2020-01-20", "2014-11-25", "2014-11-25", smart_lab.FOOTER],
)
DF_CASES = ((DF, DF.dropna()), (DF.dropna(), None))
@pytest.mark.parametrize("df_patch, df_res", DF_CASES)
def smart_lab() -> pd.DataFrame:
"""Информация по дивидендам с smart-lab.ru."""
table_name = outer.TableName(outer.SMART_LAB, outer.SMART_LAB)
requests_handler = bootstrap.get_handler()
return requests_handler.get_df(table_name)
"""Тесты событий, связанных с обновлением таблиц."""
import asyncio
from datetime import datetime
import pytest
from poptimizer.data.domain import events, services
from poptimizer.data.ports import outer
HELPER_NAME = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES)
USUAL_NAME = outer.TableName(outer.QUOTES, "SNGSP")
FAKE_END_OF_TRADING_DAY = datetime(2020, 9, 15, 15, 58)
GET_DATA_FRAME_CASES = (
(
(USUAL_NAME, True),
events.UpdateTable,
USUAL_NAME,
"_end_of_trading_day",
None,
),
(
(USUAL_NAME, False),
events.GetEndOfTradingDay,
HELPER_NAME,
"_table_name",
USUAL_NAME,
),
(
(HELPER_NAME, False),
events.UpdateTable,
"""Тесты для фабрик по созданию таблиц и их сериализации."""
from datetime import datetime
import pandas as pd
import pytest
from poptimizer.data.domain import factories, model
from poptimizer.data.ports import outer
TABLE_NAME = outer.TableName(outer.QUOTES, "VSMO")
TABLE_VARS = outer.TableTuple(
TABLE_NAME.group,
TABLE_NAME.name,
pd.DataFrame([1]),
datetime.utcnow(),
)
def test_create_table():
"""У новой таблицы не должно быть данных и времени обновления."""
table = factories.create_table(TABLE_NAME)
assert isinstance(table, model.Table)
assert table.df is None
assert table.timestamp is None
def test_recreate_table():
"""У таблицы должны быть переданные значения данных и времени обновления.
Данные должны возвращаться копией.
def get_helper_name(name: outer.TableName) -> Optional[outer.TableName]:
"""Имя вспомогательной таблицы."""
if name.group != outer.TRADING_DATES:
return outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES)
return None
