async def test_conomy_loader_wrong_name(): """Исключение при неверном наименование таблицы.""" loader = conomy.ConomyLoader() table_name = outer.TableName(outer.SECURITIES, "DSKY") with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): await loader.get(table_name)
def dohod(ticker: str) -> pd.DataFrame: """Информация по дивидендам с dohod.ru.""" table_name = outer.TableName(outer.DOHOD, ticker) requests_handler = bootstrap.get_handler() df = requests_handler.get_df(table_name) start_date = bootstrap.get_start_date() return df.loc[start_date:] # type: ignore
async def test_loader(mocker): """Группировка и сортировка полученных данных.""" mocker.patch.object(dohod.parser, "get_df_from_url", return_value=DF) loader = dohod.DohodLoader() table_name = outer.TableName(outer.DOHOD, "BELU") pd.testing.assert_frame_equal(await loader.get(table_name), DF_REZ)
def cpi() -> pd.Series: """Потребительская инфляция.""" table_name = outer.TableName(outer.CPI, outer.CPI) requests_handler = bootstrap.get_handler() df = requests_handler.get_df(table_name) start_date = bootstrap.get_start_date() return df.loc[start_date:, col.CPI] # type: ignore
def dividends(ticker: str, force_update: bool = False) -> pd.DataFrame: """Дивиденды для данного тикера.""" table_name = outer.TableName(outer.DIVIDENDS, ticker) requests_handler = bootstrap.get_handler() df = requests_handler.get_df(table_name, force_update) start_date = bootstrap.get_start_date() return df.loc[start_date:] # type: ignore
def index() -> pd.Series: """Загрузка данных по индексу полной доходности с учетом российских налогов - MCFTRR.""" table_name = outer.TableName(outer.INDEX, outer.INDEX) requests_handler = bootstrap.get_handler() df = requests_handler.get_df(table_name) start_date = bootstrap.get_start_date() return df.loc[start_date:, col.CLOSE] # type: ignore
async def test_loader_first_load(mocker): """Вариант загрузки без начальной датой.""" fake_securities_loader = mocker.AsyncMock() mocker.patch.object(moex, "_find_aliases", return_value=["a", "b"]) mocker.patch.object(moex.aiomoex, "get_market_candles", side_effect=[JSON1, JSON2]) loader = moex.QuotesLoader(fake_securities_loader) table_name = outer.TableName(outer.QUOTES, "AKRN") df_rez = await loader.get(table_name) assert df_rez.columns.tolist() == [ col.OPEN, col.CLOSE, col.HIGH, col.LOW, col.TURNOVER, ] assert df_rez.index.tolist() == [ pd.Timestamp("2011-09-27"), pd.Timestamp("2011-09-28"), pd.Timestamp("2011-09-29"), ] assert df_rez[col.CLOSE].tolist() == [ 2, 8, 3, ]
async def test_securities_loader_from_cache(mocker, caplog): """Проверка повторной загрузки из кэша.""" caplog.set_level(logging.INFO) fake_get_board_securities = mocker.patch.object( moex.aiomoex, "get_board_securities", return_value=DF_SEC, ) loader = moex.SecuritiesLoader() table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES) df_rez1 = await loader.get(table_name) fake_get_board_securities.assert_called_once() await asyncio.sleep(0.01) assert len(caplog.record_tuples) == 1 assert "Загрузка из кэша" not in caplog.record_tuples[0][-1] df_rez2 = await loader.get(table_name) fake_get_board_securities.assert_called_once() pd.testing.assert_frame_equal(df_rez1, df_rez2) await asyncio.sleep(0.01) assert len(caplog.record_tuples) == 3 assert "Загрузка из кэша" in caplog.record_tuples[2][-1]
async def _first_load(self, http_session: aiohttp.ClientSession, ticker: str) -> pd.DataFrame: """Первая загрузка - поиск старых тикеров по регистрационному номеру и объединение рядов.""" table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES) df = await self._securities_loader.get(table_name) reg_num = df.at[ticker, col.REG_NUMBER] aliases = await _find_aliases(http_session, reg_num) return await _download_many(http_session, aliases)
async def test_loader_raise_on_wrong_name(): """Не верное название таблицы.""" table_name = outer.TableName(outer.CPI, "AKRN") with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): loader = dohod.DohodLoader() await loader.get(table_name)
async def test_conomy_loader(mocker): """Группировка и сортировка полученных данных.""" mocker.patch.object(conomy, "_get_html") mocker.patch.object(conomy, "_get_col_desc") mocker.patch.object(conomy.parser, "get_df_from_html", return_value=DF) loader = conomy.ConomyLoader() table_name = outer.TableName(outer.CONOMY, "BELU") pd.testing.assert_frame_equal(await loader.get(table_name), DF_REZ)
async def test_loader_logger_mixin_raises(caplog): """Исключение в случае не совпадения названия таблицы и группы.""" caplog.set_level(logging.INFO) mixin = logger.LoaderLoggerMixin() table_name = outer.TableName(outer.QUOTES, "YAKG") with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): mixin._log_and_validate_group(table_name, outer.SECURITIES)
def get_dfs( self, group: outer.GroupName, names: Tuple[str, ...], ) -> Tuple[pd.DataFrame, ...]: """Возвращает несколько DataFrame из одной группы.""" table_names = [outer.TableName(group, name) for name in names] commands: List[events.Command] = [events.GetDataFrame(table_name) for table_name in table_names] result_dict = self._run_commands(commands) return tuple(result_dict[name] for name in table_names)
def lot_size(tickers: Tuple[str, ...]) -> pd.Series: """Информация о размере лотов для тикеров. :param tickers: Перечень тикеров, для которых нужна информация. :return: Информация о размере лотов. """ table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES) requests_handler = bootstrap.get_handler() df = requests_handler.get_df(table_name) return df.loc[list(tickers), col.LOT_SIZE]
async def test_conomy_loader(mocker, df_patch, df_res): """Проверка нижних ячеек и отбрасывание предварительных данных.""" mocker.patch.object(smart_lab.parser, "get_df_from_url", return_value=df_patch) loader = smart_lab.SmartLabLoader() table_name = outer.TableName(outer.SMART_LAB, outer.SMART_LAB) if df_res is None: with pytest.raises(outer.DataError, match="Некорректная html-таблица"): await loader.get(table_name) else: pd.testing.assert_frame_equal(await loader.get(table_name), df_res)
async def test_index_loader(mocker): """Форматирование загруженных данных по индексу акциям.""" df = pd.DataFrame([dict(dd="2020-09-29", vv=3000.0)]) mocker.patch.object(moex.aiomoex, "get_board_history", return_value=df) loader = moex.IndexLoader() table_name = outer.TableName(outer.INDEX, outer.INDEX) df_rez = await loader.get(table_name) assert df_rez.columns.tolist() == [col.CLOSE] assert df_rez.index.tolist() == [pd.Timestamp("2020-09-29")] assert df_rez.values.tolist() == [[3000.0]]
async def test_securities_loader(mocker): """Форматирование загруженных данных по торгуемым акциям.""" mocker.patch.object(moex.aiomoex, "get_board_securities", return_value=DF_SEC) loader = moex.SecuritiesLoader() table_name = outer.TableName(outer.SECURITIES, outer.SECURITIES) df_rez = await loader.get(table_name) assert df_rez.columns.tolist() == [col.REG_NUMBER, col.LOT_SIZE] assert df_rez.index.tolist() == ["GAZP"] assert df_rez.values.tolist() == [["abc", 12]]
async def test_loader(mocker): """Основной вариант работы загрузчика.""" fake_load_xlsx = mocker.patch.object(cpi, "_load_xlsx") fake_validate = mocker.patch.object(cpi, "_validate") fake_clean_up = mocker.patch.object(cpi, "_clean_up") loader = cpi.CPILoader() table_name = outer.TableName(outer.CPI, outer.CPI) assert await loader.get(table_name) is fake_clean_up.return_value fake_load_xlsx.assert_called_once_with() fake_validate.assert_called_once_with(fake_load_xlsx.return_value) fake_clean_up.assert_called_once_with(fake_load_xlsx.return_value)
async def test_loader_logger_mixin(caplog): """Логирование загрузки.""" caplog.set_level(logging.INFO) mixin = logger.LoaderLoggerMixin() table_name = outer.TableName(outer.QUOTES, "YAKG") assert mixin._log_and_validate_group(table_name, outer.QUOTES) == "YAKG" await asyncio.sleep(0.01) assert caplog.record_tuples == [ ( "LoaderLoggerMixin", 20, "Загрузка TableName(group='quotes', name='YAKG')", ), ]
"""Тесты загрузки таблицы с диапазоном торговых дат.""" import asyncio import logging import random import pandas as pd import pytest from poptimizer.data.adapters.loaders import trading_dates from poptimizer.data.config import resources from poptimizer.data.ports import outer LOGGER_CLASS_NAME = "TradingDatesLoader" TABLE_NAME = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES) JSON = ({"from": "1997-03-24", "till": "2020-09-22"}, ) DF = pd.DataFrame(JSON, dtype="datetime64[ns]") NAMES_CASES = ( outer.TableName(outer.TRADING_DATES, "test"), outer.TableName(outer.QUOTES, outer.TRADING_DATES), ) @pytest.mark.parametrize("table_name", NAMES_CASES) @pytest.mark.asyncio async def test_raise_on_wrong_name(table_name): """Не верное название данных.""" with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): loader = trading_dates.TradingDatesLoader() await loader.get(table_name)
@pytest.mark.parametrize("now, end", POTENTIAL_TRADING_DAY_CASES) def test_trading_day_potential_end(now, end, monkeypatch): """Тестирование двух краевых случаев на стыке потенциального окончания торгового дня.""" monkeypatch.setattr(services, "datetime", FakeDateTime(now)) assert services.trading_day_potential_end() == end def test_day_real_end(): """Тест на окончание реального торгового дня.""" df = pd.DataFrame([datetime(2020, 9, 11)], columns=["till"]) assert services.trading_day_real_end(df) == datetime(2020, 9, 11, 21, 45) MAIN_HELPER = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES) HELPER_NAME_CASES = ( (outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES), None), (outer.TableName(outer.CONOMY, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.DOHOD, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.SMART_LAB, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.DIVIDENDS, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.CPI, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.SECURITIES, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.INDEX, outer.TRADING_DATES), MAIN_HELPER), (outer.TableName(outer.QUOTES, outer.TRADING_DATES), MAIN_HELPER), ) @pytest.mark.parametrize("name, answer", HELPER_NAME_CASES) def test_get_helper_name(name, answer):
import pandas as pd import pytest from poptimizer.data.adapters import db from poptimizer.data.config import resources from poptimizer.data.ports import outer DF = pd.DataFrame( [[5, 6], [6, 7]], columns=["s", "f"], index=["r", "g"], ) TIMESTAMP = datetime.utcnow() TIMESTAMP = TIMESTAMP.replace(microsecond=(TIMESTAMP.microsecond // 1000) * 1000) TABLE_NAME = outer.TableName(outer.SECURITIES, outer.SECURITIES) TABLE_TUPLE = outer.TableTuple(outer.SECURITIES, outer.SECURITIES, DF, TIMESTAMP) NAME_CASES = ( [ outer.TableTuple(outer.QUOTES, "VTBR", DF, TIMESTAMP), outer.QUOTES, "VTBR" ], [outer.TableName(outer.QUOTES, "HYDR"), outer.QUOTES, "HYDR"], [TABLE_TUPLE, db.MISC, outer.SECURITIES], [TABLE_NAME, db.MISC, outer.SECURITIES], ) @pytest.mark.parametrize("table_name, collection, name", NAME_CASES)
"""Тесты для загрузки данных с MOEX.""" import asyncio import logging from datetime import datetime import pandas as pd import pytest from poptimizer.data.adapters.loaders import moex from poptimizer.data.config import resources from poptimizer.data.ports import col, outer # flake8: noqa NAMES_CASES = ( (moex.SecuritiesLoader(), outer.TableName(outer.SECURITIES, "test")), (moex.SecuritiesLoader(), outer.TableName(outer.QUOTES, outer.SECURITIES)), (moex.IndexLoader(), outer.TableName(outer.INDEX, "test")), (moex.IndexLoader(), outer.TableName(outer.QUOTES, outer.INDEX)), (moex.QuotesLoader(moex.SecuritiesLoader()), outer.TableName(outer.SECURITIES, "AKRN")), ) @pytest.mark.parametrize("loader, table_name", NAMES_CASES) @pytest.mark.asyncio async def test_loader_raise_on_wrong_name(loader, table_name): """Не верное название таблицы.""" with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): await loader.get(table_name)
def recreate_table(table_tuple: outer.TableTuple) -> model.Table: """Создает таблицу на основе данных и обновляет ее.""" name = outer.TableName(table_tuple.group, table_tuple.name) return model.Table(name, _TABLES_REGISTRY[name.group], table_tuple.df, table_tuple.timestamp)
df_clean = cpi._clean_up(df) assert df_clean.values.tolist() == [ [1.0], [3.0], [2.0], [4.0], ] assert df_clean.columns == [col.CPI] assert df_clean.index[0] == pd.Timestamp("1992-01-31") assert df_clean.index[-1] == pd.Timestamp("1992-04-30") NAMES_CASES = ( outer.TableName(outer.CPI, "test"), outer.TableName(outer.QUOTES, outer.CPI), ) @pytest.mark.parametrize("table_name", NAMES_CASES) @pytest.mark.asyncio async def test_loader_raise_on_wrong_name(table_name): """Не верное название таблицы.""" with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): loader = cpi.CPILoader() await loader.get(table_name) @pytest.mark.asyncio async def test_loader(mocker):
"""Тесты для загрузки данных с https://www.smart-lab.ru.""" import pandas as pd import pytest from poptimizer.data.adapters.loaders import smart_lab from poptimizer.data.ports import outer NAMES_CASES = ( outer.TableName(outer.SMART_LAB, "test"), outer.TableName(outer.QUOTES, outer.SMART_LAB), ) @pytest.mark.parametrize("table_name", NAMES_CASES) @pytest.mark.asyncio async def test_loader_raise_on_wrong_name(table_name): """Не верное название таблицы.""" with pytest.raises(outer.DataError, match="Некорректное имя таблицы для обновления"): loader = smart_lab.SmartLabLoader() await loader.get(table_name) DF = pd.DataFrame( [[4.0], [1.0], [2.0], [None]], index=["2020-01-20", "2014-11-25", "2014-11-25", smart_lab.FOOTER], ) DF_CASES = ((DF, DF.dropna()), (DF.dropna(), None)) @pytest.mark.parametrize("df_patch, df_res", DF_CASES)
def smart_lab() -> pd.DataFrame: """Информация по дивидендам с smart-lab.ru.""" table_name = outer.TableName(outer.SMART_LAB, outer.SMART_LAB) requests_handler = bootstrap.get_handler() return requests_handler.get_df(table_name)
"""Тесты событий, связанных с обновлением таблиц.""" import asyncio from datetime import datetime import pytest from poptimizer.data.domain import events, services from poptimizer.data.ports import outer HELPER_NAME = outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES) USUAL_NAME = outer.TableName(outer.QUOTES, "SNGSP") FAKE_END_OF_TRADING_DAY = datetime(2020, 9, 15, 15, 58) GET_DATA_FRAME_CASES = ( ( (USUAL_NAME, True), events.UpdateTable, USUAL_NAME, "_end_of_trading_day", None, ), ( (USUAL_NAME, False), events.GetEndOfTradingDay, HELPER_NAME, "_table_name", USUAL_NAME, ), ( (HELPER_NAME, False), events.UpdateTable,
"""Тесты для фабрик по созданию таблиц и их сериализации.""" from datetime import datetime import pandas as pd import pytest from poptimizer.data.domain import factories, model from poptimizer.data.ports import outer TABLE_NAME = outer.TableName(outer.QUOTES, "VSMO") TABLE_VARS = outer.TableTuple( TABLE_NAME.group, TABLE_NAME.name, pd.DataFrame([1]), datetime.utcnow(), ) def test_create_table(): """У новой таблицы не должно быть данных и времени обновления.""" table = factories.create_table(TABLE_NAME) assert isinstance(table, model.Table) assert table.df is None assert table.timestamp is None def test_recreate_table(): """У таблицы должны быть переданные значения данных и времени обновления. Данные должны возвращаться копией.
def get_helper_name(name: outer.TableName) -> Optional[outer.TableName]: """Имя вспомогательной таблицы.""" if name.group != outer.TRADING_DATES: return outer.TableName(outer.TRADING_DATES, outer.TRADING_DATES) return None