async def _load_xlsx() -> pd.DataFrame:
"""Загрузка Excel-файла с данными по инфляции."""
session = resources.get_aiohttp_session()
file_url = await _get_xlsx_url(session)
async with session.get(file_url) as resp:
xls_file = await resp.read()
return pd.read_excel(xls_file, **PARSING_PARAMETERS)
async def test_download_many(mocker):
"""Загрузка многих и выбор с максимального оборота для дублирующихся дат."""
mocker.patch.object(moex.aiomoex,
"get_market_candles",
side_effect=[JSON1, JSON2])
session = resources.get_aiohttp_session()
df = await moex._download_many(session, ["OGKB3", "OGK4"])
assert df.columns.tolist() == [
"begin",
"open",
"close",
"high",
"low",
"value",
"volume",
"end",
]
assert df["begin"].tolist() == [
"2011-09-27 00:00:00",
"2011-09-28 00:00:00",
"2011-09-29 00:00:00",
]
assert df["close"].tolist() == [
2,
8,
3,
]
async def test_get_many_times_with_cache(mocker, caplog):
"""Использования кэша после единственной загрузки."""
caplog.set_level(logging.INFO)
outer_call = mocker.patch.object(
trading_dates.aiomoex,
"get_board_dates",
side_effect=random_sleep,
)
loader = trading_dates.TradingDatesLoader()
aws = [loader.get(TABLE_NAME) for _ in range(100)]
dfs = await asyncio.gather(*aws)
outer_call.assert_called_once_with(
resources.get_aiohttp_session(),
board="TQBR",
market="shares",
engine="stock",
)
for df in dfs:
pd.testing.assert_frame_equal(df, DF)
assert set(caplog.record_tuples) == {
(
LOGGER_CLASS_NAME,
20,
"Загрузка из кэша TableName(group='trading_dates', name='trading_dates')",
),
(LOGGER_CLASS_NAME, 20,
"Загрузка TableName(group='trading_dates', name='trading_dates')"),
(LOGGER_CLASS_NAME, 20, "Последняя дата с историей: 2020-09-22"),
}
async def get(
self,
table_name: outer.TableName,
last_index: Optional[str] = None,
) -> pd.DataFrame:
"""Получение котировок акций в формате OCHLV."""
ticker = self._log_and_validate_group(table_name, outer.QUOTES)
http_session = resources.get_aiohttp_session()
if last_index is None:
df = await self._first_load(http_session, ticker)
else:
json = await aiomoex.get_market_candles(
http_session,
ticker,
start=last_index,
end=_previous_day_in_moscow(),
)
df = pd.DataFrame(json)
df = df[list(OCHLV_COL)]
df.columns = [
col.DATE,
col.OPEN,
col.CLOSE,
col.HIGH,
col.LOW,
col.TURNOVER,
]
df[col.DATE] = pd.to_datetime(df[col.DATE])
return df.set_index(col.DATE)
async def _get_html(url: str) -> str:
"""Загружает html-код страницы."""
session = resources.get_aiohttp_session()
async with session.get(url) as respond:
try:
respond.raise_for_status()
except aiohttp.ClientResponseError:
raise outer.DataError(f"Данные {url} не загружены")
return await respond.text()
async def test_download_many_empty(mocker):
"""Корректное наименование столбцов при отсутствии данных."""
mocker.patch.object(moex.aiomoex,
"get_market_candles",
side_effect=[[], []])
session = resources.get_aiohttp_session()
df = await moex._download_many(session, ["OGKB", "OGK2"])
assert df.empty
assert df.columns.tolist() == [
"begin", "open", "close", "high", "low", "value"
]
async def test_find_aliases(mocker):
"""Загрузка альтернативных тикеров в виде массива."""
json = [
{
"secid": "OGKB",
"regnumber": "1-02-65105-D"
},
{
"secid": "OGK2",
"regnumber": "1-02-65105-D"
},
{
"secid": "OGK2-001D",
"regnumber": "1-02-65105-D-001D"
},
]
mocker.patch.object(moex.aiomoex, "find_securities", return_value=json)
session = resources.get_aiohttp_session()
assert await moex._find_aliases(session,
"1-02-65105-D") == ["OGKB", "OGK2"]
async def get(self, table_name: outer.TableName) -> pd.DataFrame:
"""Получение списка торгуемых акций с регистрационным номером и размером лота."""
name = self._log_and_validate_group(table_name, outer.SECURITIES)
if name != outer.SECURITIES:
raise outer.DataError(
f"Некорректное имя таблицы для обновления {table_name}")
async with self._cache_lock:
if self._securities_cache is not None:
self._logger.info(f"Загрузка из кэша {table_name}")
return self._securities_cache
columns = ("SECID", "REGNUMBER", "LOTSIZE")
http_session = resources.get_aiohttp_session()
json = await aiomoex.get_board_securities(http_session,
columns=columns)
df = pd.DataFrame(json)
df.columns = [col.TICKER, col.REG_NUMBER, col.LOT_SIZE]
self._securities_cache = df.set_index(col.TICKER)
return self._securities_cache
async def test_get_df(mocker, caplog):
"""Загрузка данных."""
caplog.set_level(logging.INFO)
outer_call = mocker.patch.object(trading_dates.aiomoex,
"get_board_dates",
return_value=JSON)
loader = trading_dates.TradingDatesLoader()
df = await loader.get(TABLE_NAME)
await asyncio.sleep(0.01)
pd.testing.assert_frame_equal(df, DF)
outer_call.assert_called_once_with(
resources.get_aiohttp_session(),
board="TQBR",
market="shares",
engine="stock",
)
assert set(caplog.record_tuples) == {
(LOGGER_CLASS_NAME, 20,
"Загрузка TableName(group='trading_dates', name='trading_dates')"),
(LOGGER_CLASS_NAME, 20, "Последняя дата с историей: 2020-09-22"),
}
async def get(
self,
table_name: outer.TableName,
last_index: Optional[str] = None,
) -> pd.DataFrame:
"""Получение цен закрытия индекса MCFTRR."""
name = self._log_and_validate_group(table_name, outer.INDEX)
if name != outer.INDEX:
raise outer.DataError(
f"Некорректное имя таблицы для обновления {table_name}")
http_session = resources.get_aiohttp_session()
json = await aiomoex.get_board_history(
session=http_session,
start=last_index,
security=outer.INDEX,
columns=("TRADEDATE", "CLOSE"),
board="RTSI",
market="index",
)
df = pd.DataFrame(json)
df.columns = [col.DATE, col.CLOSE]
df[col.DATE] = pd.to_datetime(df[col.DATE])
return df.set_index(col.DATE)