async def _load_xlsx() -> pd.DataFrame: """Загрузка Excel-файла с данными по инфляции.""" session = resources.get_aiohttp_session() file_url = await _get_xlsx_url(session) async with session.get(file_url) as resp: xls_file = await resp.read() return pd.read_excel(xls_file, **PARSING_PARAMETERS)
async def test_download_many(mocker): """Загрузка многих и выбор с максимального оборота для дублирующихся дат.""" mocker.patch.object(moex.aiomoex, "get_market_candles", side_effect=[JSON1, JSON2]) session = resources.get_aiohttp_session() df = await moex._download_many(session, ["OGKB3", "OGK4"]) assert df.columns.tolist() == [ "begin", "open", "close", "high", "low", "value", "volume", "end", ] assert df["begin"].tolist() == [ "2011-09-27 00:00:00", "2011-09-28 00:00:00", "2011-09-29 00:00:00", ] assert df["close"].tolist() == [ 2, 8, 3, ]
async def test_get_many_times_with_cache(mocker, caplog): """Использования кэша после единственной загрузки.""" caplog.set_level(logging.INFO) outer_call = mocker.patch.object( trading_dates.aiomoex, "get_board_dates", side_effect=random_sleep, ) loader = trading_dates.TradingDatesLoader() aws = [loader.get(TABLE_NAME) for _ in range(100)] dfs = await asyncio.gather(*aws) outer_call.assert_called_once_with( resources.get_aiohttp_session(), board="TQBR", market="shares", engine="stock", ) for df in dfs: pd.testing.assert_frame_equal(df, DF) assert set(caplog.record_tuples) == { ( LOGGER_CLASS_NAME, 20, "Загрузка из кэша TableName(group='trading_dates', name='trading_dates')", ), (LOGGER_CLASS_NAME, 20, "Загрузка TableName(group='trading_dates', name='trading_dates')"), (LOGGER_CLASS_NAME, 20, "Последняя дата с историей: 2020-09-22"), }
async def get( self, table_name: outer.TableName, last_index: Optional[str] = None, ) -> pd.DataFrame: """Получение котировок акций в формате OCHLV.""" ticker = self._log_and_validate_group(table_name, outer.QUOTES) http_session = resources.get_aiohttp_session() if last_index is None: df = await self._first_load(http_session, ticker) else: json = await aiomoex.get_market_candles( http_session, ticker, start=last_index, end=_previous_day_in_moscow(), ) df = pd.DataFrame(json) df = df[list(OCHLV_COL)] df.columns = [ col.DATE, col.OPEN, col.CLOSE, col.HIGH, col.LOW, col.TURNOVER, ] df[col.DATE] = pd.to_datetime(df[col.DATE]) return df.set_index(col.DATE)
async def _get_html(url: str) -> str: """Загружает html-код страницы.""" session = resources.get_aiohttp_session() async with session.get(url) as respond: try: respond.raise_for_status() except aiohttp.ClientResponseError: raise outer.DataError(f"Данные {url} не загружены") return await respond.text()
async def test_download_many_empty(mocker): """Корректное наименование столбцов при отсутствии данных.""" mocker.patch.object(moex.aiomoex, "get_market_candles", side_effect=[[], []]) session = resources.get_aiohttp_session() df = await moex._download_many(session, ["OGKB", "OGK2"]) assert df.empty assert df.columns.tolist() == [ "begin", "open", "close", "high", "low", "value" ]
async def test_find_aliases(mocker): """Загрузка альтернативных тикеров в виде массива.""" json = [ { "secid": "OGKB", "regnumber": "1-02-65105-D" }, { "secid": "OGK2", "regnumber": "1-02-65105-D" }, { "secid": "OGK2-001D", "regnumber": "1-02-65105-D-001D" }, ] mocker.patch.object(moex.aiomoex, "find_securities", return_value=json) session = resources.get_aiohttp_session() assert await moex._find_aliases(session, "1-02-65105-D") == ["OGKB", "OGK2"]
async def get(self, table_name: outer.TableName) -> pd.DataFrame: """Получение списка торгуемых акций с регистрационным номером и размером лота.""" name = self._log_and_validate_group(table_name, outer.SECURITIES) if name != outer.SECURITIES: raise outer.DataError( f"Некорректное имя таблицы для обновления {table_name}") async with self._cache_lock: if self._securities_cache is not None: self._logger.info(f"Загрузка из кэша {table_name}") return self._securities_cache columns = ("SECID", "REGNUMBER", "LOTSIZE") http_session = resources.get_aiohttp_session() json = await aiomoex.get_board_securities(http_session, columns=columns) df = pd.DataFrame(json) df.columns = [col.TICKER, col.REG_NUMBER, col.LOT_SIZE] self._securities_cache = df.set_index(col.TICKER) return self._securities_cache
async def test_get_df(mocker, caplog): """Загрузка данных.""" caplog.set_level(logging.INFO) outer_call = mocker.patch.object(trading_dates.aiomoex, "get_board_dates", return_value=JSON) loader = trading_dates.TradingDatesLoader() df = await loader.get(TABLE_NAME) await asyncio.sleep(0.01) pd.testing.assert_frame_equal(df, DF) outer_call.assert_called_once_with( resources.get_aiohttp_session(), board="TQBR", market="shares", engine="stock", ) assert set(caplog.record_tuples) == { (LOGGER_CLASS_NAME, 20, "Загрузка TableName(group='trading_dates', name='trading_dates')"), (LOGGER_CLASS_NAME, 20, "Последняя дата с историей: 2020-09-22"), }
async def get( self, table_name: outer.TableName, last_index: Optional[str] = None, ) -> pd.DataFrame: """Получение цен закрытия индекса MCFTRR.""" name = self._log_and_validate_group(table_name, outer.INDEX) if name != outer.INDEX: raise outer.DataError( f"Некорректное имя таблицы для обновления {table_name}") http_session = resources.get_aiohttp_session() json = await aiomoex.get_board_history( session=http_session, start=last_index, security=outer.INDEX, columns=("TRADEDATE", "CLOSE"), board="RTSI", market="index", ) df = pd.DataFrame(json) df.columns = [col.DATE, col.CLOSE] df[col.DATE] = pd.to_datetime(df[col.DATE]) return df.set_index(col.DATE)