def download_history_price(symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit(download_ticker_daily_record, ticker_index) df: pd.DataFrame = future.result() df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def download_client_types_records( symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.CLIENT_TYPES_DATA_BASE_PATH ): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit( download_ticker_client_types_record, ticker_index ) df: pd.DataFrame = future.result() _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') return df_list
def download(symbols: Union[List, str], write_to_csv: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: future = executor.submit(download_ticker_daily_record, symbols_data.get_ticker_index(symbol)) df: pd.DataFrame = future.result() if df.shape[0] == 0: continue df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) df.date = pd.to_datetime(df.date, format="%Y%m%d") df.set_index("date", inplace=True) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def __init__(self, symbol: str): self.symbol = symbol self.csv_path = f"{config.DATA_BASE_PATH}/{self.symbol}.csv" self._index = symbols_data.get_ticker_index(self.symbol) self._url = tse_settings.TSE_TICKER_ADDRESS.format(self._index) self._info_url = tse_settings.TSE_ISNT_INFO_URL.format(self._index) self._history: pd.DataFrame = pd.DataFrame() if os.path.exists(self.csv_path): self.from_file() else: self.from_web()
def __init__(self, symbol: str, index: Optional[str] = None): self.symbol = symbol self.csv_path = f"{config.DATA_BASE_PATH}/{self.symbol}.csv" self._index = index or symbols_data.get_ticker_index(self.symbol) self._url = tse_settings.TSE_TICKER_ADDRESS.format(self._index) self._info_url = tse_settings.TSE_ISNT_INFO_URL.format(self._index) self._client_types_url = TSE_CLIENT_TYPE_DATA_URL.format(self._index) self._history: pd.DataFrame = pd.DataFrame() if os.path.exists(self.csv_path): self.from_file() else: self.from_web()
def download(symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) if ticker_index is None: ticker_index = get_symbol_id(symbol) if ticker_index is None: raise Exception("Can not found ticker name") else: symbols_data.append_symbol_to_file(ticker_index, symbol) future = executor.submit(download_ticker_daily_record, ticker_index) df: pd.DataFrame = future.result() if df.shape[0] == 0: continue df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) df.date = pd.to_datetime(df.date, format="%Y%m%d") if include_jdate: df['jdate'] = "" df.jdate = df.date.apply(lambda gregorian: jdatetime.date. fromgregorian(date=gregorian)) df.set_index("date", inplace=True) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def download( symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH) -> Dict[str, pd.DataFrame]: if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} future_to_symbol = {} with futures.ThreadPoolExecutor(max_workers=10) as executor: session = requests_retry_session() for symbol in symbols: if symbol.isnumeric(): ticker_index = symbol else: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit(download_ticker_daily_record, ticker_index, session) future_to_symbol[future] = symbol for future in futures.as_completed(future_to_symbol): symbol = future_to_symbol[future] df: pd.DataFrame = future.result() df = df.iloc[::-1] df = df.rename(columns=translations.HISTORY_FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<TICKER>"]) _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") session.close() return df_list