def download_client_types_records( symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.CLIENT_TYPES_DATA_BASE_PATH ): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit( download_ticker_client_types_record, ticker_index ) df: pd.DataFrame = future.result() _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') return df_list
def download_history_price(symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit(download_ticker_daily_record, ticker_index) df: pd.DataFrame = future.result() df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def download(symbols: Union[List, str], write_to_csv: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: future = executor.submit(download_ticker_daily_record, symbols_data.get_ticker_index(symbol)) df: pd.DataFrame = future.result() if df.shape[0] == 0: continue df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) df.date = pd.to_datetime(df.date, format="%Y%m%d") df.set_index("date", inplace=True) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def add_new_symbols(): try: symbol_names = symbols_data.all_symbols() except: raise TSECrawlException( 'دریافت اطلاعات با مشکل رو به رو شد! پس از مدتی دوباره تلاش کنید.') symbols = [] for symbol in symbol_names: if not Symbol.objects.filter(symbol=symbol).count(): symbols.append(Symbol.objects.create(symbol=symbol)) return symbols
def download(symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH): if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} with ThreadPoolExecutor(max_workers=10) as executor: for symbol in symbols: ticker_index = symbols_data.get_ticker_index(symbol) if ticker_index is None: ticker_index = get_symbol_id(symbol) if ticker_index is None: raise Exception("Can not found ticker name") else: symbols_data.append_symbol_to_file(ticker_index, symbol) future = executor.submit(download_ticker_daily_record, ticker_index) df: pd.DataFrame = future.result() if df.shape[0] == 0: continue df = df.iloc[::-1] df = df.rename(columns=FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"]) df.date = pd.to_datetime(df.date, format="%Y%m%d") if include_jdate: df['jdate'] = "" df.jdate = df.date.apply(lambda gregorian: jdatetime.date. fromgregorian(date=gregorian)) df.set_index("date", inplace=True) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") return df_list
def download( symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH) -> Dict[str, pd.DataFrame]: if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} future_to_symbol = {} with futures.ThreadPoolExecutor(max_workers=10) as executor: session = requests_retry_session() for symbol in symbols: if symbol.isnumeric(): ticker_index = symbol else: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit(download_ticker_daily_record, ticker_index, session) future_to_symbol[future] = symbol for future in futures.as_completed(future_to_symbol): symbol = future_to_symbol[future] df: pd.DataFrame = future.result() df = df.iloc[::-1] df = df.rename(columns=translations.HISTORY_FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<TICKER>"]) _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") session.close() return df_list