def download_client_types_records(
symbols: Union[List, str],
write_to_csv: bool = False,
include_jdate: bool = False,
base_path: str = config.CLIENT_TYPES_DATA_BASE_PATH
):
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
with ThreadPoolExecutor(max_workers=10) as executor:
for symbol in symbols:
ticker_index = symbols_data.get_ticker_index(symbol)
_handle_ticker_index(symbol, ticker_index)
future = executor.submit(
download_ticker_client_types_record,
ticker_index
)
df: pd.DataFrame = future.result()
_adjust_data_frame(df, include_jdate)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
return df_list
def download_history_price(symbols: Union[List, str],
write_to_csv: bool = False,
include_jdate: bool = False,
base_path: str = config.DATA_BASE_PATH):
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
with ThreadPoolExecutor(max_workers=10) as executor:
for symbol in symbols:
ticker_index = symbols_data.get_ticker_index(symbol)
_handle_ticker_index(symbol, ticker_index)
future = executor.submit(download_ticker_daily_record,
ticker_index)
df: pd.DataFrame = future.result()
df = df.iloc[::-1]
df = df.rename(columns=FIELD_MAPPINGS)
df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"])
_adjust_data_frame(df, include_jdate)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
if len(df_list) != len(symbols):
print("Warning, download did not complete, re-run the code")
return df_list
def download(symbols: Union[List, str],
write_to_csv: bool = False,
base_path: str = config.DATA_BASE_PATH):
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
with ThreadPoolExecutor(max_workers=10) as executor:
for symbol in symbols:
future = executor.submit(download_ticker_daily_record,
symbols_data.get_ticker_index(symbol))
df: pd.DataFrame = future.result()
if df.shape[0] == 0:
continue
df = df.iloc[::-1]
df = df.rename(columns=FIELD_MAPPINGS)
df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"])
df.date = pd.to_datetime(df.date, format="%Y%m%d")
df.set_index("date", inplace=True)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
if len(df_list) != len(symbols):
print("Warning, download did not complete, re-run the code")
return df_list
def add_new_symbols():
try:
symbol_names = symbols_data.all_symbols()
except:
raise TSECrawlException(
'دریافت اطلاعات با مشکل رو به رو شد! پس از مدتی دوباره تلاش کنید.')
symbols = []
for symbol in symbol_names:
if not Symbol.objects.filter(symbol=symbol).count():
symbols.append(Symbol.objects.create(symbol=symbol))
return symbols
def download(symbols: Union[List, str],
write_to_csv: bool = False,
include_jdate: bool = False,
base_path: str = config.DATA_BASE_PATH):
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
with ThreadPoolExecutor(max_workers=10) as executor:
for symbol in symbols:
ticker_index = symbols_data.get_ticker_index(symbol)
if ticker_index is None:
ticker_index = get_symbol_id(symbol)
if ticker_index is None:
raise Exception("Can not found ticker name")
else:
symbols_data.append_symbol_to_file(ticker_index, symbol)
future = executor.submit(download_ticker_daily_record,
ticker_index)
df: pd.DataFrame = future.result()
if df.shape[0] == 0:
continue
df = df.iloc[::-1]
df = df.rename(columns=FIELD_MAPPINGS)
df = df.drop(columns=["<PER>", "<OPEN>", "<TICKER>"])
df.date = pd.to_datetime(df.date, format="%Y%m%d")
if include_jdate:
df['jdate'] = ""
df.jdate = df.date.apply(lambda gregorian: jdatetime.date.
fromgregorian(date=gregorian))
df.set_index("date", inplace=True)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
if len(df_list) != len(symbols):
print("Warning, download did not complete, re-run the code")
return df_list
def download(
symbols: Union[List, str],
write_to_csv: bool = False,
include_jdate: bool = False,
base_path: str = config.DATA_BASE_PATH) -> Dict[str, pd.DataFrame]:
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
future_to_symbol = {}
with futures.ThreadPoolExecutor(max_workers=10) as executor:
session = requests_retry_session()
for symbol in symbols:
if symbol.isnumeric():
ticker_index = symbol
else:
ticker_index = symbols_data.get_ticker_index(symbol)
_handle_ticker_index(symbol, ticker_index)
future = executor.submit(download_ticker_daily_record,
ticker_index, session)
future_to_symbol[future] = symbol
for future in futures.as_completed(future_to_symbol):
symbol = future_to_symbol[future]
df: pd.DataFrame = future.result()
df = df.iloc[::-1]
df = df.rename(columns=translations.HISTORY_FIELD_MAPPINGS)
df = df.drop(columns=["<PER>", "<TICKER>"])
_adjust_data_frame(df, include_jdate)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
if len(df_list) != len(symbols):
print("Warning, download did not complete, re-run the code")
session.close()
return df_list