def _extract_ticker_client_types_data(ticker_index: str) -> List: url = TSE_CLIENT_TYPE_DATA_URL.format(ticker_index) with requests_retry_session() as session: response = session.get(url, timeout=5) data = response.text.split(";") data = [row.split(",") for row in data] return data
def client_types(self): response = utils.requests_retry_session().get(self._client_types_url, timeout=10) data = response.text.split(";") data = [row.split(",") for row in data] client_types_data_frame = pd.DataFrame( data, columns=[ "date", "individual_buy_count", "corporate_buy_count", "individual_sell_count", "corporate_sell_count", "individual_buy_vol", "corporate_buy_vol", "individual_sell_vol", "corporate_sell_vol", "individual_buy_value", "corporate_buy_value", "individual_sell_value", "corporate_sell_value" ]) for i in [ "individual_buy_", "individual_sell_", "corporate_buy_", "corporate_sell_" ]: client_types_data_frame[f"{i}mean_price"] = ( client_types_data_frame[f"{i}value"].astype(float) / client_types_data_frame[f"{i}vol"].astype(float)) client_types_data_frame["individual_ownership_change"] = ( client_types_data_frame["corporate_sell_vol"].astype(float) - client_types_data_frame["corporate_buy_vol"].astype(float)) return client_types_data_frame
def shareholders(self) -> pd.DataFrame: page = utils.requests_retry_session(retries=1).get( self._shareholders_url, timeout=5) soup = bs4.BeautifulSoup(page.content, 'html.parser') table: bs4.PageElement = soup.find_all("table")[0] shareholders_df = utils.get_shareholders_html_table_as_csv(table) shareholders_df = shareholders_df.rename( columns=translations.SHAREHOLDERS_FIELD_MAPPINGS) return shareholders_df
def download_ticker_daily_record(ticker_index: str): url = tse_settings.TSE_TICKER_EXPORT_DATA_ADDRESS.format(ticker_index) response = requests_retry_session().get(url, timeout=10) try: response.raise_for_status() except HTTPError: return download_ticker_daily_record(ticker_index) data = StringIO(response.text) return pd.read_csv(data)
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo: response = utils.requests_retry_session().get(self._info_url, timeout=5) return RealtimeTickerInfo( int(response.text.split()[1].split(",")[1]), int(response.text.split()[1].split(",")[2]), int(response.text.split(";")[2].split("@")[1]), int(response.text.split(";")[2].split("@")[2]), int(response.text.split(";")[2].split("@")[4]), int(response.text.split(";")[2].split("@")[3]), )
def get_symbol_id(symbol_name: str): url = tse_settings.TSE_SYMBOL_ID_URL.format(symbol_name.strip()) response = requests_retry_session().get(url, timeout=10) try: response.raise_for_status() except HTTPError: raise Exception("Sorry, tse server did not respond") symbol_full_info = response.text.split(';')[0].split(',') if (to_arabic(symbol_name) == symbol_full_info[0].strip()): return symbol_full_info[2] # symbol id return None
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo: """ notes on usage: - Real time data might not be always available check for None values before usage """ session = utils.requests_retry_session() response = session.get(self._info_url, timeout=5) session.close() # in some cases last price or adj price is undefined try: last_price = int(response.text.split()[1].split(",")[1]) except (ValueError, IndexError): # When instead of number value is `F` last_price = None try: adj_close = int(response.text.split()[1].split(",")[2]) except (ValueError, IndexError): adj_close = None orders_data = response.text.split(";")[2] buy_orders, sell_orders = get_orders(orders_data) best_demand_vol = (buy_orders[0].volume if 0 < len(buy_orders) else None) best_demand_price = (buy_orders[0].price if 0 < len(buy_orders) else None) best_supply_vol = (sell_orders[0].volume if 0 < len(sell_orders) else None) best_supply_price = (sell_orders[0].price if 0 < len(sell_orders) else None) return RealtimeTickerInfo( last_price, adj_close, best_demand_vol=best_demand_vol, best_demand_price=best_demand_price, best_supply_vol=best_supply_vol, best_supply_price=best_supply_price, buy_orders=buy_orders, sell_orders=sell_orders, )
def download( symbols: Union[List, str], write_to_csv: bool = False, include_jdate: bool = False, base_path: str = config.DATA_BASE_PATH) -> Dict[str, pd.DataFrame]: if symbols == "all": symbols = symbols_data.all_symbols() elif isinstance(symbols, str): symbols = [symbols] df_list = {} future_to_symbol = {} with futures.ThreadPoolExecutor(max_workers=10) as executor: session = requests_retry_session() for symbol in symbols: if symbol.isnumeric(): ticker_index = symbol else: ticker_index = symbols_data.get_ticker_index(symbol) _handle_ticker_index(symbol, ticker_index) future = executor.submit(download_ticker_daily_record, ticker_index, session) future_to_symbol[future] = symbol for future in futures.as_completed(future_to_symbol): symbol = future_to_symbol[future] df: pd.DataFrame = future.result() df = df.iloc[::-1] df = df.rename(columns=translations.HISTORY_FIELD_MAPPINGS) df = df.drop(columns=["<PER>", "<TICKER>"]) _adjust_data_frame(df, include_jdate) df_list[symbol] = df if write_to_csv: Path(base_path).mkdir(parents=True, exist_ok=True) df.to_csv(f'{base_path}/{symbol}.csv') if len(df_list) != len(symbols): print("Warning, download did not complete, re-run the code") session.close() return df_list
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo: """ notes on usage: - Real time data might not be always available check for None values before usage """ response = utils.requests_retry_session().get(self._info_url, timeout=5) # check supply and demand data exists if response.text.split(";")[2] != "": best_demand_vol = int(response.text.split(";")[2].split("@")[1]) best_demand_price = int(response.text.split(";")[2].split("@")[2]) best_supply_vol = int(response.text.split(";")[2].split("@")[4]) best_supply_price = int(response.text.split(";")[2].split("@")[3]) else: best_demand_vol = None best_demand_price = None best_supply_vol = None best_supply_price = None # in some cases last price or adj price is undefined try: last_price = int(response.text.split()[1].split(",")[1]) except (ValueError, IndexError): # When instead of number value is `F` last_price = None try: adj_close = int(response.text.split()[1].split(",")[2]) except (ValueError, IndexError): adj_close = None return RealtimeTickerInfo( last_price, adj_close, best_demand_vol=best_demand_vol, best_demand_price=best_demand_price, best_supply_vol=best_supply_vol, best_supply_price=best_supply_price, )
def ticker_page_response(self): return utils.requests_retry_session().get(self._url, timeout=10)