def _extract_ticker_client_types_data(ticker_index: str) -> List:
url = TSE_CLIENT_TYPE_DATA_URL.format(ticker_index)
with requests_retry_session() as session:
response = session.get(url, timeout=5)
data = response.text.split(";")
data = [row.split(",") for row in data]
return data
def client_types(self):
response = utils.requests_retry_session().get(self._client_types_url,
timeout=10)
data = response.text.split(";")
data = [row.split(",") for row in data]
client_types_data_frame = pd.DataFrame(
data,
columns=[
"date", "individual_buy_count", "corporate_buy_count",
"individual_sell_count", "corporate_sell_count",
"individual_buy_vol", "corporate_buy_vol",
"individual_sell_vol", "corporate_sell_vol",
"individual_buy_value", "corporate_buy_value",
"individual_sell_value", "corporate_sell_value"
])
for i in [
"individual_buy_", "individual_sell_", "corporate_buy_",
"corporate_sell_"
]:
client_types_data_frame[f"{i}mean_price"] = (
client_types_data_frame[f"{i}value"].astype(float) /
client_types_data_frame[f"{i}vol"].astype(float))
client_types_data_frame["individual_ownership_change"] = (
client_types_data_frame["corporate_sell_vol"].astype(float) -
client_types_data_frame["corporate_buy_vol"].astype(float))
return client_types_data_frame
def shareholders(self) -> pd.DataFrame:
page = utils.requests_retry_session(retries=1).get(
self._shareholders_url, timeout=5)
soup = bs4.BeautifulSoup(page.content, 'html.parser')
table: bs4.PageElement = soup.find_all("table")[0]
shareholders_df = utils.get_shareholders_html_table_as_csv(table)
shareholders_df = shareholders_df.rename(
columns=translations.SHAREHOLDERS_FIELD_MAPPINGS)
return shareholders_df
def download_ticker_daily_record(ticker_index: str):
url = tse_settings.TSE_TICKER_EXPORT_DATA_ADDRESS.format(ticker_index)
response = requests_retry_session().get(url, timeout=10)
try:
response.raise_for_status()
except HTTPError:
return download_ticker_daily_record(ticker_index)
data = StringIO(response.text)
return pd.read_csv(data)
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo:
response = utils.requests_retry_session().get(self._info_url,
timeout=5)
return RealtimeTickerInfo(
int(response.text.split()[1].split(",")[1]),
int(response.text.split()[1].split(",")[2]),
int(response.text.split(";")[2].split("@")[1]),
int(response.text.split(";")[2].split("@")[2]),
int(response.text.split(";")[2].split("@")[4]),
int(response.text.split(";")[2].split("@")[3]),
)
def get_symbol_id(symbol_name: str):
url = tse_settings.TSE_SYMBOL_ID_URL.format(symbol_name.strip())
response = requests_retry_session().get(url, timeout=10)
try:
response.raise_for_status()
except HTTPError:
raise Exception("Sorry, tse server did not respond")
symbol_full_info = response.text.split(';')[0].split(',')
if (to_arabic(symbol_name) == symbol_full_info[0].strip()):
return symbol_full_info[2] # symbol id
return None
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo:
"""
notes on usage:
- Real time data might not be always available
check for None values before usage
"""
session = utils.requests_retry_session()
response = session.get(self._info_url, timeout=5)
session.close()
# in some cases last price or adj price is undefined
try:
last_price = int(response.text.split()[1].split(",")[1])
except (ValueError, IndexError): # When instead of number value is `F`
last_price = None
try:
adj_close = int(response.text.split()[1].split(",")[2])
except (ValueError, IndexError):
adj_close = None
orders_data = response.text.split(";")[2]
buy_orders, sell_orders = get_orders(orders_data)
best_demand_vol = (buy_orders[0].volume
if 0 < len(buy_orders) else None)
best_demand_price = (buy_orders[0].price
if 0 < len(buy_orders) else None)
best_supply_vol = (sell_orders[0].volume
if 0 < len(sell_orders) else None)
best_supply_price = (sell_orders[0].price
if 0 < len(sell_orders) else None)
return RealtimeTickerInfo(
last_price,
adj_close,
best_demand_vol=best_demand_vol,
best_demand_price=best_demand_price,
best_supply_vol=best_supply_vol,
best_supply_price=best_supply_price,
buy_orders=buy_orders,
sell_orders=sell_orders,
)
def download(
symbols: Union[List, str],
write_to_csv: bool = False,
include_jdate: bool = False,
base_path: str = config.DATA_BASE_PATH) -> Dict[str, pd.DataFrame]:
if symbols == "all":
symbols = symbols_data.all_symbols()
elif isinstance(symbols, str):
symbols = [symbols]
df_list = {}
future_to_symbol = {}
with futures.ThreadPoolExecutor(max_workers=10) as executor:
session = requests_retry_session()
for symbol in symbols:
if symbol.isnumeric():
ticker_index = symbol
else:
ticker_index = symbols_data.get_ticker_index(symbol)
_handle_ticker_index(symbol, ticker_index)
future = executor.submit(download_ticker_daily_record,
ticker_index, session)
future_to_symbol[future] = symbol
for future in futures.as_completed(future_to_symbol):
symbol = future_to_symbol[future]
df: pd.DataFrame = future.result()
df = df.iloc[::-1]
df = df.rename(columns=translations.HISTORY_FIELD_MAPPINGS)
df = df.drop(columns=["<PER>", "<TICKER>"])
_adjust_data_frame(df, include_jdate)
df_list[symbol] = df
if write_to_csv:
Path(base_path).mkdir(parents=True, exist_ok=True)
df.to_csv(f'{base_path}/{symbol}.csv')
if len(df_list) != len(symbols):
print("Warning, download did not complete, re-run the code")
session.close()
return df_list
def get_ticker_real_time_info_response(self) -> RealtimeTickerInfo:
"""
notes on usage:
- Real time data might not be always available
check for None values before usage
"""
response = utils.requests_retry_session().get(self._info_url,
timeout=5)
# check supply and demand data exists
if response.text.split(";")[2] != "":
best_demand_vol = int(response.text.split(";")[2].split("@")[1])
best_demand_price = int(response.text.split(";")[2].split("@")[2])
best_supply_vol = int(response.text.split(";")[2].split("@")[4])
best_supply_price = int(response.text.split(";")[2].split("@")[3])
else:
best_demand_vol = None
best_demand_price = None
best_supply_vol = None
best_supply_price = None
# in some cases last price or adj price is undefined
try:
last_price = int(response.text.split()[1].split(",")[1])
except (ValueError, IndexError): # When instead of number value is `F`
last_price = None
try:
adj_close = int(response.text.split()[1].split(",")[2])
except (ValueError, IndexError):
adj_close = None
return RealtimeTickerInfo(
last_price,
adj_close,
best_demand_vol=best_demand_vol,
best_demand_price=best_demand_price,
best_supply_vol=best_supply_vol,
best_supply_price=best_supply_price,
)
def ticker_page_response(self):
return utils.requests_retry_session().get(self._url, timeout=10)
