def _fetch_twse_data_history(self, code):
"""
get specific stock monthly trading data from twse website
:param code:
:return:
"""
trading_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/exchangeReport/STOCK_DAY'
query_date = self.date_.strftime('%Y%m%d')
params = {
'response': 'json',
'date': query_date,
'stockNo': code
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data']
columns = ['trading_volume', 'trade_value', 'opening_price', 'highest_price', 'lowest_price',
'closing_price', 'change', 'transaction']
for data in data_list:
date_ = StockTools.republic_era_to_ad(data[0])
date_ = datetime.strptime(date_, '%Y/%m/%d').date()
stock_trading = TwseTradingObject(**(dict(zip(columns, data[1:]))))
for attr in stock_trading.__dict__.copy():
if attr not in columns:
delattr(stock_trading, attr)
trading_dict[date_] = stock_trading
return trading_dict
def _fetch_tpex_data_all(self):
"""
fetch all margin trading data in specific date from tpex website
:return:
"""
margin_trading_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/margin_trading/margin_balance/margin_bal_result.php'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace('-', '/'),
params = {
'l': 'zh-tw',
'o': 'json',
'd': query_date
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = ['code', 'name', 'cash_balance_of_previous_day', 'margin_purchase', 'margin_sells', 'cash_redemption',
'cash_balance_of_the_day', 'cash_belong_to_securities_finance', 'cash_utilization_rate',
'cash_quota', 'stock_balance_of_previous_day', 'short_covering', 'short_sale', 'stock_redemption',
'stock_balance_of_the_day', 'stock_belong_to_securities_finance', 'stock_utilization_rate',
'stock_quota', 'offset', 'note']
for data in data_list:
stock_margin_trading = TpexMarginTradingObject(**dict(zip(columns, data)))
margin_trading_dict[stock_margin_trading.code] = stock_margin_trading
return margin_trading_dict
def _fetch_twse_data_all(self):
"""
fetch all margin trading data in specific date from twse website
:return:
"""
margin_trading_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/exchangeReport/MI_MARGN'
query_date = self.date_.strftime('%Y%m%d')
params = {
'response': 'json',
'date': query_date,
'selectType': 'ALL'
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data']
columns = ['code', 'name', 'margin_purchase', 'margin_sells', 'cash_redemption', 'cash_balance_of_previous_day',
'cash_balance_of_the_day', 'cash_quota', 'short_covering', 'short_sale', 'stock_redemption',
'stock_balance_of_previous_day', 'stock_balance_of_the_day', 'stock_quota', 'offset', 'note']
for data in data_list:
stock_margin_trading = TwseMarginTradingObject(**dict(zip(columns, data)))
margin_trading_dict[stock_margin_trading.code] = stock_margin_trading
return margin_trading_dict
def _fetch_tpex_data_all(self):
"""
fetch all trading data in specific date from tpex website
:return:
"""
trading_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/aftertrading/otc_quotes_no1430/stk_wn1430_result.php'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace('-', '/'),
params = {
'l': 'zh-tw',
'd': query_date,
'se': 'EW'
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = ['code', 'name', 'closing_price', 'change', 'opening_price', 'highest_price', 'lowest_price',
'trading_volume', 'trade_value', 'transaction', 'last_best_bid_price', 'last_best_bid_volume',
'last_best_ask_price', 'last_best_ask_volume', 'issued_shares', 'next_limit_up', 'next_limit_down']
for data in data_list:
stock_trading = TpexTradingObject(**dict(zip(columns, data)))
trading_dict[stock_trading.code] = stock_trading
return trading_dict
def _fetch_single_data(code, date_):
url = f'{Domain.TDCC}/smWeb/QryStockAjax.do'
headers = {
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) '
'AppleWebKit/537.36 (KHTML, liek Gecko) Chrome/34.0.1847.131 Safari/537.36'
}
payload = {
'scaDates': date_,
'scaDate': date_,
'SqlMethod': 'StockNo',
'StockNo': code,
'radioStockNo': code,
'StockName': '',
'REQ_OPR': 'SELECT',
'clkStockNo': code,
'clkStockName': '',
}
while True:
response = ResponseHandler.post(url=url,
payload=payload,
headers=headers)
if '無此資料' not in response:
return response
print('fetching data error, retry after 3 seconds...')
sleep(3)
def get_newest():
"""
fetch newest shareholdings data from official website
:return:
"""
shareholdings_dict = dict()
url = f'{Domain.SMART_TDCC}/opendata/getOD.ashx'
headers = {
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) '
'AppleWebKit/537.36 (KHTML, liek Gecko) Chrome/34.0.1847.131 Safari/537.36'
}
params = {'id': '1-5'}
response = ResponseHandler.get(url=url, params=params, headers=headers)
df = pd.read_csv(response.url)
columns = [
'date', 'code', 'index', 'number_of_shareholders', 'total_shares',
'percentage_over_total_shares'
]
for data in df.values.tolist():
stock_shareholdings = StockShareholdingsObject(
**(dict(zip(columns, data))))
if not stock_shareholdings.number_of_shares:
continue
if stock_shareholdings.code not in shareholdings_dict:
shareholdings_dict[stock_shareholdings.code] = dict()
shareholdings_dict[stock_shareholdings.code].update(
{stock_shareholdings.index: stock_shareholdings})
return shareholdings_dict
def _fetch_tpex_data_history(self, code):
"""
get specific stock monthly trading data from tpex website
:param code:
:return:
"""
trading_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/aftertrading/daily_trading_info/st43_result.php'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace('-', '/'),
params = {
'l': 'zh-tw',
'd': query_date,
'stkno': code
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = ['trading_volume', 'trade_value', 'opening_price', 'highest_price', 'lowest_price',
'closing_price', 'change', 'transaction']
for data in data_list:
date_ = StockTools.republic_era_to_ad(data[0])
date_ = datetime.strptime(date_, '%Y/%m/%d').date()
stock_trading = TpexTradingObject(**(dict(zip(columns, data[1:]))))
for attr in stock_trading.__dict__.copy():
if attr not in columns:
delattr(stock_trading, attr)
trading_dict[date_] = stock_trading
return trading_dict
def _fetch_twse_data_all(self):
"""
fetch all trading data in specific date from twse website
:return:
"""
trading_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/exchangeReport/MI_INDEX'
query_date = self.date_.strftime('%Y%m%d')
params = {
'response': 'json',
'date': query_date,
'type': 'ALL'
}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data9']
columns = ['code', 'name', 'trading_volume', 'transaction', 'trade_value', 'opening_price', 'highest_price',
'lowest_price', 'closing_price', 'different', 'change', 'last_best_bid_price',
'last_best_bid_volume', 'last_best_ask_price', 'last_best_ask_volume', 'price_earning_rate']
for data in data_list:
stock_trading = TwseTradingObject(**dict(zip(columns, data)))
trading_dict[stock_trading.code] = stock_trading
return trading_dict
def _fetch_tpex_data_all(self):
"""
fetch all p/e ratio, dividend yield and p/b ratio data in specific date from tpex website
:return:
"""
p_e_ratio_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/aftertrading/peratio_analysis/pera_result.php'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace(
'-', '/'),
params = {'l': 'zh-tw', 'o': 'json', 'd': query_date}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = [
'code', 'name', 'per', 'dividend_per_share', 'dividend_year',
'yield_ratio', 'pbr'
]
for data in data_list:
p_e_ratio = StockPERatioObject(**dict(zip(columns, data)))
p_e_ratio_dict[p_e_ratio.code] = p_e_ratio
return p_e_ratio_dict
def _fetch_twse_data_all(self):
"""
fetch all p/e ratio, dividend yield and p/b ratio data in specific date from twse website
:return:
"""
p_e_ratio_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/exchangeReport/BWIBBU_d'
query_date = self.date_.strftime('%Y%m%d')
params = {'response': 'json', 'date': query_date, 'selectType': 'ALL'}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data']
columns = [
'code', 'name', 'yield_ratio', 'dividend_year', 'per', 'pbr',
'fiscal_year_quarter'
]
for data in data_list:
p_e_ratio = StockPERatioObject(**dict(zip(columns, data)))
p_e_ratio_dict[p_e_ratio.code] = p_e_ratio
return p_e_ratio_dict
def _fetch_tpex_data_history(self, code):
"""
get specific stock monthly p/e ratio, dividend yield and p/b ratio data from tpex website
:param code:
:return:
"""
p_e_ratio_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/aftertrading/peratio_stk/pera_result.php?'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace(
'-', '/'),
params = {'l': 'zh-tw', 'd': query_date, 'stkno': code}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = ['per', 'yield_ratio', 'dividend_year', 'pbr']
for data in data_list:
date_ = StockTools.republic_era_to_ad(data[0])
date_ = datetime.strptime(date_, '%Y/%m/%d').date()
p_e_ratio = StockPERatioObject(**(dict(zip(columns, data[1:]))))
for attr in p_e_ratio.__dict__.copy():
if attr not in columns:
delattr(p_e_ratio, attr)
p_e_ratio_dict[date_] = p_e_ratio
return p_e_ratio_dict
def _fetch_twse_data_history(self, code):
"""
get specific stock monthly p/e ratio, dividend yield and p/b ratio data from twse website
:param code:
:return:
"""
p_e_ratio_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/exchangeReport/BWIBBU'
query_date = self.date_.strftime('%Y%m%d')
params = {'response': 'json', 'date': query_date, 'stockNo': code}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data']
columns = [
'yield_ratio', 'dividend_year', 'per', 'pbr', 'fiscal_year_quarter'
]
for data in data_list:
date_ = self._translate_date(data[0])
date_ = StockTools.republic_era_to_ad(date_)
date_ = datetime.strptime(date_, '%Y/%m/%d').date()
p_e_ratio = StockPERatioObject(**(dict(zip(columns, data[1:]))))
for attr in p_e_ratio.__dict__.copy():
if attr not in columns:
delattr(p_e_ratio, attr)
p_e_ratio_dict[date_] = p_e_ratio
return p_e_ratio_dict
def _get_available_date_list():
url = f'{Domain.TDCC}/smWeb/QryStockAjax.do'
headers = {
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) '
'AppleWebKit/537.36 (KHTML, liek Gecko) Chrome/34.0.1847.131 Safari/537.36'
}
payload = {'REQ_OPR': 'qrySelScaDates'}
while True:
response = ResponseHandler.post(url=url,
payload=payload,
headers=headers)
date_list = response.json()
date_list = [
datetime.strptime(date_, '%Y%m%d').date()
for date_ in date_list
]
if date_list:
return date_list
print('fetching date list error, retry after 3 seconds...')
sleep(3)
def check_is_open_date(cls, date_):
"""
check if input date is open date (available in after-hour time)
:param date_:
:return:
"""
if not isinstance(date_, date):
raise TypeError('input date must be type of datetime.date')
if date_ > datetime.now().date():
raise AttributeError(
'The input date must earlier than today.'.format(date))
query_date = cls.ad_to_republic_era(date_=date_).replace('-', '/')
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/aftertrading/index_summary/summary_result.php'
params = {'l': 'zh-tw', 'd': query_date}
response = ResponseHandler.get(url=url, params=params)
json_data = response.json()['aaData']
if json_data:
return True
else:
return False
def _fetch_twse_data_all(self):
"""
fetch all institutional investors data in specific date from twse website
:return:
"""
institutional_investors_dict = dict()
url = f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION}/fund/T86'
query_date = self.date_.strftime('%Y%m%d')
params = {'response': 'json', 'date': query_date, 'selectType': 'ALL'}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
stats = json_data['stat']
if not stats == 'OK':
return None
data_list = json_data['data']
columns = [
'code', 'name', 'foreign_mainland_area_buy',
'foreign_mainland_area_sell', 'foreign_mainland_area_diff',
'foreign_buy', 'foreign_sell', 'foreign_diff', 'trust_buy',
'trust_sell', 'trust_diff', 'proprietary_dealers_buy',
'proprietary_dealers_sell', 'proprietary_dealers_diff',
'hedge_dealers_buy', 'hedge_dealers_sell', 'hedge_dealers_diff',
'total_diff'
]
removed_indices = {11}
for data in data_list:
data = [
value for index, value in enumerate(data)
if index not in removed_indices
]
if len(columns) != len(data):
print(f'{data[0]} MISSING INSTITUTIONAL INVESTORS MISSING')
continue
stock_institutional_investors = InstitutionalInvestorsObject(
**dict(zip(columns, data)))
institutional_investors_dict[stock_institutional_investors.
code] = stock_institutional_investors
return institutional_investors_dict
def _fetch_tpex_data_all(self):
"""
fetch all institutional investors data in specific date from tpex website
:return:
"""
institutional_investors_dict = dict()
url = f'{Domain.TAIPEI_EXCHANGE}/web/stock/3insti/daily_trade/3itrade_hedge_result.php'
query_date = StockTools.ad_to_republic_era(date_=self.date_).replace(
'-', '/'),
params = {'l': 'zh-tw', 'd': query_date, 'se': 'EW', 't': 'D'}
response = ResponseHandler.get(url=url, params=params)
if not response:
return None
json_data = response.json()
data_list = json_data['aaData']
if not data_list:
return None
columns = [
'code', 'name', 'foreign_mainland_area_buy',
'foreign_mainland_area_sell', 'foreign_mainland_area_diff',
'foreign_buy', 'foreign_sell', 'foreign_diff', 'trust_buy',
'trust_sell', 'trust_diff', 'proprietary_dealers_buy',
'proprietary_dealers_sell', 'proprietary_dealers_diff',
'hedge_dealers_buy', 'hedge_dealers_sell', 'hedge_dealers_diff',
'total_diff'
]
removed_indices = {8, 9, 10, 20, 21, 22, 24}
for data in data_list:
data = [
value for index, value in enumerate(data)
if index not in removed_indices
]
if len(columns) != len(data):
print(f'{data[0]} MISSING INSTITUTIONAL INVESTORS MISSING')
continue
stock_institutional_investors = InstitutionalInvestorsObject(
**dict(zip(columns, data)))
institutional_investors_dict[stock_institutional_investors.
code] = stock_institutional_investors
return institutional_investors_dict
def main(cls):
"""
:return:
"""
# create csv dir if dir not exists
if not os.path.isdir(Config.STOCK_CSV_DIR):
os.mkdir(Config.STOCK_CSV_DIR)
print('UPDATING STOCK INFO')
for category, str_mode in cls._STR_MODE_DICT.items():
params = {'strMode': str_mode}
response = ResponseHandler.get(
url=
f'{Domain.TAIWAN_STOCK_EXCHANGE_CORPORATION_ISIN}/isin/C_public.jsp',
params=params)
df = pd.read_html(response.text)[0]
df_columns = df.iloc[0].values.tolist() + ['類別']
new_data = list()
tmp_type = None
for data in df.values.tolist()[1:]:
if not StockPattern.CODE_NAME_PATTERN.search(data[0]) and len(
set(data)) == 1:
tmp_type = data[0]
elif StockPattern.CODE_NAME_PATTERN.search(data[0]):
data.append(tmp_type)
new_data.append(data)
df = pd.DataFrame(new_data, columns=df_columns)
df['有價證券代號及名稱'] = df['有價證券代號及名稱'].apply(
lambda x: x.replace('\u3000', ' '))
code_name_df = pd.DataFrame(df['有價證券代號及名稱'].str.split(' ',
1).tolist(),
columns=['證券代號', '證券名稱'])
df = df.drop(['有價證券代號及名稱', '備註'], axis=1)
df = pd.concat([code_name_df, df], axis=1)
csv_path = os.path.join(Config.STOCK_CSV_DIR, f'{category}.csv')
df.to_csv(csv_path)
# to avoiding banned by twse website
sleep(3)
print('FINISHED')