def TSE_HistoricalPrice(i):
output = []
try:
date = i.strftime('%Y%m%d')
url = 'https://www.twse.com.tw/exchangeReport/MI_INDEX?response=json&date=' + \
date + '&type=ALLBUT0999&_=1553846233285'
js = crawl_json_data(url)
data = js['data9']
for k in range(len(data)):
o, h, l, c = [
changeToFloat(x.replace(',', '')) for x in data[k][5:9]
]
output.append(
update_data_dict({
'Date':
i.strftime('%Y-%m-%d'),
'Ticker':
data[k][0],
'Open':
o,
'High':
h,
'Low':
l,
'Close':
c,
'Volume':
int(changeToFloat(data[k][2].replace(',', ''))),
}))
except Exception as e:
print(e)
pass
return output
def OTC_HistoricalPrice(i):
output = []
try:
date = '/'.join([str(i.year - 1911), i.strftime('%m/%d')])
url = 'https://www.tpex.org.tw/web/stock/aftertrading/otc_quotes_no1430/stk_wn1430_result.php?l=zh-tw&d=' + \
date + '&se=EW&_=1553850502415'
js = crawl_json_data(url)
data = js["aaData"]
for k in range(len(data)):
o, h, l, c = [
changeToFloat(x.replace(',', ''))
for x in data[k][4:7] + data[k][2:3]
]
volume = int(changeToFloat(data[k][7].replace(',', '')))
# if i >= datetime(2020,7,24):
# volume = int(changeToFloat(data[k][7].replace(',','')))
output.append(
update_data_dict({
'Date': i.strftime("%Y-%m-%d"),
'Ticker': data[k][0],
'Open': o,
'High': h,
'Low': l,
'Close': c,
'Volume': volume,
}))
except Exception as e:
print(e)
pass
return output
def OTC_CapitalReduction(start, end):
selectdate_start = start.strftime('%Y/%m/%d')
selectdate_end = end.strftime('%Y/%m/%d')
urlfirst = 'https://www.tpex.org.tw/web/stock/exright/revivt/revivt_result.php?l=en-us&d='
urllast = '&ed='
url = urlfirst + selectdate_start + urllast + selectdate_end
js = crawl_json_data(url)
output = DataFrame(columns=['Date', 'Ticker', 'Action', 'Value'])
try:
data = js['aaData']
k = 0
for i in range(len(data)):
if data[i][8] == 'Making up losses': #彌補虧損
output.loc[k, 'Date'] = '-'.join([
str(data[i][0])[0:4],
str(data[i][0])[4:6],
str(data[i][0])[6:8]
])
output.loc[k, 'Ticker'] = data[i][1]
output.loc[k, 'Action'] = 'RS'
output.loc[k, 'Value'] = float(
re.search('parm6=(.*)&parm7', data[i][9])[1])
k = k + 1
elif data[i][8] == 'Cash refund': #現金減資
output.loc[k, 'Date'] = '-'.join([
str(data[i][0])[0:4],
str(data[i][0])[4:6],
str(data[i][0])[6:8]
])
output.loc[k, 'Ticker'] = data[i][1]
output.loc[k, 'Action'] = 'RS'
output.loc[k, 'Value'] = float(
re.search('parm6=(.*)&parm7', data[i][9])[1])
output.loc[k + 1, 'Date'] = '-'.join([
str(data[i][0])[0:4],
str(data[i][0])[4:6],
str(data[i][0])[6:8]
])
output.loc[k + 1, 'Ticker'] = data[i][1]
output.loc[k + 1, 'Action'] = 'RC'
output.loc[k + 1, 'Value'] = float(
re.search('parm7=(.*)&parm8', data[i][9])[1])
k = k + 2
except:
pass
return output
def StopTradeTWSE(date: datetime = datetime.today()):
if date <= date.replace(hour=12, minute=0, second=0):
date -= timedelta(1)
dateStr = date.strftime('%Y%m%d')
# dateStr = '20210316'
url = f'https://www.twse.com.tw/exchangeReport/TWTAWU?response=json&startDate={dateStr}&endDate={dateStr}&stockNo=&querytype=3&selectType=&_=1616002611610'
datas = crawl_json_data(url)
if '抱歉' in datas['stat']:
return
print(datas)
cols = datas['fields'][1:]
data = datas['data']
final_data = [
dict((k, v) for k, v in zip(cols, temps[1:])) for temps in data
]
print(final_data)
def crawl_foriegn_holding_ratio_otc(date):
full_df = []
cols = '產業別,家數,總發行股數,僑外資及陸資持有總股數,僑外資及陸資持股比率'.split(',')
try:
dateStr = '/'.join([str(date.year - 1911), date.strftime('%m/%d')])
js = crawl_json_data(f'https://www.tpex.org.tw/web/stock/3insti/qfii_sect/qfiisect_result.php?l=zh-tw&_=1606059780282&o=json&d={dateStr}')
data = js['aaData']
for ds in data:
temp_dict = dict(changeType(col, d) for col, d in zip(cols, ds))
temp_dict['Date'] = date.strftime('%Y-%m-%d')
full_df.append(temp_dict)
time.sleep(10)
except Exception as e:
print(e)
time.sleep(10)
else:
return full_df
def crawl5SecIndex(date):
try:
dateStr = date
if isinstance(dateStr, datetime):
dateStr = dateStr.strftime('%Y-%m-%d')
url = f'https://www.twse.com.tw/exchangeReport/MI_5MINS_INDEX?response=json&date={dateStr.replace("-","")}'
data = crawl_json_data(url)
cols = data['fields']
datas = data['data']
KBar_dict = {}
full_dict_list = []
for d in datas:
for col, idx in zip(cols[1:], d[1:]):
datatime = d[0].strip()
if len(datatime) == 5:
datatime += ':00'
close = float(idx.replace(',', ''))
full_dict_list.append({
'Date': dateStr,
'Time': datatime,
"IndexName": col,
'Close': close
})
if col not in KBar_dict:
KBar_dict[col] = {
'Date': dateStr,
'IndexName': col,
'Open': close,
'High': close,
'Low': close,
'Close': close,
}
else:
KBar_dict[col]['High'] = max(KBar_dict[col]['High'], close)
KBar_dict[col]['Low'] = min(KBar_dict[col]['Low'], close)
KBar_dict[col]['Close'] = close
except ConnectionError:
time.sleep(5)
return crawl5SecIndex(dateStr)
except Exception as e:
raise e
else:
return full_dict_list, list(KBar_dict.values())
def crawl_foriegn_holding_ratio_listed(date):
full_df = []
cols = None
try:
dateStr = date.strftime('%Y%m%d')
js = crawl_json_data(f'https://www.twse.com.tw/fund/MI_QFIIS_cat?response=json&date={dateStr}')
cols = js['fields']
data = js['data']
for ds in data:
temp_dict = dict(changeType(col, d) for col, d in zip(cols, ds))
temp_dict['Date'] = date.strftime('%Y-%m-%d')
full_df.append(temp_dict)
time.sleep(10)
except Exception as e:
print(e)
time.sleep(10)
else:
return full_df
def OTC_ExRightDividend(start, end):
selectdate_start = start.strftime('%Y/%m/%d')
selectdate_end = end.strftime('%Y/%m/%d')
urlfirst = 'https://www.tpex.org.tw/web/stock/exright/dailyquo/exDailyQ_result.php?l=en-us&d='
urllast = '&ed='
url = urlfirst + selectdate_start + urllast + selectdate_end
js = crawl_json_data(url)
output = DataFrame(columns=['Date', 'Ticker', 'Action', 'Value'])
try:
data = js['aaData']
k = 0
for i in range(len(data)):
if data[i][7] == 'DR':
output.loc[k, 'Date'] = data[i][0].replace('/', '-')
output.loc[k, 'Ticker'] = data[i][1]
output.loc[k, 'Action'] = 'XR'
output.loc[k, 'Value'] = float(data[i][13])
output.loc[k + 1, 'Date'] = data[i][0].replace('/', '-')
output.loc[k + 1, 'Ticker'] = data[i][1]
output.loc[k + 1, 'Action'] = 'XD'
output.loc[k + 1, 'Value'] = float(data[i][12])
k = k + 2
elif data[i][7] == 'XD':
output.loc[k, 'Date'] = data[i][0].replace('/', '-')
output.loc[k, 'Ticker'] = data[i][1]
output.loc[k, 'Action'] = 'XD'
output.loc[k, 'Value'] = float(data[i][12])
k = k + 1
elif data[i][7] == 'XR':
output.loc[k, 'Date'] = data[i][0].replace('/', '-')
output.loc[k, 'Ticker'] = data[i][1]
output.loc[k, 'Action'] = 'XR'
output.loc[k, 'Value'] = float(data[i][13])
k = k + 1
output = output[output['Value'] >
0] #OTC 除權資料包括“增資”，但是增資卻不會影響股價 ex: 2017/04/20
except:
pass
return output
def OddLot_Intraday(date):
"""
盤中零股交易歷史資料
"""
try:
dateStr = date.strftime('%Y-%m-%d')
url = f'https://www.twse.com.tw/exchangeReport/TWTC7U?response=json&date={dateStr.replace("-","")}&selectType=&_=1606553379802'
js = crawl_json_data(url)
columns = js['fields']
datas = js['data']
full_dict = []
for data in datas:
temp_data = dict(
changeTypes(col, d) for col, d in zip(columns, data))
temp_data.update({'Date': dateStr})
full_dict.append(temp_data)
except Exception as e:
print('Intraday, Error : ', e)
else:
return full_dict
def TSE_CapitalReduction(start, end):
selectdate_start = start.strftime('%Y%m%d')
selectdate_end = end.strftime('%Y%m%d')
url = f'https://www.twse.com.tw/exchangeReport/TWTAUU?response=json&strDate={selectdate_start}&endDate={selectdate_end}'
js = crawl_json_data(url)
output = DataFrame(columns=['Date', 'Ticker', 'Action', 'Value'])
try:
js['data']
k = 0
url1 = 'https://www.twse.com.tw/zh'
for i in range(len(js['data'])):
"""
res1 = requests.get(url1 + js['data'][i][10].split('\'')[1].split('.')[6], timeout = 5)
soup1 = bs(res1.text, "html.parser")
"""
res1 = urllib.request.Request(
url1 + js['data'][i][10].split('\'')[1].split('.')[6])
res1.add_header('upgrade-insecure-requests', '1')
res1.add_header(
'user-agent',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
)
rd1 = urllib.request.urlopen(res1).read().decode()
soup1 = bs(rd1, "html.parser")
j = 0
RawData1 = DataFrame(columns=['CapitalReduction'])
for tr1 in soup1.select('table tr')[2:5]:
for td1 in tr1.select('td')[1:]:
RawData1.loc[j, 'CapitalReduction'] = re.sub(
'[^0-9.]', '', str(td1.text))
j = j + 1
if float(RawData1.loc[2, 'CapitalReduction']) == 0:
output.loc[k, 'Date'] = '-'.join([
str(int(js['data'][i][0].split('/')[0]) + 1911),
js['data'][i][0].split('/')[1],
js['data'][i][0].split('/')[2]
])
output.loc[k, 'Ticker'] = js['data'][i][1]
output.loc[k,
'Action'] = 'RS' #換發新股 Replacement issue of stock
output.loc[k, 'Value'] = RawData1.loc[1, 'CapitalReduction']
k = k + 1
else:
output.loc[k, 'Date'] = '-'.join([
str(int(js['data'][i][0].split('/')[0]) + 1911),
js['data'][i][0].split('/')[1],
js['data'][i][0].split('/')[2]
])
output.loc[k, 'Ticker'] = js['data'][i][1]
output.loc[k,
'Action'] = 'RS' #換發新股 Replacement issue of stock
output.loc[k, 'Value'] = RawData1.loc[1, 'CapitalReduction']
output.loc[k + 1, 'Date'] = '-'.join([
str(int(js['data'][i][0].split('/')[0]) + 1911),
js['data'][i][0].split('/')[1],
js['data'][i][0].split('/')[2]
])
output.loc[k + 1, 'Ticker'] = js['data'][i][1]
output.loc[k + 1, 'Action'] = 'RC' #配發現金 Return of capital
output.loc[k + 1, 'Value'] = RawData1.loc[2,
'CapitalReduction']
k = k + 2
time.sleep(5)
except:
pass
return output
def TSE_ExRightDividend(start, end):
selectdate_start = start.strftime('%Y%m%d')
selectdate_end = end.strftime('%Y%m%d')
url = f'https://www.twse.com.tw/exchangeReport/TWT49U?response=json&strDate={selectdate_start}&endDate={selectdate_end}'
js = crawl_json_data(url)
url1 = 'https://www.twse.com.tw'
output = DataFrame(columns=['Date', 'Ticker', 'Action', 'Value'])
try:
js['data']
k = 0
for i in range(len(js['data'])):
ticker = js['data'][i][1]
res = urllib.request.Request(
url1 + js['data'][i][11].split('\'')[1].split('.')[6])
res.add_header('upgrade-insecure-requests', '1')
res.add_header(
'user-agent',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
)
rd = urllib.request.urlopen(res).read().decode()
soup = bs(rd, "html.parser")
RawData1 = DataFrame(columns=['ExRightDividend'])
j = 0
for tr in soup.select('table tr')[0:5]:
for td in tr.select('td')[1:]:
RawData1.loc[j, 'ExRightDividend'] = re.sub(
'[^0-9.]', '', str(td.text))
j = j + 1
if len(RawData1) > 0:
if float(RawData1.iat[4, 0]) != 0 and float(
RawData1.iat[2,
0]) != 0: #Ex Right(XR) & Ex Dividend(XD)
# Ex Dividend(XD) 除息
output.loc[k, 'Date'] = datetime.strptime(
js['data'][i][11].split('\'')[1][-8:],
'%Y%m%d').date().strftime("%Y-%m-%d")
output.loc[k, 'Ticker'] = ticker
output.loc[k, 'Action'] = 'XD'
output.loc[k, 'Value'] = RawData1.iat[2, 0]
#Ex Right(XR) 除權
output.loc[k + 1, 'Date'] = datetime.strptime(
js['data'][i][11].split('\'')[1][-8:],
'%Y%m%d').date().strftime("%Y-%m-%d")
output.loc[k + 1, 'Ticker'] = ticker
output.loc[k + 1, 'Action'] = 'XR'
output.loc[k + 1, 'Value'] = RawData1.iat[4, 0]
k = k + 2
elif float(RawData1.iat[2, 0]) != 0: # Ex Dividend(XD) 除息
output.loc[k, 'Date'] = datetime.strptime(
js['data'][i][11].split('\'')[1][-8:],
'%Y%m%d').date().strftime("%Y-%m-%d")
output.loc[k, 'Ticker'] = ticker
output.loc[k, 'Action'] = 'XD'
output.loc[k, 'Value'] = RawData1.iat[2, 0]
k = k + 1
elif float(RawData1.iat[4, 0]) != 0: #Ex Right(XR) 除權
output.loc[k, 'Date'] = datetime.strptime(
js['data'][i][11].split('\'')[1][-8:],
'%Y%m%d').date().strftime("%Y-%m-%d")
output.loc[k, 'Ticker'] = ticker
output.loc[k, 'Action'] = 'XR'
output.loc[k, 'Value'] = RawData1.iat[4, 0]
k = k + 1
time.sleep(5)
except:
pass
return output