def get_pro():
try:
set_token()
pro = ts.pro_api()
except Exception as e:
if isinstance(e, NameError):
print('请设置tushare pro的token凭证码')
else:
print('请升级tushare 至最新版本 pip install tushare -U')
print(e)
pro = None
return pro
def ingest(environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
symbols = ['600019.SH']
pro_api = ts.pro_api(tushare_token)
dtype = [('start_date', 'datetime64[ns]'),
('end_date', 'datetime64[ns]'),
('auto_close_date', 'datetime64[ns]'),
('symbol', 'object')]
metadata = DataFrame(empty(len(symbols), dtype=dtype))
with maybe_show_progress(symbols, show_progress,
label='Loading CN A %s pricing data: ' % (symbols)) as it:
for sid, symbol in enumerate(it):
tushare_daily = ts.pro_bar(pro_api=pro_api,
ts_code=symbol,
asset='E',
start_date=start_session.strftime('%Y%m%d'),
end_date=end_session.strftime('%Y%m%d'),
adj='qfq')
tushare_daily['day'] = pd.to_datetime(tushare_daily['trade_date'])
tushare_daily['volume'] = tushare_daily['vol']
tushare_daily['id'] = tushare_daily['ts_code']
tushare_daily = tushare_daily.filter(items=['day', 'open', 'high', 'low', 'close', 'volume'])
tushare_daily = tushare_daily.set_index('day').sort_index()
start_date = tushare_daily.index[0]
end_date = tushare_daily.index[-1]
end_date = start_date if start_date > end_date else end_date
# The auto_close date is the day after the last trade.
ac_date = end_date + Timedelta(days=1)
metadata.iloc[sid] = start_date, end_date, ac_date, symbol
daily_bar_writer.write([(sid, tushare_daily)], show_progress=show_progress)
metadata['exchange'] = 'SSE'
asset_db_writer.write(equities=metadata)
adjustment_writer.write(None)
def ipo():
ts.set_token('b4c94429dc00fee32d14c52507d9cd44c9621ca91eaa161fcec14041')
pro = ts.pro_api()
# 查询当前所有正常上市交易的股票列表
df = pro.stock_basic(exchange_id='', list_status='L', fields='ts_code,symbol,name,area,industry,list_date')
print(df)
data = df.to_dict('index')
print(data.items())
for item, value in sorted(data.items()):
code = value['symbol']
ipo_date = value['list_date']
Stock.objects(code=code).update_one(code=code, ipo_date=ipo_date, upsert=True)
def init_trade_calendar():
pro_api = ts.pro_api(tushare_token)
trade_cal = pro_api.trade_cal(exchange='', start_date='19990101', end_date='20191231')
not_open = trade_cal['is_open'] == 0
trade_date = trade_cal[not_open]['cal_date']
file_path = os.path.join(os.path.split(__file__)[0], 'tushare_not_trading_date.py')
with open(file_path, 'w') as of:
of.write('import pandas as pd\n\n\n')
of.write('not_trading_date=pd.to_datetime([')
for cal_date in trade_date:
of.write('"')
of.write(cal_date)
of.write('",\n')
of.write('])\n')
def get_stockholder(code, start, end):
# 十大非流通
global pro
try:
stockholder = pro.top10_holders(ts_code=code, start_date=start, end_date=end)
time.sleep(1)
stockfloat = pro.top10_floatholders(ts_code=code, start_date=start, end_date=end)
time.sleep(1)
except Exception as e:
print(e)
time.sleep(10)
ts.set_token(token)
pro = ts.pro_api()
else:
if stockholder.empty and stockfloat.empty:
return pd.DataFrame(), pd.DataFrame()
else:
return stockholder, stockfloat
def get_pro():
return ts.pro_api('4b98f5087a086ac0e0d759ce67daeb8a2de2773e12553e3989b303dd')
mpl.rcParams['axes.unicode_minus'] = False
#引入TA-Lib库
import talib as ta
#查看包含的技术指标和数学运算函数
#print(ta.get_functions())
#print(ta.get_function_groups())
ta_fun = ta.get_function_groups()
ta_fun.keys()
#使用tushare获取上证指数数据作为示例
import tushare as ts
ts.set_token('a119134c895dca96f7caedef1de1fcf51409888f8df48aabf62c0399')
pro = ts.pro_api()
#df=ts.get_k_data('sh',start='2000-01-01')
df = pro.query('daily',
ts_code='600519.SH',
start_date='20100101',
end_date='20211217')
#df.index=pd.to_datetime(df.date)
df.index = pd.to_datetime(df.trade_date)
df = df.sort_index()
# Cycle Indicators
# Hilbert Transform
df['dcperiod'] = ta.HT_DCPERIOD(df.close)
df['dcphase'] = ta.HT_DCPHASE(df.close)
df['inhpase'], df['quadrature'] = ta.HT_PHASOR(df.close)
df['sine'], df['leadsine'] = sine, leadsine = ta.HT_SINE(df.close)
Created on Wed Nov 27 13:51:54 2019
@author: yuba316
"""
import pandas as pd
import datetime
import calendar
import time
import matplotlib.pyplot as plt
import talib as ta
import numpy as np
from scipy.stats import norm
import tushare as ts
pro = ts.pro_api("f6233ac191e0221b30736b8cbca6dc232b2824b217ef4a2bfd97d798")
#%% ===========================================================================
#
# 获取期权信息模块：依据每月期权到期日标的收盘价判断并获取次月平值/虚二档期权
#
# =============================================================================
def GetOpInfo(): # 获取50etf历史上所有档位的期权信息
etf_o_info = pro.opt_basic(
exchange='SSE',
fields='ts_code,name,call_put,exercise_price,maturity_date')
etf_o_info = etf_o_info.sort_values(
by=["maturity_date", "call_put",
"exercise_price"]) # 按到期月份/认购认沽/执行价从小到大排序
#print(type(stocks.ix[0,'name']))
#print(type(stocks.ix[0,'code']))
#print(type(stocks.ix[0,'industry']))
#print("%s[%06d]\n%s"%(stocks.ix[0,'name'],stocks.ix[0,'code'],stocks.ix[0,'industry']))
#print(type(stocks))
#data = pd.DataFrame(stocks,columns=['code','name','industry'])
#print(data)
#print(type(np.array(dt).tolist()))
#list =dt #np.array(dt).tolist()
#print(dt.ix[0])
#print(dt.index[0]+dt.ix[0,'name']+'\n'+dt.ix[0,'industry'])
#d = ts.get_hist_data('600848',start='2019-09-30', ktype='5') #获取5分钟k线数据
#pd.set_option("display.max_columns",500)
#rdata = ts.get_realtime_quotes('002504')
pro = ts.pro_api('604a0f99c257e0a0f3ae4ed6291a99e986f482be8083e561f09622ad')
#df = pro.daily(ts_code='002504.SH',start_date='20191008',end_date='20191019')
data = pro.stock_basic(exchange='',
list_status='L',
fields='ts_code,symbol,name,industry,market')
print(data)
df = pro.daily(ts_code='002504.SH', trade_date='20191018')
print(df)
def func():
for i in range(0, 200, 9):
listCode = []
for j in range(i, i + 9, 1):
if j >= 200:
break
pd.set_option('display.width', None) # 设置字符显示宽度
pd.set_option('display.max_rows', None) # 设置显示最大行
pd.set_option('display.max_columns', None) # 设置显示最大行
if __name__ == '__main__':
# 连接数据库
client = MongoClient('localhost', 27017)
db = client.futures
market = db.market
# 填写日期
begin = datetime.date(2019, 6, 10)
end = datetime.date(2019, 6, 10)
pro = fushare.pro_api(
'c0cad8f56caba4e70702d606290d04f88514a6bef046f60d13144151')
for i in range((end - begin).days + 1):
day = begin + datetime.timedelta(days=i)
days = day.strftime('%Y%m%d')
# df = pro.fut_daily(trade_date=days, exchange='',
# fields='ts_code,trade_date,pre_close,pre_settle,open,high,low,close,settle,vol')
df = pro.fut_holding(trade_date=days)
frames = [df]
print(df)
try:
# 合并四大交易所行情
df2 = pd.concat(frames)
# 计算品种指数收盘价
class NoticesSpider(scrapy.Spider):
name = 'notices'
allowed_domains = ['eastmoney.com']
start_urls = ['http://eastmoney.com/']
# 股票列表
shangshi = list(ts.pro_api().stock_basic(list_status='L')['ts_code'].drop_duplicates())
tuishi = list(ts.pro_api().stock_basic(list_status='D')['ts_code'].drop_duplicates())
zanting = list(ts.pro_api().stock_basic(list_status='P')['ts_code'].drop_duplicates())
ts_code_list = list(set(shangshi + tuishi + zanting))
code_list = [x.split('.')[0] for x in ts_code_list]
code_list.sort()
# code_list = ['000001', '000002']
url_ashx = "https://data.eastmoney.com/notices/getdata.ashx" # http有问题用https
# handle_httpstatus_list = [200, 403]
# 对应数据库
db = None
def start_requests(self):
""""
第一次请求数据。指定page_size，若未指定则请求该股票所有数据。
"""
self.db = MongoClient(self.settings.get('REMOTEMONGO')['uri'])[self.settings.get('REMOTEMONGO')['notices']]
# PAGE_SIZE参数
if self.__dict__.get('PAGE_SIZE', None):
p_page_size =int(self.__dict__.get('PAGE_SIZE', None)) # 命令行中 -a PAGE_SIZE=50
else:
p_page_size = self.settings.get('PAGE_SIZE') # settings.py中
if p_page_size:
to_parse = self.code_list
self.logger.info('增量更新PAGE_SIZE：{},to_parse股票数量：{}'.format(p_page_size, len(to_parse)))
for stk in to_parse:
item = NoticeItem()
item['code'] = stk
params = {
'StockCode': stk,
'CodeType': 1,
'PageIndex': 1,
'PageSize': p_page_size,
}
url = self.url_ashx + '?' + urllib.parse.urlencode(params)
yield scrapy.Request(
url=url, callback=self.parse, meta={'item': copy.deepcopy(item)}
)
else:
# existed = TextMongo().get_notices_stk()
# to_parse = list(set(self.code_list).difference(set(existed)))
to_parse = list(set(self.code_list))
to_parse.sort()
self.logger.info('全量更新：PAGE_SIZE为None,to_parse数量{}'.format(len(to_parse)))
for stk in to_parse:
item = NoticeItem()
item['code'] = stk
params = {
'StockCode': stk,
'CodeType': 1,
'PageIndex': 1, # 证券市场，hsa为1，必须要有，否则TotalCount会出问题。
'PageSize': 50,
}
url = self.url_ashx + '?' + urllib.parse.urlencode(params)
first = requests.get(url)
try:
page_size = ashx_json(first.text)['TotalCount']
except Exception as e:
self.logger.error(f'{e}')
page_size = 0 # 有些证券，网站没有数据。page_size为0，parse函数中会报错，所以眺过
continue
print('运行时间{}证券{}数据总数{}'.format(datetime.datetime.now(), item['code'], page_size))
page_total = floor(page_size/50)
for page_index in list(range(1, page_total+1)): # 分页取
params = {
'StockCode': stk,
'CodeType': 1,
'PageIndex': page_index,
'PageSize': 50,
}
url = self.url_ashx + '?' + urllib.parse.urlencode(params)
yield scrapy.Request(
url=url, callback=self.parse, meta={'item': copy.deepcopy(item)}
)
# fixme设置单页全取，有时会出错
# params = {
# 'StockCode': stk,
# 'CodeType': 1,
# 'PageIndex': 1,
# 'PageSize': page_size,
# }
# url = self.url_ashx + '?' + urllib.parse.urlencode(params)
# yield scrapy.Request(
# url=url, callback=self.parse, meta={'item': copy.deepcopy(item)}
# )
def parse(self, response):
"""
分析返回的数据结构，获取公告的摘要信息。
"""
item = response.meta['item']
assert item['code'] == re.findall(r'StockCode=(.*?)&', response.url)[0]
# 已存在的数据，且content不为空。
# TODO 按需设置有效数据的规则，例如pdf处理
# exsit_md5 = self.db[item['code']].find({'content_source': {'$ne': 0}}, {'_id': 1, 'href_md5': 1})
exsit_md5 = self.db[item['code']].find({'content_source': {'$in': [0, 1]}}, {'_id': 1, 'href_md5': 1})
exsit_md5 = [x.get('href_md5') for x in exsit_md5]
total = ashx_json(response.body_as_unicode())
if total.get('data') is None:
self.logger.error(f'{total}')
for each in total.get('data'):
item['ann_date'] = each.get('NOTICEDATE')
item['ann_title'] = each.get('NOTICETITLE')
item['ann_type'] = each.get('ANN_RELCOLUMNS')[0].get('COLUMNNAME') # 有些type不属于公告分类table，而是'其它' '股票'这种字段
item['href'] = each.get('Url')
item['href_md5'] = hashlib.md5(item['href'].encode('utf8')).hexdigest()
if item['href_md5'] in exsit_md5:
continue
copy_item = copy.deepcopy(item)
yield scrapy.Request(
copy_item['href'], callback=self.parse_content, meta={'item': copy_item}
)
def parse_content(self, response):
""" 获取公告对应的文本内容。"""
item = response.meta['item']
try:
temp = response.xpath("//div[@class='detail-body']/div/text()").extract()
temp = [x for x in temp if str(x).strip()]
temp = '\r\n'.join(temp)
item['content'] = temp
item['content_source'] = 1
except Exception as e:
self.logger.warning('链接文本为空{}'.format(item['href'])) # TODO 做pdf的提取
item['content'] = ''
item['content_source'] = 0
return item
def getUp10(start, end):
'''
弄一份10年到现在连续涨停超过3天的数据，去除St里面需要有这几天的日期和股票号码
我需要开始涨停的日期和结束的，这两个比较重要
不需要是完全连续一字涨停，只要连续三天收盘价是涨停就可以
新上市也帮我排除
'''
path = "./%sTo%s" % (start, end)
if (os.path.exists(path) == False): #判断目标是否存在
os.mkdir(path) #创建目录
pro = ts.pro_api()
i = 0
codeList = pro.query(
'stock_basic',
exchange='',
list_status='L',
fields='ts_code,symbol,name,list_date') #TS代码,代码,名称,上市日期
#stockCodeList =codeList['ts_code'].tolist()
dfResult = pd.DataFrame()
writerR = pd.ExcelWriter('resultTop10.xls')
# for code in stockCodeList:
for i in range(0, len(codeList)):
#code ='000651.SZ'
code = codeList.iloc[i]['ts_code']
list_date = codeList.iloc[i]['list_date']
name = codeList.iloc[i]['name']
if (name.find('ST') >= 0):
print('skip name=%s,code=%s' % (name, code))
continue
savefileName = path + '/' + code + '(' + start + 'To' + end + ').xls'
# i =i+1
if (os.path.exists(savefileName)):
print("...[%d]%s exist" % (i, savefileName))
continue
time.sleep(0.3)
print("...reading Stock =" + code + " from " + start + " to " + end +
" ...")
# df =ts.get_hist_data(code,start=start,end=end)
# df.rename(columns={'date':'日期', 'open':'开盘价','high':'最高价','close':'收盘价','low':'最低价','volume':'成交量','price_change':'价格变动','p_change':'涨跌幅','ma5':'5日均价','ma10':'10日均价','ma20':'20日均价','v_ma5':'5日均量','v_ma10':'10日均量','v_ma20':'20日均量','turnover':'换手率'},inplace = True)
df1 = pro.daily(ts_code=code, start_date=start, end_date=end)
df = df1.sort_values(by=['trade_date'])
df = df.reset_index()
df.drop(['index'], axis=1, inplace=True)
dfTop10 = df[df['pct_chg'] > 9.98]
for j in range(len(dfTop10)):
if (j >= 3):
if ((dfTop10.index[j] - dfTop10.index[j - 1]) == 1):
if ((dfTop10.index[j - 1] - dfTop10.index[j - 2]) == 1):
dfTmp = dfTop10.iloc[j - 2]
dfTmp['name'] = name
dfTmp['list_date'] = list_date
dfResult = dfResult.append(dfTmp)
dfTmp = dfTop10.iloc[j - 1]
dfTmp['name'] = name
dfTmp['list_date'] = list_date
dfResult = dfResult.append(dfTmp)
dfTmp = dfTop10.iloc[j]
dfTmp['name'] = name
dfTmp['list_date'] = list_date
dfResult = dfResult.append(dfTmp)
#df.rename(columns={'ts_code':'股票代码','trade_date':'交易日期','open':'开盘价','high':'最高价','low':'最低价','close':'收盘价','pre_close':'昨收价','change':'涨跌额','pct_chg':'涨跌幅(未复权)','vol':'成交量 （手）','amount':'成交额(千元)'}, inplace = True)
writer = pd.ExcelWriter(savefileName)
dfTop10.to_excel(writer, sheet_name=code)
writer.save()
print("...[%d]finish writing Stock =%s data to %s" %
(i, code, savefileName))
dfResult = dfResult.drop_duplicates()
order = [
'ts_code', 'name', 'trade_date', 'open', 'high', 'low', 'close',
'pre_close', 'change', 'pct_chg', 'vol', 'amount', 'list_date'
]
dfR = dfResult[order]
dfR.to_excel(writerR, sheet_name='Result')
writerR.save()
#!/usr/bin/python3
# -*- coding: UTF-8 -*-
import tushare as ts
pro = ts.pro_api('5683e168c0c9f0f7d9066cead21854067738e997dcd0e204275f8dd4')
data = pro.hsgt_top10(trade_date='20180725', market_type='1')
print(data)
df = pro.top_list(trade_date='20191119')
print(df)
df = pro.top_inst(trade_date='20191119')
print(df)
def model_eva(stock,state_dt,para_window,para_dc_window):
# 建立数据库连接，设置tushare token
db = pymysql.connect(host='127.0.0.1', user='******', passwd='root', db='stock', charset='utf8')
cursor = db.cursor()
ts.set_token('9479832d9c2b7719809d7a39f606172d37d27af2f1713f6814493dfc')
pro = ts.pro_api()
# print 'into_eva'*10
# 建评估时间序列, para_window参数代表回测窗口长度
model_test_date_start = (datetime.datetime.strptime(state_dt, '%Y-%m-%d') - datetime.timedelta(days=para_window)).strftime(
'%Y%m%d')
model_test_date_end = datetime.datetime.strptime(state_dt,('%Y-%m-%d')).strftime('%Y%m%d')
print "model_test_date_start,model_test_date_end: ",model_test_date_start,model_test_date_end
df = pro.trade_cal(exchange_id='', is_open = 1,start_date=model_test_date_start, end_date=model_test_date_end)
date_temp = list(df.iloc[:,1])
model_test_date_seq = [(datetime.datetime.strptime(x, "%Y%m%d")).strftime('%Y-%m-%d') for x in date_temp]
# 清空评估用的中间表model_ev_mid
sql_truncate_model_test = 'truncate table model_ev_mid'
cursor.execute(sql_truncate_model_test)
db.commit()
# print 'after_commit_'*10
return_flag = 0
# 开始回测，其中para_dc_window参数代表建模时数据预处理所需的时间窗长度
for d in range(len(model_test_date_seq)):
model_test_new_start = (datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d') - datetime.timedelta(days=para_dc_window)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
print "model_test_new_start,model_test_new_end: ",model_test_new_start,model_test_new_end
try:
dc = DC.data_collect(stock, model_test_new_start, model_test_new_end)
if len(set(dc.data_target)) <= 1:
continue
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 将预测结果插入到中间表
# print 'predict'*10
sql_insert = "insert into model_ev_mid(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (model_test_new_end, stock, float(ans2[0]))
# print sql_insert
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
return -1
else:
# 在中间表中刷真实值
for i in range(len(model_test_date_seq)):
sql_select = "select * from stock_all a where a.stock_code = '%s' and a.state_dt >= '%s' order by a.state_dt asc limit 2" % (stock, model_test_date_seq[i])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
if len(done_set2) <= 1:
break
resu = 0
if float(done_set2[1][3]) / float(done_set2[0][3]) > 1.00:
resu = 1
sql_update = "update model_ev_mid w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.stock_code = '%s'" % (resu, model_test_date_seq[i], stock)
cursor.execute(sql_update)
db.commit()
# 计算查全率
sql_resu_recall_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
if recall_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
recall = float(recall_son) / recall_mon
# 计算查准率
sql_resu_acc_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = float(acc_son) / acc_mon
# 计算查准率(负样本)
sql_resu_acc_neg_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg_mon = -1
acc_neg = -1
else:
acc_neg = float(acc_neg_son) / acc_neg_mon
# 计算 F1 分值
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from model_ev_mid a where a.state_dt = '%s'" % (model_test_date_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
# 将评估结果存入结果表model_ev_resu中
sql_final_insert = "insert into model_ev_resu(state_dt,stock_code,acc,recall,f1,acc_neg,bz,predict)values('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (state_dt, stock, acc, recall, f1, acc_neg, 'svm', str(predict))
cursor.execute(sql_final_insert)
db.commit()
db.close()
print(str(state_dt) + ' Precision : ' + str(acc) + ' Recall : ' + str(recall) + ' F1 : ' + str(f1) + ' Acc_Neg : ' + str(acc_neg))
return 1
import tushare as ts
import pandas as pd
ts.set_token('7844a61f5276d7889cdc4171e5081161ca3037e7a58be41487650a95')
pro = ts.pro_api('7844a61f5276d7889cdc4171e5081161ca3037e7a58be41487650a95')
data = pro.stock_basic(exchange='',
list_status='L',
fields='ts_code,symbol,name,area,industry,list_date')
# data = pro.query('stock_basic', exchange='', list_status='L', fields='ts_code,symbol,name,area,industry,list_date')
# print(data['ts_code'])
stock_list = data['ts_code']
print(stock_list)
for n, i in enumerate(stock_list):
try:
print(n)
print(i)
df = ts.pro_bar(pro_api=pro,
ts_code=str(i),
adj='None',
start_date='20171001',
end_date='20191001',
freq='D',
ma=[5, 10, 20],
factors=['tor', 'vr'])
print(df)
df.to_excel(
'/Users/eminem/Desktop/stock_data/stock_D3/{}.xlsx'.format(i))
import tushare as ts
import pandas as pd
from sqlalchemy import create_engine
import time
pro = ts.pro_api('a0630d57359280860360068ff19e3073cdb73b5485eed124083a74fe')
dates = pro.trade_cal(exchange='', start_date='20190103', end_date='20190523')
x = dates['cal_date'].loc[dates['is_open'].isin(['1'])]
for date in x:
print(date)
data = pro.daily(trade_date=date)
print(data)
print(date)
myconn = create_engine(
'mysql+pymysql://gangch:@[email protected]:3306/gangch?charset=utf8'
)
pd.io.sql.to_sql(data,
'stock_kd',
myconn,
schema='gangch',
if_exists='append')
time.sleep(30)
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import matplotlib.animation as animation
import tushare as ts
from datetime import timedelta, datetime
pro = ts.pro_api('b7bbd6907437d1f894145e7487709d27ee220b6c3772205ef05c83fb')
days = 4
now = datetime.now().date()
fd = now - timedelta(days=days)
ax = None
# code = "000425"
code = '600703'
for d in [str(fd + timedelta(days=i)) for i in range(days + 1)]:
df = ts.get_tick_data(code, date=d, src='tt')
a = df.groupby(["price"]).agg({"volume": "sum"})
a = a.head(90)
if not ax:
ax = a.plot()
a.plot(ax=ax, figsize=(30, 10), legend=d)
plt.xticks(rotation=30)
plt.title(d)
plt.show()
def __init__(self):
cfg = ConfigFile()
tushare_conf = cfg.get_dict(constants.TUSHARE_KEY)
self.key = tushare_conf["key"]
ts.set_token(self.key)
self.ts_pro = ts.pro_api()
def __init__(self):
self.pro = ts.pro_api(
'2b9929516221569516ff57324dc130c6100a76fa66d4202de854ba58')
def __fetch_stock_holder_data(**kwargs) -> pd.DataFrame:
uri = kwargs.get('uri')
result = check_execute_test_flag(**kwargs)
if result is None:
# period = kwargs.get('due_date')
ts_code = pickup_ts_code(kwargs)
# since, until = normalize_time_serial(period, default_since(), today())
pro = ts.pro_api(TS_TOKEN)
# time_iter = DateTimeIterator(since, until)
#
# result = None
# while not time_iter.end():
# # The max items count retrieved per 1 fetching: 1000
# # The max items per 1 year: 52 (one new item per 7days for PledgeStatus)
# # So the iter years should not be larger than 20 years
#
# sub_since, sub_until = time_iter.iter_years(15)
# ts_since = sub_since.strftime('%Y%m%d')
# ts_until = sub_until.strftime('%Y%m%d')
#
# clock = Clock()
# delayer.delay()
# if uri == 'Stockholder.PledgeStatus':
# sub_result = pro.pledge_stat(ts_code=ts_code, start_date=ts_since, end_date=ts_until)
# elif uri == 'Stockholder.PledgeHistory':
# sub_result = pro.pledge_detail(ts_code=ts_code, start_date=ts_since, end_date=ts_until)
# else:
# sub_result = None
# print(uri + ' Network finished, time spending: ' + str(clock.elapsed_ms()) + 'ms')
#
# if sub_result is not None:
# if result is None:
# result = sub_result
# else:
# result.append(result)
if not str_available(ts_code):
result = None
else:
clock = Clock()
if uri == 'Stockholder.PledgeStatus':
ts_delay('pledge_stat')
result = pro.pledge_stat(ts_code=ts_code)
elif uri == 'Stockholder.PledgeHistory':
ts_delay('pledge_detail')
result = pro.pledge_detail(ts_code=ts_code)
else:
result = None
print('%s: [%s] - Network finished, time spending: %sms' %
(uri, ts_code, clock.elapsed_ms()))
check_execute_dump_flag(result, **kwargs)
if result is not None:
result.fillna(0.0)
if uri == 'Stockholder.PledgeStatus':
result['due_date'] = result['end_date']
result['total_share'] = result['total_share'] * 10000
result['rest_pledge'] = result['rest_pledge'] * 10000
result['unrest_pledge'] = result['unrest_pledge'] * 10000
result['pledge_count'] = result['pledge_count'].astype(np.int64)
result['pledge_ratio'] = result['pledge_ratio'].astype(float)
elif uri == 'Stockholder.PledgeHistory':
result['due_date'] = result['ann_date']
result['pledge_amount'] = result['pledge_amount'] * 10000
result['holding_amount'] = result['holding_amount'] * 10000
result['pledged_amount'] = result['pledged_amount'] * 10000
convert_ts_code_field(result)
convert_ts_date_field(result, 'due_date')
# result['due_date'] = pd.to_datetime(result['due_date'])
# result['stock_identity'] = result['ts_code']
# result['stock_identity'] = result['stock_identity'].str.replace('.SH', '.SSE')
# result['stock_identity'] = result['stock_identity'].str.replace('.SZ', '.SZSE')
return result
def insertOneDay(conn, date, tb_name):
#get ts data
print date
time.sleep(1)
pro = ts.pro_api()
df = 0
num = 0
while True:
num += 1
try:
df = pro.daily(trade_date=date)
except Exception as e:
print e
print e.message
print(type(e))
print dir(e)
print u"第{}次从ts查询{}数据失败！".format(str(num), date)
if (num < 10):
time.sleep(3)
continue
else:
exit(0)
else:
break
# try:
# df = pro.daily(ts_code=ts_code)
# except Exception as e:
# print e
# print e.message
# print u"从ts查询{}日线数据再次失败！".format(ts_code)
# exit(0)
if (df.empty):
print u"{}没有数据！".format(date)
return
vlist = []
for i in df.index:
arr = df.iloc[i].tolist()
for k in range(len(arr)):
if (k < 2):
# arr[k] = arr[k].encode('utf-8')
pass
else:
arr[k] = float(arr[k])
tup = tuple(arr)
vlist.append(tup)
# 存到数据库
cursor = conn.cursor()
sql = 'insert into {} values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(
tb_name)
# print sql
try:
cursor.executemany(sql, vlist[:])
conn.commit()
except Exception as e:
print e
# print e.read()
print u"执行插入{}数据失败，回滚！".format(date)
conn.rollback()
cursor.close()
def flagTop10(stockCodeList, start, end):
'''
最高价等于昨收价的1.1倍（四舍五入）
并且6月1日的最高价未高于其前5个交易的最高价（即前5个交易日有价格高于6月1日的最高价即可）
数据符合条件 然后在该行后标记个1，在其后面一行（72行）标记为2
做标记的目的是方便筛选
'''
path = "./%sTo%s" % (start, end)
if (os.path.exists(path) == False): #判断目标是否存在
os.mkdir(path) #创建目录
pro = ts.pro_api()
i = 0
for code in stockCodeList:
savefileName = path + '/' + code + '(' + start + 'To' + end + ').xls'
i = i + 1
if (os.path.exists(savefileName)):
print("...[%d]%s exist" % (i, savefileName))
continue
time.sleep(0.3)
print("...reading Stock =" + code + " from " + start + " to " + end +
" ...")
# df =ts.get_hist_data(code,start=start,end=end)
# df.rename(columns={'date':'日期', 'open':'开盘价','high':'最高价','close':'收盘价','low':'最低价','volume':'成交量','price_change':'价格变动','p_change':'涨跌幅','ma5':'5日均价','ma10':'10日均价','ma20':'20日均价','v_ma5':'5日均量','v_ma10':'10日均量','v_ma20':'20日均量','turnover':'换手率'},inplace = True)
df1 = pro.daily(ts_code=code, start_date=start, end_date=end)
df = df1.sort_values(by=['trade_date'])
####
#比如002321这份数据 第71行（2018年6月1日）
# 最高价等于昨收价的1.1倍（四舍五入）
# 并且6月1日的最高价未高于其前5个交易的最高价（即前5个交易日有价格高于6月1日的最高价即可）
# 数据符合条件 然后在该行后标记个1，在其后面一行（72行）标记为2
# 做标记的目的是方便筛选
df = df.reset_index()
df.drop(['index'], axis=1, inplace=True)
# df.drop(['index'],axis=1,inplace=True)
df_temp = df[round(df['high'] / df['pre_close'], 2) == 1.1]
df['flag'] = ''
for j in range(len(df_temp)):
ind = df_temp.index[j]
if (ind > 5):
value = df.iloc[ind]['high']
if ((df.iloc[ind - 1]['high'] >= value)
or (df.iloc[ind - 2]['high'] >= value)
or (df.iloc[ind - 3]['high'] >= value)
or (df.iloc[ind - 4]['high'] >= value)
or (df.iloc[ind - 5]['high'] >= value)):
df.loc[ind, 'flag'] = 1
df.loc[ind + 1, 'flag'] = 2
####
df['最高涨幅'] = df['high'] / df['pre_close'] - 1
df['最低涨幅'] = df['low'] / df['pre_close'] - 1
df['收盘涨幅'] = df['close'] / df['pre_close'] - 1
df.rename(columns={
'ts_code': '股票代码',
'trade_date': '交易日期',
'open': '开盘价',
'high': '最高价',
'low': '最低价',
'close': '收盘价',
'pre_close': '昨收价',
'change': '涨跌额',
'pct_chg': '涨跌幅(未复权)',
'vol': '成交量 （手）',
'amount': '成交额(千元)'
},
inplace=True)
order = [
'股票代码', '交易日期', '开盘价', '最高价', '最高涨幅', '最低价', '最低涨幅', '收盘价', '收盘涨幅',
'昨收价', '涨跌额', '涨跌幅(未复权)', '成交量 （手）', '成交额(千元)', 'flag'
]
dfR = df[order]
writer = pd.ExcelWriter(savefileName)
dfR.to_excel(writer, sheet_name=code)
writer.save()
print("...[%d]finish writing Stock =%s data to %s" %
(i, code, savefileName))
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 15 09:35:29 2020
@author: yuba316
"""
import copy
import math
import numpy as np
import pandas as pd
import datetime
import matplotlib.pyplot as plt
import tushare as ts
pro = ts.pro_api('da949c80ceb5513dcc45b50ba0b0dec1bc518132101bec0dfb19da56')
'''
stock = pd.read_csv(r"D:\work\back_test_system\DataBase\Stock\Stock.csv")
stock['trade_date'] = stock['trade_date'].apply(str)
factor = stock[['trade_date','ts_code','close','pre_close']]
del stock
factor['score'] = factor['close']/factor['pre_close']-1
factor.drop(['pre_close'],axis=1,inplace=True)
factor.reset_index(drop=True,inplace=True)
'''
#%%
def FactorTest(df, freq=5, pct=0.1):
factor = copy.deepcopy(df)
trade_date = list(np.sort(factor['trade_date'].unique()))
import pandas_datareader as web
import matplotlib.pyplot as plt
import tushare as ts
import numpy as np
a = np.array([1, 2, 3])
print(type(a))
# tushare
# web.DataReader
# 读取融捷股份的数据
# RJ = web.DataReader('300450.SZ',data_source='yahoo',start='3/01/2015')
# print(RJ.info())
# print(RJ.tail())
# 画图
# RJ['Close'].plot(figsize=(8,5))
# # 单个对数收益率
# plt.show()
token = 'bfbf67e56f47ef62e570fc6595d57909f9fc516d3749458e2eb6186a'
ts_code = '002192.SZ'
startday = '20150101'
endday = '20190417'
pro = ts.pro_api(token)
# 获取数据
RJ = pro.daily(ts_code=ts_code, start_date=startday, end_date=endday)
# print(RJ.info())
print(RJ['trade_date'][RJ['vol'] > 100000])
print('初始金额: %.2f' % cerebro.broker.getvalue())
back = cerebro.run()
#总份额，年化，回撤，夏普
ratio_list=[[
x.analyzers.SP.get_analysis()['sharperatio'],#夏普比率
x.analyzers.RE.get_analysis()['rtot']*100, #总复合收益率
x.analyzers.DD.get_analysis()['max']['drawdown'], #最大回撤
x.analyzers.DD.get_analysis()['max']['len']]#最大回撤周期
for x in back] #夏普
ratio_df = pd.DataFrame(ratio_list,columns=['夏普','年化%','最大回撤','最大回撤周期'])
print(ratio_df)
print('历年收益率：%s' %back[0].analyzers.AR.get_analysis())#每年年化收益率
print('标准收益率：%s' %back[0].analyzers.RE.get_analysis()['rnorm100']) # 年化标准化回报以100%展示
cerebro.plot(b)
# cerebro.plot(style='candle')
print('最终收益: %.2f' % cerebro.broker.getvalue())
if __name__ == '__main__':
# stockpool=['600884.sh','601012.sh','300059.sz']
stockpool=['600884.sh']
pro = ts.pro_api('ebe4734e785004ada3e0f4e03da59a5dee8c7da0b7820ce5c50fb30e')
runstart()
# -*- coding: UTF-8 -*- import urllib.request, http.client, json, time, datetime import os import settings import django import sys import time import tushare as ts # 300875 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) os.environ['DJANGO_SETTINGS_MODULE'] = 'gjsicence.settings' django.setup() from server.models import Stock, FinancialStatement, IncomeStatement pro = ts.pro_api(token="c3ebbfac31ace68059c0acae3c76eff6a391d09fa59fd20849fca141") stock_list = Stock.objects.all() date_list = ["20161231", ] for date in date_list: print(date) for stock in stock_list: exchange = ".SZ" if stock.exchange == "SZSE" else ".SH" time.sleep(1.2) df = pro.income(ts_code=stock.symbol+exchange, period=date) IncomeStatement.objects.create( stock_code=stock.symbol, exchange=stock.exchange, publish_date=datetime.datetime(int(date[0:4]), int(date[4:6]), int(date[6:8])), statement_content=json.dumps(df.to_json()) )
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 30 11:55:29 2020
获取月线数据并入库
@author: 李博
"""
import os
import pandas as pd
import json
import datetime
import time
# MONGODB CONNECT
import tushare as ts
pro = ts.pro_api('003137d7baa1439f01f9d2917992de6b8511f70f84612c78574d6996')
today = time.strftime('%Y%m%d', )
#计算指定日期的前N天的时间戳
def get_day_time(n):
#the_date = datetime.datetime(2018,11,10) #指定当前日期 2018-11-10
the_date = datetime.datetime.now()
#the_date_str = the_date.strftime('%Y%m%d')
pre_date = the_date - datetime.timedelta(days=n - 1)
pre_date_str = pre_date.strftime('%Y%m%d') #将日期转换为指定的显示格式
return pre_date_str
#获取当前分类的成份股
def getstocks(ccode):
import time
import tushare as ts
import pandas as pd
from datetime import datetime
import datetime as dt
from django.db.models import Min, Max
from stock.models import Stock, Share, DailyBasic
from django.core.cache import cache
from stock_web.settings import TS_TOKEN
pro = ts.pro_api(TS_TOKEN)
def load_shares_from_api():
"""
从tushare api获取shares数据
"""
current_date = datetime.now().strftime('%Y%m%d')
# 在日志中记录
print(current_date, 'shares')
fields = ('ts_code', 'end_date', 'ann_date', 'div_proc', 'stk_div',
'stk_bo_rate', 'stk_co_rate', 'cash_div', 'cash_div_tax',
'record_date', 'ex_date', 'pay_date', 'div_listdate',
'imp_ann_date', 'base_date', 'base_share')
# 设空dividend合并
dividend = pd.DataFrame(columns=fields)
# 前溯一周
for delta in range(32):
current_date = (datetime.now() -
dt.timedelta(days=delta)).strftime('%Y%m%d')
import sqlite3;
import requests;
from pytdx.hq import TdxHq_API
import tushare as ts
from numpy import *
import pandas as pd
#48710947193021c24febb57e1b7b84ffa335577e50deff62f4aec47d
#取000001的前复权行情
print(ts.__version__)
ts.set_token('48710947193021c24febb57e1b7b84ffa335577e50deff62f4aec47d')
api = ts.pro_api()
#df = ts.pro_bar(pro_api=api, ts_code='600886.SH', adj='qfq', start_date='20180101', end_date='20190124')
#df = ts.pro_bar(pro_api=api, ts_code='600070.SH', adj='qfq')
df = api.adj_factor(ts_code='300197.SZ', trade_date='20180524')
print (df)
df1 = api.adj_factor(ts_code='300197.SZ')
print (df1)
df2 = api.daily(ts_code='300197.SZ')
print (df2)
result = pd.merge(df1, df2, how='right', on=['trade_date'])
print (result)
print(result.iat[0,2])
df3 = api.daily(ts_code='511880.SH')
print (df3)
from django.shortcuts import render
from django.http import HttpResponse, Http404
# Create your views here.
import tushare as ts
pro = ts.pro_api("d9f49767544519208fbf91e00a109558fe92e84bdcb70c9173144c24")
import pandas as pd
from .models import Stock, StockDetail, getModel
from django.db.models import Sum
def index(request):
industrys = list(set(Stock.objects.all().values_list('industry')))
industrys = [i[0] for i in industrys]
#ss = StockDetail.objects.all()
#reg = StockDetail.objects.values('net_mf_vol').annotate(number=Sum())
return render(request, "index.html", locals())
def detail(request, industry):
arg = request.GET.get("order_by")
if arg:
if industry == "all_infos":
ary1 = StockDetail.objects.all().order_by(arg)
return render(request, "detail.html", locals())
ary1 = StockDetail.objects.filter(industry=industry).order_by(arg)
if ary1:
return render(request, "detail.html", locals())
return render(request, "detail.html", {"industry": industry})
else:
import pymongo
import tushare as ts
from sqlalchemy import create_engine
import psycopg2
import warnings
warnings.filterwarnings("ignore")
import QUANTAXIS as QA
from QUANTAXIS.QAUtil.QALogs import (QA_util_log_debug, QA_util_log_expection,
QA_util_log_info)
from QUANTAXIS.QAUtil.QADate_trade import QA_util_get_trade_range
from QUANTAXIS.QAUtil.QADate import (QA_util_date_str2int,QA_util_date_int2str)
token='xxxx' #你的tusharepro token
pro=ts.pro_api(token)
def cilent_pg():
Account='postgres'#pg账号
Password='******' #pg密码
database='quantaxis' #该数据库需要提前手动建立，应该有自动建立方法，还不会。。。
client = create_engine('postgresql+psycopg2://'+Account+':'+Password+'@127.0.0.1:5432/'+database)
return client
def save_data_to_postgresql(name_biao,data,if_exists='replace',client=cilent_pg()):
data.to_sql(name_biao,client,index=False,if_exists=if_exists)
def load_data_from_postgresql(mes='',client=cilent_pg()):
res=pd.read_sql(mes,client)
return res
def download_day_data_from_tushare(trade_date='20190102'):
def __init__(self, token=_TOKEN):
self.pro = ts.pro_api(token=token)
self._daily = {}
self._basic = {}
self._trade_dates = None
import requests as req #web请求相关
import re, json, sys, datetime
from dateutil.relativedelta import relativedelta
import prettytable as pt #格式化成表格输出到html文件
from pyecharts.charts import Bar, Page, Line #画图
from pyecharts import options as opts
import pandas as pd #数据读取
import webbrowser #打开浏览器
'''手动安装 talib 去https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib 下载对应的版本“TA_Lib‑0.4.19‑cp37‑cp37m‑win_amd64.whl” 然后 pip3 install TA_Lib‑0.4.19‑cp37‑cp37m‑win_amd64.whl'''
import talib #Technical Analysis Library”, 即技术分析库 是Python金融量化的高级库，涵盖了150多种股票、期货交易软件中常用的技术分析指标，如MACD、RSI、KDJ、动量指标、布林带等等。
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec #分割子图
import mpl_finance as mpf # python中可以用来画出蜡烛图、线图的分析工具，目前已经从matplotlib中独立出来，非常适合用来画K线
import tushare as ts
pro = ts.pro_api('d0bf482fc51bedbefa41bb38877e169a43d00bd9ebfa1f21d28151c7')
import warnings
warnings.filterwarnings('ignore') #控制台不输出warning 信息
#ts.set_token('d0bf482fc51bedbefa41bb38877e169a43d00bd9ebfa1f21d28151c7')
import baostock as bs
outfile = ''
###获取股票代码
def get_stockcode(stockname):
if stockname.isdigit(): #如果输入的是代码
return stockname
else:
stockdata = pd.DataFrame(
pro.stock_basic(
import pandas as pd
import time
import pymysql
import tushare as ts
import config
from setting import get_mysql_conn
conn = get_mysql_conn('db_stock', 'local')
cursor = conn.cursor()
token = config.token
ts.set_token(token)
pro = ts.pro_api()
def get_stock_list():
df = pro.stock_basic(exchange='', list_status='L', fields='ts_code,symbol,name,area,industry,list_date')
return dict(zip(list(df['ts_code'].values), list(df['name'].values)))
# 生产日期 2000到2018
def create_date():
start_date = '20{}0101'
end_date = '20{}1231'
date_list = []
for i in range(18, 0, -1):
print(start_date.format(str(i).zfill(2)))
print(end_date.format(str(i).zfill(2)))
import tushare as ts
import datetime
import pymysql
pro = ts.pro_api('ef567216647a8a6c69c71ccace7803d6415db95b3478d1e780d81f7c')
start_dt = '20100101'
time_temp = datetime.datetime.now() - datetime.timedelta(days=1)
end_dt = time_temp.strftime('%Y%m%d')
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='123456',
db='stock',
charset='utf8')
cursor = db.cursor()
stock_pool = ['603912.SH', '300666.SZ', '300618.SZ', '002049.SZ', '300672.SZ']
total = len(stock_pool)
for i in range(total):
try:
df = pro.daily(ts_code=stock_pool[i],
start_date=start_dt,
end_date=end_dt)
print('Seq: ' + str(i + 1) + ' of ' + str(total) + ' Code: ' +
str(stock_pool[i]))
c_len = df.shape[0]
except Exception as aa:
print(aa)
print('No DATA Code: ' + str(i))
continue
for j in range(c_len):
resu0 = list(df.ix[c_len - 1 - j])
resu = []
def suspendInfo():
pro = ts.pro_api(settings.TOKEN)
df = pro.suspend(resume_date=datetime.today().strftime("%Y%m%d"), fields='')
return df.to_csv()
def __fetch_stock_holder_statistics(**kwargs) -> pd.DataFrame or None:
uri = kwargs.get('uri')
result = check_execute_test_flag(**kwargs)
if result is None:
period = kwargs.get('period')
ts_code = pickup_ts_code(kwargs)
since, until = normalize_time_serial(period, default_since(), today())
# See TushareApi.xlsx
# since_limit = years_ago_of(until, 3)
# since = max([since, since_limit])
pro = ts.pro_api(TS_TOKEN)
ts_since = since.strftime('%Y%m%d')
ts_until = until.strftime('%Y%m%d')
if is_slice_update(ts_code, since, until):
result = None
else:
clock = Clock()
ts_delay('top10_holders')
result_top10 = pro.top10_holders(ts_code=ts_code,
start_date=ts_since,
end_date=ts_until)
ts_delay('top10_floatholders')
result_top10_float = pro.top10_floatholders(ts_code=ts_code,
start_date=ts_since,
end_date=ts_until)
print('%s: [%s] - Network finished, time spending: %sms' %
(uri, ts_code, clock.elapsed_ms()))
# Process top10_holders data
del result_top10['ts_code']
convert_ts_date_field(result_top10, 'ann_date')
convert_ts_date_field(result_top10, 'end_date')
result_top10 = result_top10.fillna('')
grouped_stockholder_top_10 = result_top10.groupby('end_date')
data_dict = {'period': [], 'stockholder_top10': []}
for g, df in grouped_stockholder_top_10:
data_dict['period'].append(g)
del df['end_date']
data_dict['stockholder_top10'].append(df.to_dict('records'))
grouped_stockholder_top_10_df = pd.DataFrame(data_dict)
grouped_stockholder_top_10_df[
'stock_identity'] = ts_code_to_stock_identity(ts_code)
# Process top10_floatholders data
del result_top10_float['ts_code']
convert_ts_date_field(result_top10_float, 'ann_date')
convert_ts_date_field(result_top10_float, 'end_date')
result_top10_float = result_top10_float.fillna('')
grouped_stockholder_top_10_float = result_top10_float.groupby(
'end_date')
data_dict = {'period': [], 'stockholder_top10_float': []}
for g, df in grouped_stockholder_top_10_float:
data_dict['period'].append(g)
del df['end_date']
data_dict['stockholder_top10_float'].append(
df.to_dict('records'))
grouped_stockholder_top_10_float_df = pd.DataFrame(data_dict)
grouped_stockholder_top_10_float_df[
'stock_identity'] = ts_code_to_stock_identity(ts_code)
# Merge together
result = pd.merge(grouped_stockholder_top_10_df,
grouped_stockholder_top_10_float_df,
on=['stock_identity', 'period'],
how='outer')
result = result.sort_values('period')
# 002978.SZ
# 20070518 - 20200517
# top10_floatholders() may get empty DataFrame
# if isinstance(result_top10, pd.DataFrame) and not result_top10.empty:
# del result_top10['ts_code']
# del result_top10['ann_date']
#
# result_top10.fillna(0.0)
# result_top10['hold_ratio'] = result_top10['hold_ratio'] / 100
#
# result_top10_grouped = pd.DataFrame({'stockholder_top10': result_top10.groupby('end_date').apply(
# lambda x: x.drop('end_date', axis=1).to_dict('records'))}).reset_index()
# else:
# result_top10_grouped = None
#
# if result_top10_float is not None and len(result_top10_float) > 0:
# del result_top10_float['ts_code']
# del result_top10_float['ann_date']
#
# result_top10_float_grouped = pd.DataFrame({'stockholder_top10_nt': result_top10_float.groupby('end_date').apply(
# lambda x: x.drop('end_date', axis=1).to_dict('records'))}).reset_index()
# else:
# result_top10_float_grouped = None
#
# if result_count is None or result_top10 is None or result_top10_float is None:
# print('Fetch stockholder statistics data fail.')
# return None
#
# result = result_top10_grouped \
# if result_top10_grouped is not None and len(result_top10_grouped) > 0 else None
# result = pd.merge(result, result_top10_float_grouped, how='outer', on='end_date', sort=False) \
# if result is not None else result_top10_float_grouped
# result = pd.merge(result, result_count, how='left', on='end_date', sort=False) \
# if result is not None else result_count
# result['ts_code'] = ts_code
# del result_top10['ts_code']
# del result_top10['ann_date']
# del result_top10_float['ts_code']
# del result_top10_float['ann_date']
#
# result_top10.fillna(0.0)
# result_top10['hold_ratio'] = result_top10['hold_ratio'] / 100
#
# try:
# result_top10_grouped = pd.DataFrame({'stockholder_top10': result_top10.groupby('end_date').apply(
# lambda x: x.drop('end_date', axis=1).to_dict('records'))}).reset_index()
# result_top10_float_grouped = pd.DataFrame({'stockholder_top10_nt': result_top10_float.groupby('end_date').apply(
# lambda x: x.drop('end_date', axis=1).to_dict('records'))}).reset_index()
#
# result = pd.merge(result_top10_grouped, result_top10_float_grouped, how='outer', on='end_date', sort=False)
# result = pd.merge(result, result_count, how='left', on='end_date', sort=False)
# result['ts_code'] = ts_code
# except Exception as e:
# print('Fetching stockholder data error:')
# print(e)
# print(traceback.format_exc())
# finally:
# pass
# # Ts data may have issues, just detect it.
# for index, row in result.iterrows():
# end_date = row['end_date']
# stockholder_top10 = row['stockholder_top10']
# stockholder_top10_nt = row['stockholder_top10_nt']
#
# if isinstance(stockholder_top10, list):
# if len(stockholder_top10) != 10:
# print('%s: stockholder_top10 length is %s' % (end_date, len(stockholder_top10)))
# else:
# print('%s: stockholder_top10 type error %s' % (end_date, str(stockholder_top10)))
#
# if isinstance(stockholder_top10_nt, list):
# if len(stockholder_top10_nt) != 10:
# print('%s: stockholder_top10_nt length is %s' % (end_date, len(stockholder_top10_nt)))
# else:
# print('%s: stockholder_top10 type error %s' % (end_date, str(stockholder_top10_nt)))
check_execute_dump_flag(result, **kwargs)
# if result is not None:
# result.fillna('')
# result['period'] = pd.to_datetime(result['end_date'])
# result['stock_identity'] = result['ts_code']
# result['stock_identity'] = result['stock_identity'].str.replace('.SH', '.SSE')
# result['stock_identity'] = result['stock_identity'].str.replace('.SZ', '.SZSE')
return result
