def Get_gnbk(self, code):
#做第一次操作
data = Mysql().ReadsqlMysql('test3', code)
#做第二个值的时候应该切换
# data = Mysql().ReadMySql('stock_gainian','stock_org_basics_'+code)
#库要做修改
Mysql().SaveMySqlEight('stock_gainian', 'stock_org_basics_new', data)
def IntoData(self):
#获取行业信息
c_name,c_code,c_code_cname = MyXlsx().ReadXlsx('c_name.xlsx')
#获取股票列表信息
data = Mysql().ReadMySql('Stock_Dasic_Data','stock_basics')
#合并行业信息和股票列表信息
results = data.merge(c_name,left_on='code',right_on='code',how='outer')
#20177-11-23修复数据没有的情况下默认值的问题
result = results.fillna('0')
# result.to_csv('chenjietest.csv')
Mysql().SaveAppendMySql(result,'Stock_Dasic_Data','stock_basics')
Mysql().SaveAppendMySql(c_code,'Stock_Dasic_Data','stock_plet_basics')
Mysql().SaveAppendMySql(c_code_cname,'Stock_Dasic_Data','stock_plate')
print "=============== data finish=================="
def Get_5_10_20_data(self, code, start, end):
data_5_data = ts.get_hist_data(code, start=start, end=end)
#防止过多拉取数据源 网站屏蔽IP
cons = ts.get_apis()
# 增加前复权 adj='qfq' 后复权 adj='hfq'（bob_jie ： 2017-11-01)
data_5_data = ts.bar(code,
conn=cons,
start_date=start,
adj='qfq',
end_date=end,
ma=[5, 10, 20],
factors=['vr', 'tor'])
try:
data_5_data.values[0][0]
except Exception as e:
time.sleep(2)
#防止过多拉取数据源 网站屏蔽IP
cons = ts.get_apis()
# 增加前复权 adj='qfq' 后复权 adj='hfq'（bob_jie ： 2017-11-01)
data_5_data = ts.bar(code,
conn=cons,
start_date=start,
adj='qfq',
end_date=end,
ma=[5, 10, 20],
factors=['vr', 'tor'])
ts.get_h_data('002337', autype='qfq')
Mysql().SaveMySqlTWO(data_5_data, 'Stock_Basics_Info_All',
code + 'stock_basics', code)
def OrgData(self):
#获取股票列表信息
data = Mysql().ReadMySql('test4','stock_basics')
for indexs in data.index:
try:
CleanDataSource().CleanDataOrgInfoTwo(data, indexs)
except:
continue;
print "=============== data finish=================="
def CleanData(self):
count = 1
#如果，确认这个表存在 2017-10-30
# StockInfo().Getstock_basics()
data = Mysql().ReadMySql('Stock_Dasic_Data','stock_basics')
pool = ThreadPoolExecutor(max_workers=40) # 创建一个最大可容纳2个task的线程池(开启线程池)
futures = []
for indexs in data.index:
futures.append(pool.submit(
CleanDataSource().CleanDataChenjie,data,indexs))
print wait(futures)
def PeData(self):
#获取股票列表信息
data = Mysql().ReadMySql('Stock_Dasic_Data','stock_basics')
#开启多线程
pool = ThreadPoolExecutor(max_workers=40) # 创建一个最大可容纳2个task的线程池(开启线程池)
futures = []
for indexs in data.index:
futures.append(pool.submit(
CleanDataSource().CleanDataPeAll,data,indexs))
print wait(futures)
print "=============== data finish=================="
def Get_TradeCal_data(self, start, end):
token = "7f47d59fa3d950a1de2026fbbb72220681968623a43123bc4e2d732c7edeec75"
URL = 'https://api.wmcloud.com/data/v1//api/master/getTradeCal.json?field=&exchangeCD=XSHG,XSHE&beginDate=' + start + '&endDate=' + end
# /api/master/getTradeCal.json?field=&exchangeCD=XSHG,XSHE&beginDate=&endDate=
Requtes = Requests().GetRequests(URL,
headers={
"Authorization":
"Bearer " + token,
"Accept-Encoding": "gzip, deflate"
})
Mysql().SaveMySqlThroue(Requtes, 'Stock_Dasic_Data', 'stock_TradeCal')
def CleanDataOrg(self):
count = 1
#如果，确认这个表存在 2017-10-30
# StockInfo().Getstock_basics()
# data = Mysql().ReadMySql('test3','stock_basics')
data = Mysql().ReadMySql('stock_gainian','shaixuan')
pool = ThreadPoolExecutor(max_workers=2) # 创建一个最大可容纳2个task的线程池(开启线程池)
futures = []
for indexs in data.index:
futures.append(pool.submit(
CleanDataSource().CleanDataOrgInfoThree,data,indexs))
print wait(futures)
def Get_pct_chg_Info(self, code, data, start_data, end_data, expend):
if code[0] == '0' or code[0] == '3': codenew = code + '.SZ'
else: codenew = code + '.SH'
#启动wind客户端
WindStock().Init_Wind()
#拉取wind数据
wsd_data = WindStock().Get_Wind_Info(codenew, data, start_data,
end_data, expend)
#存入数据库
Mysql().SaveMySqlSix(code, wsd_data, 'test3',
'stock_org_basics_copy_new')
Get_Organization_Info
def Get_sz_zz_Info(self, code, data, start_data, end_data, expend):
# if code[0]=='0' or code[0]=='3':codenew = code+'.SZ'
# else:codenew = code+'.SH'
#启动wind客户端
try:
WindStock().Init_Wind()
#拉取wind数据
wsd_data = WindStock().Get_Wind_Info(code, data, start_data,
end_data, expend)
#存入数据库
Mysql().SaveMySqlSix(code, wsd_data, 'zz_data',
'stock_zz_basics_' + code)
except Exception as e:
raise e
def JshuanGY(self):
data = Mysql().ReadMySql('test4','stock_basics')
# data = Mysql().ReadMySql('stock_gainian','shaixuan')
pool = ThreadPoolExecutor(max_workers=0)# 创建一个最大可容纳2个task的线程池(开启线程池)
futures = []
errcode = []
i=0
for code in data['code']:
try:
StockInfo().Get_guiyi(code)
print "===============%d data finish=================="%(i)
i+=1
except:
errcode.append(code)
print wait(futures)
print "=============== data finish=================="
def CleanDataOrgInfoTwo(self, data, indexs):
if data.loc[indexs].values[16] == 0:
print "this stock don't timeToMarket"
else:
# 最小的日期-1
StartTime = "2014-01-01"
EndTime = "2017-12-27"
# code = data.loc[indexs].values[1]
code = '000001'
# InfoDate = Mysql().ReadMySql('test3',code+'stock_org_basics')
InfoDate = Mysql().ReadMySqlTwo('test3', 'stock_org_basics_'+code)
WindInfo().Get_pct_chg_Info(code,
"high,low",
StartTime,
EndTime,
"unit=1;traderType=1;PriceAdj=F")
print "code"
print "===============%d data finish==================" % (indexs)
def Get_Pe_data(self, code, start, end):
#获取数据源 bob_jie 2017-11-21
#深圳的（有了）.XSHE
#上海的.XSHG
if code[0] == '0' or code[0] == '3': codenew = code + '.XSHE'
else: codenew = code + '.XSHG'
token = "7f47d59fa3d950a1de2026fbbb72220681968623a43123bc4e2d732c7edeec75"
URL = 'https://api.wmcloud.com/data/v1//api/market/getMktEquPEJL.json?field=&secID=' + codenew + '&startDate=' + start + '&endDate=' + end
Requtes = Requests().GetRequests(URL,
headers={
"Authorization":
"Bearer " + token,
"Accept-Encoding": "gzip, deflate"
})
#存取数据库 bob_jie 2017-11-21 正式代码
# Mysql().SaveMySqlThree(Requtes,'Stock_Basics_Info_All_New',code+'stock_basics',code)
#2017-11-27调试代码
Mysql().SaveMySqlThree(Requtes, 'test', code + 'stock_basics', code)
def CleanDataOrgInfoOne(self, data, indexs):
if data.loc[indexs].values[16] == 0:
print "this stock don't timeToMarket"
else:
# 最小的日期-1
StartTime = "2017-08-03"
EndTime = "2017-08-03"
# code = data.loc[indexs].values[1]
code = '000001'
# InfoDate = Mysql().ReadMySql('test3',code+'stock_org_basics')
InfoDate = Mysql().ReadMySqlTwo('test3', 'stock_org_basics_'+code)
if '2017-12-19' == str(InfoDate['max(a.index)'][0]):
WindInfo().Get_pct_chg_Info(code,
"",
StartTime,
EndTime,
"unit=1;traderType=1;PriceAdj=F")
else:
print "code"
print "===============%d data data cunzai finish==================" % (indexs)
print "code"
print "===============%d data finish==================" % (indexs)
def CleanDataChenjieOne(self,code,DATAtwo,DATAfour,DATAseven):
datastwo = pd.DataFrame()
datasthree = pd.DataFrame()
#正式代码
# DATAold = Mysql().ReadMySql('Stock_Basics_Info_All_New',code.split('.')[0]+'stock_basics')
#调试代码test
DATAold = Mysql().ReadMySql('test',code.split('.')[0]+'stock_basics')
DateTimeOne = DATAtwo['DateTime']
datastwo['date'] = DATAtwo['DateTime']
datastwo['BuyingVolume'] = DATAtwo[code]
datastwo['SellingVolume'] = DATAfour[code]
datasthree['date'] = DATAseven['DateTime']
datasthree['amount'] = DATAseven[code]
resultees = DATAold.merge(datasthree,left_on='date',right_on='date',how='outer')
results = resultees.merge(datastwo,left_on='date',right_on='date',how='outer')
results.drop('index',axis=1, inplace=True)
newresults = results.dropna(axis=0,thresh=3)
newresults['OrgVolume'] = newresults['BuyingVolume']+newresults['SellingVolume']
print newresults
#处理残缺数据 2017-11-29 bob_jie
#调试代码test
# Mysql().SaveAppendMySql(results.dropna(axis=0,thresh=3),'Stock_Basics_Info_All_New',code.split('.')[0]+'stock_basics')
# Mysql().SaveAppendMySql(newresults,'test',code.split('.')[0]+'stock_basics')
print "===============%s data finish=================="%(code)
"""拉取交易日信息进入数据库 2017-11-30 bob_jie"""
def Get_Organization_Info(self, code, data, start_data, end_data, expend):
if code[0] == '0' or code[0] == '3': codenew = code + '.SZ'
else: codenew = code + '.SH'
#启动wind客户端
Mysql().SaveMySqlFive(code, data, 'test3', code + 'stock_org_basics')
def Get_guiyi(self, code):
data = Mysql().ReadMySql('test4', 'stock_org_basics_' + code)
Mysql().SaveMySqlSeven('test4', 'stock_org_basics_' + code, data)
def Get_BanKui_data(self, dbname, tablename):
#读取数据库板块信息
data = Mysql().ReadMySql(dbname, tablename)
Mysql().SaveMySqlFour(data, dbname, 'stock_basics_plate')
"""拉取交易日信息进入数据库 2017-11-30 bob_jie"""
def CleanDataTradeCal(self):
StockInfo().Get_TradeCal_data("19841001","20171124")
print "=============== data finish=================="
"""拉取wind拉取机构信息信息进入数据库 2017-12-13 bob_jie """
def CleanDataOrgInfo(self,data,indexs):
if data.loc[indexs].values[16] == 0:print "this stock don't timeToMarket"
else:
#最小的日期-1
StartTime = "2014-01-01"
EndTime = "2015-08-02"
code = data.loc[indexs].values[1]
# InfoDate = Mysql().ReadMySql('test3',code+'stock_org_basics')
InfoDate = Mysql().ReadMySqlTwo('test3', 'stock_org_basics_'+code)
if EndTime == str(InfoDate['min(a.index)'][0]):
WindInfo().Get_pct_chg_Info(code,
"MFD_BUYAMT_D,MFD_SELLAMT_D,MFD_NETBUYAMT,MFD_BUYVOL_D,MFD_SELLVOL_D,MFD_NETBUYVOL,MFD_BUYAMT_A,MFD_SELLAMT_A,MFD_NETBUYAMT_A,MFD_BUYVOL_A,MFD_SELLVOL_A,MFD_NETBUYVOL_A,CLOSE,PCT_CHG,TOTAL_SHARES,FREE_FLOAT_SHARES,VOLUME, AMT,PE_TTM,VAL_PE_DEDUCTED_TTM",
StartTime,
EndTime,
"unit=1;traderType=1;PriceAdj=F")
if EndTime == str(InfoDate['min(a.index)'][0]):
WindInfo().Get_pct_chg_Info(code,
"MFD_BUYAMT_D,MFD_SELLAMT_D,MFD_NETBUYAMT,MFD_BUYVOL_D,MFD_SELLVOL_D,MFD_NETBUYVOL,MFD_BUYAMT_A,MFD_SELLAMT_A,MFD_NETBUYAMT_A,MFD_BUYVOL_A,MFD_SELLVOL_A,MFD_NETBUYVOL_A,CLOSE,PCT_CHG,TOTAL_SHARES,FREE_FLOAT_SHARES,VOLUME, AMT,PE_TTM,VAL_PE_DEDUCTED_TTM",
StartTime,
EndTime,
"unit=1;traderType=1;PriceAdj=F")
print "code"
print "===============%d data data cunzai finish==================" % (indexs)
print "code"
def Get_k_data(self, code, start, end):
data_h_data = ts.get_k_data(code, autype='qfq', start=start, end=end)
# #正式代码了这里
Mysql().SaveMySqlTWO(data_h_data, 'Stock_Basics_Info_All_New',
code + 'stock_basics')
def Get_hist_data(self, code):
data_h_data_all = ts.get_h_data(code)
Mysql().SaveMySqlTWO(data_h_data_all, 'Stock_Basics_Info',
code + 'stock_basics')
def Getstock_basics(self):
datateeeeeeee = ts.get_stock_basics()
# self.SaveCSV(datateeeeeeee, 'getstockbasics.csv')
#2017-11-27更新了这块代码
Mysql().SaveMySql(datateeeeeeee, 'Stock_Dasic_Data', 'stock_basics')