def data_alltogether_wande():
"""
:return: 所有股票roe，pe情况
"""
index = [
'总市值', 'PE', 'PE预测', '年化收益率_100weeks', '年化收益率_2years', '预测净利润', 'ROE',
'预测ROE', '评级', '预测目标价格', '收盘价', '成交量'
]
w.start()
with open('src/Source/wande.txt', 'r') as f:
l = f.readlines()
while '\n' in l:
l.remove('\n')
for i in l:
i = i.replace('\n', '')
i = i.replace('2020-04-19', datetime.now().strftime("%Y-%m-%d"))
x = eval(i).Data
y = pd.DataFrame()
for j in range(len(index)):
y[index[j]] = x[j + 2]
y.to_csv('Data/Stock_Data/' + x[0][0] + '.csv',
index=False,
encoding='utf-8-sig')
w.stop()
def import_data_from_wind(d_start, d_end, file_folder='C:\\dev\\src\\ktlib\\data3\\', freq='d', contracts=[]):
df = load_live_cont(d_start, d_end)
contList = [str(inst) for inst in df.index]
startDates = df.start_date
endDates = df.expiry
fields = 'open,close,high,low,volume,oi'
if len(contracts) !=0:
contList = contracts
#w.start()
for cont in contList:
if cont[1].isalpha(): key = cont[:2]
else: key = cont[:1]
ex = 'SH'
for exch in product_code.keys():
if key in product_code[exch]:
ex = exch
ticker = cont + '.' + ex
mth = int(cont[-2:])
if (key not in contMonth):
continue
if mth not in contMonth[key]:
continue
start_d = max(d_start, startDates.ix[cont])
end_d = min(d_end, endDates.ix[cont])
try:
if freq == 'm':
print "loading min data for ticker = %s" % ticker
raw_data = w.wsi(ticker,fields,start_d,end_d)
if len(raw_data.Data)>1:
outfile = file_folder + cont+'_min.csv'
output={'datetime':raw_data.Times,
'open':raw_data.Data[2],
'close':raw_data.Data[3],
'high':raw_data.Data[4],
'low':raw_data.Data[5],
'volume':raw_data.Data[6],
'openInterest':raw_data.Data[7]}
dump2csvfile(output,outfile)
else:
print "no min data obtained for ticker=%s" % ticker
else:
print "loading daily data for ticker = %s" % ticker
raw_data = w.wsd(ticker,fields,start_d,end_d)
if len(raw_data.Data)>1:
outfile = file_folder + cont+'_daily.csv'
output={'datetime':raw_data.Times,
'open':raw_data.Data[0],
'close':raw_data.Data[1],
'high':raw_data.Data[2],
'low':raw_data.Data[3],
'volume':raw_data.Data[4],
'openInterest':raw_data.Data[5]}
dump2csvfile(output,outfile)
else:
print "no daily data obtained for ticker=%s" % ticker
except ValueError:
pass
w.stop()
return True
def __del__(self):
try:
w.stop()
except AttributeError:
pass
except NameError:
pass
def post(self):
"""
json str:{"codes": "XT1522613.XT",
"fields": "fund_setupdate,fund_maturitydate,fund_mgrcomp,fund_existingyear,fund_fundmanager", "options": ""}
:return: 返回万得返回数据dict
"""
args = receive_wss_parser.parse_args()
logger.info('/wss/ args:%s', args)
# codes = args['codes']
# fields = args['fields']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.wss(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('wss(%s) ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
# 将 Data数据中所有 datetime date 类型的数据转换为 string
data_len = len(ret_data.Data)
logger.debug('ret_data.Data len:%d', data_len)
logger.debug('ret_data.Codes : %s', ret_data.Codes)
for n_data in range(data_len):
data = ret_data.Data[n_data]
data_len2 = len(data)
if data_len2 > 0:
if type(data[0]) in (datetime, date):
ret_data.Data[n_data] = [
format_2_date_str(dt) for dt in data
]
logger.info('%d column["%s"] date to str', n_data,
ret_data.Fields[n_data])
# print('ret_data.Data:\n', ret_data.Data)
# 组成 DataFrame
ret_df = pd.DataFrame(ret_data.Data,
index=ret_data.Fields,
columns=ret_data.Codes)
ret_dic = ret_df.to_dict()
# print('ret_dic:\n', ret_dic)
return ret_dic
def get_wind_data(inst_list, start_date, end_date, save_loc='C:\\dev\\data\\', freq='m'):
exch_map = {v: k for k, v in wind_exch_map.items()}
for instID in inst_list:
exch = misc.inst2exch(instID)
ex = exch_map[exch]
ticker = instID + '.' + ex
product = misc.inst2product(instID)
sdate = start_date
edate = end_date
stime = datetime.time(9, 0, 0)
etime = datetime.time(15, 0, 0)
if product in ['T', 'TF']:
stime = datetime.time(9, 15, 0)
etime = datetime.time(15, 15, 0)
elif product in misc.night_session_markets:
stime = datetime.time(21, 0, 0)
sdate = misc.day_shift(sdate, '-1b')
smin = datetime.datetime.combine(sdate, stime)
emin = datetime.datetime.combine(edate, etime)
fields = 'open,high,low,close,volume,oi'
try:
if freq == 'm':
outfile = save_loc + instID + '_min.csv'
if os.path.isfile(outfile):
continue
raw_data = w.wsi(ticker, fields, smin, emin)
if len(raw_data.Data) > 1:
output = {'datetime': raw_data.Times,
'open': raw_data.Data[0],
'high': raw_data.Data[1],
'low': raw_data.Data[2],
'close': raw_data.Data[3],
'volume': raw_data.Data[4],
'openInterest': raw_data.Data[5]}
dump2csvfile(output, outfile)
else:
print "no min data obtained for ticker=%s" % ticker
elif freq == 'd':
outfile = save_loc + instID + '_daily.csv'
if os.path.isfile(outfile):
continue
raw_data = w.wsd(ticker, fields, start_date, end_date)
if len(raw_data.Data) > 1:
output = {'datetime': raw_data.Times,
'open': raw_data.Data[0],
'high': raw_data.Data[1],
'low': raw_data.Data[2],
'close': raw_data.Data[3],
'volume': raw_data.Data[4],
'openInterest': raw_data.Data[5]}
dump2csvfile(output, outfile)
else:
print "no daily data obtained for ticker=%s" % ticker
except ValueError:
pass
w.stop()
return True
def get_wind_api():
""" 登录Wind API """
from WindPy import w
w.start() # 默认命令超时时间为120秒，如需设置超时时间可以加入waitTime参数
if w.isconnected(): # 判断WindPy是否已经登录成功
yield w
else:
_log.error("Wind API fail to be connected.")
w.stop()
def sectorClassify(self, outPutPath):
''' 行业分类'''
# 内置参数
cnMarketDict = {
u'industryType=1': u'申万行业一级',
u'industryType=2': u'申万行业二级',
u'industryType=3': u'申万行业三级'
}
mktFileNameDict = {
u'industryType=1': u'industryType=1',
u'industryType=2': u'industryType=2',
u'industryType=3': u'industryType=3'
}
periodType = 'M'
uniPeriodList = self.uniPeriodListDcit[periodType]
periodList = self.periodListDcit[periodType]
stockCodelist = copy.deepcopy(self.stockCodelist)
colName = copy.deepcopy(stockCodelist)
joinStockCode = u','.join(stockCodelist)
indexTypeList = [i for i in cnMarketDict]
w.start() #下载数据
for i in range(len(indexTypeList)):
dataSet = {}
tempDateRecord = []
for p in range(len(uniPeriodList)): # 周期迭代
period = uniPeriodList[p]
periodLocSer = pd.Series(periodList.loc[period])
tempSlice = periodList.index.get_loc(period)
if len(periodLocSer) >= 2:
stLoc = tempSlice.start
enLoc = tempSlice.stop - 1
else:
stLoc = tempSlice
enLoc = tempSlice
dateRange = [self.strDayList[stLoc], self.strDayList[enLoc]]
# 下载数据
downLoadFormula = 'w.wsd(joinStockCode, "industry_swcode", dateRange[-1], dateRange[-1],'
result = eval(downLoadFormula + 'indexTypeList[i])')
dataSet[period] = result
tempDateRecord.append(dateRange[-1])
print(period)
print(u'%s全时间段下载完毕' % indexTypeList[i]) # 保存数据至csv文件
strFilename = outPutPath + u'/' + cnMarketDict[indexTypeList[i]] + \
self.startDate + u'-' + str(self.lastTradeday) + u'.csv'
self.processedFilenameDict[mktFileNameDict[
indexTypeList[i]]] = strFilename
csvFile = open(strFilename, u'w', newline=u'')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName) # 第一行股票代码
for j in range(len(uniPeriodList)): # 不同周期
row = dataSet[uniPeriodList[j]].Data[0] # j 时间段所有股票代码数据
row.insert(0, tempDateRecord[j])
csvWriter.writerow(row)
csvFile.close()
w.stop() # 函数结束
def ifNewStock(self,
startTime=None,
endTime=None,
newStockDays=40,
tradeDaySource=u'Api'):
'''
对原始日线数据处理，生成是否是新股的boolean值
用到交易所交易日--通过接口获取
:return: 1 #新股(上市日期小于40或未上市)， 0 #非新股
'''
volume = self._read_csv(u'volume_stocks', startTime, endTime)
timeIndex = volume.index.astype(str)
length1 = len(timeIndex)
firstDate = timeIndex[0]
primaryDate = pd.date_range(end=firstDate, periods=1000,
freq=u'D').astype(str)[0]
if tradeDaySource == u'Wind':
from WindPy import w
w.start()
prevList = pd.Index(
w.tdays(primaryDate, firstDate, u"").Data[0][:-1]).astype(str)
w.stop()
elif tradeDaySource == u'Api':
prevList = pd.Index(
self.APIeg.getTradeDate(start=primaryDate, end=firstDate))
else: # 从本地读取交易所交易日
prevList = self._read_csv(u'tradeDay_stocks').index.astype(str)
tradedayList = prevList.union(timeIndex) # 整个交易日Index
newStockDF = copy.deepcopy(volume)
IPODf = self._read_csv(u'ipo_date_stocks')
IPOSeries = IPODf.iloc[0]
stockName = IPOSeries.index
firstDateLoc = tradedayList.get_loc(firstDate)
newTimeList = tradedayList[(firstDateLoc - newStockDays):]
length2 = len(newTimeList)
prevFirstDate = newTimeList[0]
for i in range(len(stockName)):
stock = stockName[i]
ipoDate = IPOSeries[i]
tempSeries = copy.deepcopy(
pd.Series(np.zeros(length1), index=timeIndex))
if ipoDate > prevFirstDate:
newTimeListLoc = newTimeList.get_loc(ipoDate)
unNewDateLoc = min(
len(newTimeList) - 1, newTimeListLoc + newStockDays)
if unNewDateLoc <= length2 - 1 and unNewDateLoc >= newStockDays:
tempSeries[:newTimeListLoc] = 1
newStockDF[stock] = pd.Series(tempSeries)
newStockDF.to_csv(DataProcessing.CSV_PATH +
u'is_new_stock_%s_stocks.csv' % newStockDays)
def start(self):
'''the main function to perform all operations'''
print('开始程序')
w.start(60)
#if cannot connect to wind, just stop
if not w.isconnected():
print('无法连接万德')
print('程序结束')
w.stop()
return
#otherwise
print('万德已连接')
print('开始处理数据')
#期货行情
today = datetime.now()
print('今天是', today.strftime('%Y-%m-%d'))
loop = asyncio.get_event_loop()
exchanges = self.config['futures']['exchanges']
for eid in dict.keys(exchanges):
instruments = exchanges[eid]
instrument = None
for instrument in instruments:
print('处理品种 ' + instrument + ' 交易所 ' + eid)
items = []
i = 0
for i in range(today.month, today.month + 13, 1):
iid = instrument
if i > 12:
iid = iid + self.fix_instrument(str(today.year + 1)[2:] + \
pad_month(i % 12), eid)
else:
iid = iid + self.fix_instrument(str(today.year)[2:] + \
pad_month(i), eid)
iid = iid + '.' + eid
items.append(iid)
tasks = []
for item in items:
tasks.append(asyncio.ensure_future(self.process_item(w, item)))
try:
loop.run_until_complete(asyncio.wait(tasks))
except:
print('循环产生异常：处理品种' + instrument + '期货价格数据')
finally:
print('完成品种 ' + instrument + ' 交易所 ' + eid)
loop.close()
print('程序结束')
w.stop()
def post(self):
"""
json str:{"begin_time": "2017-3-31", "end_time": "2017-3-31", "options": ""}
:return: 返回万得返回数据dict
"""
args = tdays_parser.parse_args()
logger.info('/tdays/ args:%s', args)
# begin_time = args['begin_time']
# end_time = args['end_time']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.tdays(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('tdays(%s) ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
# 将 Data数据中所有 datetime date 类型的数据转换为 string
if len(ret_data.Data) > 0 and len(ret_data.Data[0]) > 0:
# date_str = format_datetime_to_str(ret_data.Data[0][0])
# ret_df = pd.DataFrame({'date': [format_datetime_to_str(d) for d in ret_data.Data[0]]})
# ret_df.index = [str(idx) for idx in ret_df.index]
# ret_dic = {'date': [format_datetime_to_str(d) for d in ret_data.Data[0]]}
ret_dic = [format_2_date_str(d) for d in ret_data.Data[0]]
else:
logger.warning('tdays(%s) No value return' % args)
ret_dic = []
# ret_dic = ret_df.to_dict()
# print('tdays:\n', ret_dic)
return ret_dic
def main():
w.start()
windstock = WindStock()
start_date = '20130520'
end_date = '20170520'
symbols = windstock.getAStockCodesWind(end_date) #symbols类型为list?
fm = pd.DataFrame(symbols)
fm = fm.T
print fm
windstock.AStockHisData(symbols, start_date, end_date)
#save the data into csv files
w.stop()
def post(self):
"""
json str:{"offset": "1", "begin_time": "2017-3-31", "options": ""}
:return: 返回万得返回数据dict
"""
args = tdays_offset_parser.parse_args()
logger.info('/tdaysoffset/ args:%s', args)
# offset = int(args['offset'])
# begin_time = args['begin_time']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.tdaysoffset(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('tdaysoffset("%s") ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
# 将 Data数据中所有 datetime date 类型的数据转换为 string
if len(ret_data.Data) > 0 and len(ret_data.Data[0]) > 0:
date_str = format_2_date_str(ret_data.Data[0][0])
else:
logger.warning('tdaysoffset(%s) No value return' % args)
date_str = ''
ret_dic = {'Date': date_str}
# print('offset:\n', ret_dic)
return ret_dic
def main():
w.start()
windstock = WindStock()
start_date = '20160520'
end_date = '20160729'
#获取A股所有的股票代码
symbols = windstock.getAStockCodesWind(end_date)
#print "hello"
fm = pd.DataFrame(symbols)
fm = fm.T
#print fm
windstock.AStockHisData(symbols, start_date, end_date)
#save the data into csv files
w.stop()
def downloadSimpleData(self, simpleDataList):
'''
下载简单数据，比如ipo_date
:param simpleDataList:
:return:
'''
cnSimDict = {'ipo_date': u'ipo_date'}
simFileNameDict = {'ipo_date': u'ipo_date'}
# 设置参数
startDate = copy.deepcopy(self.startDate)
lastTradeday = copy.deepcopy(self.lastTradeday)
stockCodelist = copy.deepcopy(self.stockCodelist)
joinStockCode = u','.join(stockCodelist)
colName = stockCodelist
colName.insert(0, 'Date')
# 下载数据
w.start()
for i in range(len(simpleDataList)):
tempDf = pd.DataFrame()
strFilename = os.getcwd() + u'/DownloadedData/OriginalData/' + cnSimDict[simpleDataList[i]] + \
startDate + u'-' + str(lastTradeday) + u'.csv'
self.processedFilenameDict[simFileNameDict[
simpleDataList[i]]] = strFilename
csvFile = open(strFilename, 'w', newline='')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName)
if simpleDataList[i] in ['ipo_date']: # ipo只需要最新时间的数据
result = w.wsd(joinStockCode, simpleDataList[i],
str(lastTradeday), str(lastTradeday), "")
row = [str(i.date()) for i in result.Data[0]]
row.insert(0, str(lastTradeday))
csvWriter.writerow(row)
tempDf = tempDf.append([row])
print(cnSimDict[simpleDataList[i]] + u'全时段数据下载完成')
w.stop()
print(u'简单数据下载完成')
def import_data_from_wind(d_start,
d_end,
file_folder='C:\\dev\\src\\ktlib\\data3\\',
freq='d',
contracts=[]):
df = load_live_cont(d_start, d_end)
contList = [str(inst) for inst in df.index]
startDates = df.start_date
endDates = df.expiry
fields = 'open,close,high,low,volume,oi'
if len(contracts) != 0:
contList = contracts
#w.start()
for cont in contList:
if cont[1].isalpha(): key = cont[:2]
else: key = cont[:1]
ex = 'SH'
for exch in product_code.keys():
if key in product_code[exch]:
ex = exch
ticker = cont + '.' + ex
mth = int(cont[-2:])
if (key not in contMonth):
continue
if mth not in contMonth[key]:
continue
start_d = max(d_start, startDates.ix[cont])
end_d = min(d_end, endDates.ix[cont])
try:
if freq == 'm':
print "loading min data for ticker = %s" % ticker
raw_data = w.wsi(ticker, fields, start_d, end_d)
if len(raw_data.Data) > 1:
outfile = file_folder + cont + '_min.csv'
output = {
'datetime': raw_data.Times,
'open': raw_data.Data[2],
'close': raw_data.Data[3],
'high': raw_data.Data[4],
'low': raw_data.Data[5],
'volume': raw_data.Data[6],
'openInterest': raw_data.Data[7]
}
dump2csvfile(output, outfile)
else:
print "no min data obtained for ticker=%s" % ticker
else:
print "loading daily data for ticker = %s" % ticker
raw_data = w.wsd(ticker, fields, start_d, end_d)
if len(raw_data.Data) > 1:
outfile = file_folder + cont + '_daily.csv'
output = {
'datetime': raw_data.Times,
'open': raw_data.Data[0],
'close': raw_data.Data[1],
'high': raw_data.Data[2],
'low': raw_data.Data[3],
'volume': raw_data.Data[4],
'openInterest': raw_data.Data[5]
}
dump2csvfile(output, outfile)
else:
print "no daily data obtained for ticker=%s" % ticker
except ValueError:
pass
w.stop()
return True
def download_fundData(self, fundCodeList=[]):
# 内置参数
startDate = copy.deepcopy(self.startDate)
endDate = copy.deepcopy(self.endDate)
lastTradeday = copy.deepcopy(self.lastTradeday)
cnIndexDict = {}
for i in range(len(fundCodeList)):
if fundCodeList[i][-3] == '.':
cnIndexDict[fundCodeList[i]] = fundCodeList[i][:-3]
# 设置指数数据
w.start()
# 指数数据下载
fieldName = ["NAV_acc", "NAV_adj", "nav", u'fund_investobject']
joinFieldName = ','.join(fieldName)
fieldCNName = ['单位净值', '累计单位净值', '复权单位净值', u'投资目标']
colName2 = copy.deepcopy(fieldCNName)
colName2.insert(0, u'Date')
for i in range(len(fundCodeList)):
dataSet = {}
# 按年下载数据
yearNum = int(startDate.split(u'-')[0])
yearTradedaySet = {}
tempStartDate = startDate
latestYearNum, lastEndDate = int(endDate.split(u'-')[0]), int(
endDate.split(u'-')[2])
while yearNum <= latestYearNum:
if yearNum == int(latestYearNum):
tempEndDate = endDate
else:
tempEndDate = str(yearNum) + u'-12-31'
yearTradedayList = w.tdays(tempStartDate, tempEndDate,
u"").Data[0]
yearFirstTradeday = yearTradedayList[0].date()
yearLastTradeday = yearTradedayList[-1].date()
# 下载数据
indexResult = w.wsd(fundCodeList[i], joinFieldName,
str(yearFirstTradeday),
str(yearLastTradeday), u"PriceAdj=F")
dataSet[str(yearNum)] = indexResult
yearTradedaySet[str(yearNum)] = yearTradedayList
yearNum += int(1)
tempStartDate = str(yearNum) + u'-01-01'
print(cnIndexDict[fundCodeList[i]] + u'全时间段下载完毕')
# 创建文件
tempDf = pd.DataFrame()
strFilename = os.getcwd() + u'/DownloadedData/OriginalData/' + cnIndexDict[fundCodeList[i]] + \
startDate + r'-' + str(lastTradeday) + u'.csv'
# self.processedFilenameDict[indexFileNameDict[indexCodeList[i]]] = strFilename
csvFile = open(strFilename, u'w', newline=u'')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName2) # 第一行股票代码
for key in yearTradedaySet: # 不同年
csvTradedayList = yearTradedaySet[key]
for j in range(len(csvTradedayList)):
row = [i[j] for i in dataSet[key].Data] # j 时间段所有股票代码数据
row.insert(0, csvTradedayList[j].date())
csvWriter.writerow(row)
tempDf = tempDf.append([row])
csvFile.close()
w.stop()
print(u'所有指数数据下载完毕')
def downloadIfSetor(self, sectorid=[]):
# 内置参数
startDate = copy.deepcopy(self.startDate)
lastTradeday = copy.deepcopy(self.lastTradeday)
stockCodelist = copy.deepcopy(self.stockCodelist)
sectorCNDict = {
u'1000006526000000': u'ST股判断条件',
u'1000000090000000': u'沪深300判断条件',
u'1000008491000000': u'中证500判断条件',
u'1000000087000000': u'上证50判断条件',
u'1000012163000000': u'中证1000判断条件'
}
sectorFileDict = {
u'1000006526000000': u'is_be_st',
u'1000000090000000': u'isHS300',
u'1000008491000000': u'isZZ500',
u'1000000087000000': u'isSZ50',
u'1000012163000000': u'iszz1000'
}
# 设置1，0 判断条件
w.start()
if sectorid == []:
sectorid = [
u'1000006526000000', u'1000000090000000', u'1000008491000000',
u'1000000087000000'
]
colName = copy.deepcopy(stockCodelist)
colName.insert(0, u'Date')
# xx板块股票代码下载
for q in range(len(sectorid)):
periodType = 'M'
dataSet = {}
uniPeriodList = self.uniPeriodListDcit[periodType]
periodList = self.periodListDcit[periodType]
tempDateRecord = []
for p in range(len(uniPeriodList)): # 周期迭代
period = uniPeriodList[p]
periodLocSer = pd.Series(periodList.loc[period])
tempSlice = periodList.index.get_loc(period)
if len(periodLocSer) >= 2:
stLoc = tempSlice.start
enLoc = tempSlice.stop - 1
else:
stLoc = tempSlice
enLoc = tempSlice
dateRange = [self.strDayList[stLoc], self.strDayList[enLoc]]
# 下载数据
tempSTDataSet = w.wset(
u"sectorconstituent", u"date=" + dateRange[-1] +
u";sectorid=" + sectorid[q] + u";field=date,wind_code")
stCode = tempSTDataSet.Data[1]
tempStockCode = copy.deepcopy(stockCodelist)
for j in range(len(tempStockCode)):
if stCode.count(tempStockCode[j]) == int(1):
tempStockCode[j] = 1
else:
tempStockCode[j] = 0
dataSet[period] = tempStockCode
tempDateRecord.append(dateRange[-1])
print(sectorCNDict[sectorid[q]] + u'全时间段下载完毕') # 保存数据至csv文件
strFilename = os.getcwd() + u'/DownloadedData/OriginalData/' + sectorCNDict[sectorid[q]] + \
startDate + u'-' + str(lastTradeday) + u'.csv'
self.processedFilenameDict[sectorFileDict[
sectorid[q]]] = strFilename
# 创建文件
csvFile = open(strFilename, u'w', newline=u'')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName) # 第一行股票代码
for j in range(len(uniPeriodList)): # 不同周期
row = dataSet[uniPeriodList[j]] # j 时间段所有股票代码数据
row.insert(0, tempDateRecord[j])
csvWriter.writerow(row)
csvFile.close()
w.stop()
print(u'所有板块数据下载完毕')
'''
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from WindPy import w
import pymysql
w.start();
def saveTradeDate(tradeDates):
if tradeDates.ErrorCode!=0:
print('error code:'+str(tradeDates.ErrorCode)+'\n');
return();
print('Data length:',len(tradeDates.Data[0]))
conn=pymysql.connect(host='192.168.10.222',port='3306',user='******',passwd='cc2718281828',db='AMGMH',charset='utf8')
cur=conn.cursor()
for i in range(0,len(tradeDates.Data[0])):
strTemp=''
#if len(tradeDates.Times)>1:
# strTemp=str(tradeDates.Times[i])+' '
for k in range(0, len(tradeDates.Fields)):
strTemp=strTemp+str(tradeDates.Data[k][i])+' '
sta=cur.execute('insert into TDays values('+'\'SHE\','+str(tradeDates.Data[k][i])+')')
print(sta)
print(strTemp)
tradeDate = w.tdays('2016-03-25', '2016-12-31', 'TradingCalendar=SZSE')
saveTradeDate(tradeDate)
w.stop();
def main():
book = xw.Book.caller()
"""爬取农业部数据"""
today = datetime.date.today()
#获取当月第一天
firstmonthday = datetime.datetime(today.year, today.month, 1)
#获取当年第一天
firstday = datetime.datetime(today.year, 1, 1)
oneday = datetime.timedelta(days=1)
all = {} #存放所有爬取的网页链接，key为日期，value为链接
url = 'http://www.scs.moa.gov.cn/scxxfb/'
#爬取主页
response = requests.get(url)
content = response.content
page = etree.HTML(content)
data = page.find('.//div[@class="sj_e_tonzhi_list"]')
for i in data:
infos = i.findall('.//li')
for info in infos:
rrr = info.find('.//a')
link = url + str(rrr.get('href'))
date = re.findall(r'.\w+.t(\d+)\w+', link)
all[date[0]] = str(link)
for i in range(1, 13):
url = 'http://www.scs.moa.gov.cn/scxxfb/index_' + str(i) + '.htm'
response = requests.get(url)
content = response.content
page = etree.HTML(content)
data = page.find('.//div[@class="sj_e_tonzhi_list"]')
for i in data:
infos = i.findall('.//li')
for info in infos:
rrr = info.find('.//a')
link = 'http://www.scs.moa.gov.cn/scxxfb/' + str(
rrr.get('href'))
date = re.findall(r'.\w+.t(\d+)\w+', link)
all[date[0]] = str(link)
#print(all)
#爬取目标页、正则提取猪肉价格
def price_get(link):
response = requests.get(link)
content = response.content
page = etree.HTML(content)
info = page.find('.//div[@class="TRS_Editor"]')
text = info.find('.//p').text
price = re.findall(r'猪肉\D+(\d+.\d+)元', text)
return price
price1 = {} #存放猪肉价格，key为日期，value为价格
#今天的价格，若未更新则为前一天价格
while today.strftime('%Y%m%d') not in all.keys():
today -= oneday
else:
d_p_price = price_get(str(all[today.strftime('%Y%m%d')]))
price1[today.strftime('%Y%m%d')] = d_p_price
#本月初价格，更新时间为本月第一个工作日
while firstmonthday.strftime('%Y%m%d') not in all.keys():
firstmonthday += oneday
else:
m_p_price = price_get(str(all[firstmonthday.strftime('%Y%m%d')]))
price1[firstmonthday.strftime('%Y%m%d')] = m_p_price
#本年初价格，更新时间为本年第一个工作日
while firstday.strftime('%Y%m%d') not in all.keys():
firstday += oneday
else:
y_p_price = price_get(str(all[firstday.strftime('%Y%m%d')]))
price1[firstday.strftime('%Y%m%d')] = y_p_price
#对应价格的列表
#l=[price1[today.strftime('%Y%m%d')],price1[firstmonthday.strftime('%Y%m%d')],price1[firstday.strftime('%Y%m%d')]]
print(price1)
'''
最终结果是price1是一个字典
pric1e[today.strftime('%Y%m%d')]是今日价格
price1[firstmonthday.strftime('%Y%m%d')]是本月初
price1[firstday.strftime('%Y%m%d')]是本年初
'''
"""爬取二元能繁母猪数据"""
#百度云账号
APP_ID = '#####'
API_KEY = '########'
SECRECT_KEY = '########'
client = AipOcr(APP_ID, API_KEY, SECRECT_KEY)
#爬取主页，获取目标网页链接
url = 'http://sousuo.gov.cn/s.htm?q=%E4%BA%8C%E5%85%83%E6%AF%8D%E7%8C%AA%E9%94%80%E5%94%AE%E4%BB%B7%E6%A0%BC&t=govall&timetype=timeqb&mintime=&maxtime=&sort=pubtime&sortType=1&nocorrect='
response = requests.get(url)
content = response.content
page = etree.HTML(content)
table = page.find('.//h3[@class="res-title"]')
channels = table.find('.//a')
link = channels.get('href')
#print(link)
#爬取最新公告的标题
html = requests.get(link)
html.encoding = 'utf-8'
text = html.text
page1 = etree.HTML(text)
info = page1.find('.//div[@class="article oneColumn pub_border"]')
t = info.find('.//h1')
title = t.text
#print(title)
#从公告标题中提取更新数据对应的日期
datestr = title[len(title) - 14:len(title) - 9]
date = '2020年' + datestr
date1 = datetime.datetime.strptime(date, '%Y年%m月%d日')
#print(date1)
#爬取公告中的图片
content1 = page1.find('.//div[@class="pages_content"]')
channels1 = content1.find('.//img')
link_img = channels1.get('src')
links = str(link)
pic_urls = links[:len(links) - 19] + link_img
#调用百度api对图片进行文本识别，从中提取价格内容
prices = client.basicGeneralUrl(pic_urls)
r = prices['words_result']
info = r[5]
price = info['words']
#print(price)
#更新每周二元母猪价格
pork_price = {}
week = date1.strftime("%W")
pork_price[week + '周'] = price #存放每周二元母猪价格，key为周数，对应价格
print(pork_price)
"""抓取wind数据 写入excel"""
#链接到wind数据库
w.start()
w.isconnected()
#统计仔猪数据
##download仔猪数据
pig_baby_codes = ['############'] ###仔猪代码已打码
pig_baby = w.edb(pig_baby_codes,
datetime.date.today() + datetime.timedelta(days=-5),
datetime.date.today(),
usedf=True,
ShowBlank=0)
pig_baby = pig_baby[1]
pig_baby.columns = ['###########'] ###仔猪地区标签已打码
##分地区统计仔猪数据
pig_baby_mean = pd.DataFrame([])
pig_baby_mean_names = ['##########'] ###仔猪分地区统计的地区标签已打码
for i in range(1, 13, 2):
pig_baby_mean[pig_baby_mean_names[int(
(i - 1) /
2)]] = (pig_baby.iloc[:, i - 1] + pig_baby.iloc[:, i]) / 2
print(pig_baby_mean)
#生猪
##download生猪数据
pig_codes = ["###############"] ###生猪代码已打码
pig = w.edb(pig_codes,
datetime.date.today() + datetime.timedelta(days=-4),
datetime.date.today(),
usedf=True,
ShowBlank=0)
pig = pig[1]
pig.columns = ["###############"] ###生猪地区标签已打码
##分地区统计仔猪数据
pig_mean = pd.DataFrame(np.zeros((4, 5)))
pig_mean_names = ["###########"] ###生猪分地区统计的地区标签已打码
pig_mean.columns = pig_mean_names
print(pig_mean)
pig_mean.index = pig.index[1:]
for name in pig_mean_names:
i = 0
for n in list(pig.columns):
if name in n:
pig_mean[name] = pig_mean[name] + pig[n]
i += 1
pig_mean[name] = pig_mean[name] / i
print(pig_baby_mean)
#统计玉米数据
##donload玉米价格
corn_codes = ['S5005793']
corn = w.edb(corn_codes,
datetime.date.today() + datetime.timedelta(days=-5),
datetime.date.today(),
usedf=True,
ShowBlank=0)
corn = corn[1]
corn.columns = ['现货价:玉米:平均价']
corn = corn.T
print(corn)
#关闭Wind接口
w.stop()
#仔猪、生猪、猪肉、玉米价格汇总
pig_baby_mean = pig_baby_mean.T
pig_mean = pig_mean.T
pig_baby_data = list(pig_baby_mean[pig_baby_mean.columns[-1]])
pig_baby_data.append(np.mean(pig_baby_data))
pig_data = list(pig_mean[pig_mean.columns[-1]])
pig_data.append(np.mean(pig_data))
corn_data = list(corn[corn.columns[-1]])
pig_baby_data.extend(pig_data)
pig_baby_data.extend(corn_data)
pig_baby_data.append(float(price1[today.strftime('%Y%m%d')][0]))
alldata = pig_baby_data
print(alldata)
#最近5日日期的一个list——days是datetime格式列表，days1是字符格式列表
days = [
datetime.datetime.today() + datetime.timedelta(days=-i)
for i in range(5)
]
days1 = [days[i].strftime('%Y-%m-%d') for i in range(5)]
days.reverse()
days1.reverse()
print(days)
#最近五周的一个list——week_nows
week_list = {}
today = datetime.date.today()
weeks = today.strftime("%W")
week_n = int(weeks)
week_list[week_n] = week_n
l = [week_list[week_n] - i for i in range(5)]
for i in range(5):
l[i] = str(l[i]) + '周'
l.reverse()
print(l)
week_nows = l
#链接到目标表格
sht = book.sheets[0]
#判断二元能繁母猪年度数据、月度数据是否要更新
firstday_week = datetime.datetime(datetime.date.today().year,
datetime.date.today().month,
1).strftime("%W") + '周'
if week_nows[-1] == '1周':
sht.range('Q8').value = float(price)
if week_nows[-1] == firstday_week:
sht.range('P8').value = float(price)
#判断仔猪、生猪、猪肉、玉米年度数据、月度数据是否要更新
if days1[-1][6:] == '01-01':
sht.range('Q11:Q25').options(transpose=True).value = alldata
if days1[-1][9:] == '01':
sht.range('P11:P25').options(transpose=True).value = alldata
#更新主体数据（若今天数据已更新则不再更）
##二元能繁母猪
if sht.range('K7').value == week_nows[-1]:
pass
else:
sht.range('G8:J8').value = sht.range('H8:K8').value
sht.range('K8').value = float(price)
##仔猪、生猪、猪肉、玉米
if sht.range('K9').value.date() == days[-1].date():
pass
else:
sht.range('G7:K7').value = week_nows
sht.range('G9:K9').value = days1
sht.range('G11:J25').value = sht.range('H11:K25').value
sht.range('K11:K25').options(transpose=True).value = alldata
def logout():
if WDServer.isconnected():
logout = WDServer.stop()
print(colored("WDServer is disconnected successfully!", "green"))
else:
print(colored("WDServer is already disconnected!", "green"))
def get_min_data(filename):
file_folder = 'C:\\dev\\src\\ktlib\\data3\\'
reader = csv.reader(file(filename, 'rb'))
contList = []
fields = 'open,close,high,low,volume,oi'
contStart = []
contEnd = []
for line in reader:
contList.append(line[0])
contStart.append(line[1])
contEnd.append(line[2])
#w.start()
for cont, startD, endD in zip(contList, contStart, contEnd):
if cont[1].isalpha(): key = cont[:2]
else: key = cont[:1]
ex = 'SH'
for exch in product_code.keys():
if key in product_code[exch]:
ex = exch
ticker = cont + '.' + ex
startmin = ""
endMin = ""
if ex == 'CFE':
startMin = " 09:14:00"
endMin = " 15:15:00"
else:
startMin = " 08:59:00"
endMin = " 15:00:00"
if ex == 'CFE':
startMin = " 09:14:00"
endMin = " 15:15:00"
else:
startMin = " 08:59:00"
endMin = " 15:00:00"
try:
sdate = datetime.datetime.strptime(startD, "%Y/%m/%d")
edate = datetime.datetime.strptime(endD, "%Y/%m/%d")
sMin = datetime.datetime.strptime(startD + startMin,
"%Y/%m/%d %H:%M:%S")
eMin = datetime.datetime.strptime(endD + endMin,
"%Y/%m/%d %H:%M:%S")
print ticker, sMin, eMin
if edate > datetime.date(2009, 1, 1):
raw_data = w.wsi(ticker, fields, sMin, eMin)
if len(raw_data.Data) > 1:
outfile = file_folder + cont + '_min.csv'
output = {
'datetime': raw_data.Times,
'open': raw_data.Data[0],
'close': raw_data.Data[1],
'high': raw_data.Data[2],
'low': raw_data.Data[3],
'volume': raw_data.Data[4],
'openInterest': raw_data.Data[5]
}
dump2csvfile(output, outfile)
else:
print "no min data obtained for ticker=%s" % ticker
raw_data = w.wsd(ticker, fields, sdate, edate)
if len(raw_data.Data) > 1:
outfile = file_folder + cont + '_daily.csv'
output = {
'datetime': raw_data.Times,
'open': raw_data.Data[0],
'close': raw_data.Data[1],
'high': raw_data.Data[2],
'low': raw_data.Data[3],
'volume': raw_data.Data[4],
'openInterest': raw_data.Data[5]
}
dump2csvfile(output, outfile)
else:
print "no daily data obtained for ticker=%s" % ticker
except ValueError:
pass
w.stop()
return True
def AShareEODDerivativeIndicator(timestr):
w.start()
w.isconnected()
timey = timestr[0:4]
timem = timestr[4:6]
timed = timestr[6:8]
date = datetime.datetime(int(timey), int(timem), int(timed))
windds1 = w.wset('SectorConstituent', 'date=' + timestr + ';sector=全部A股')
datac = windds1.Data[1]
#datat=['000001.SZ','000002.SZ']
date52b = date + datetime.timedelta(-364)
startdate = str(date52b)
windds6 = w.wss(datac, "latelyrd_bt", "tradeDate=" + timestr)
lt = list(set(windds6.Data[0]))
latelyrd = [[] for x in range(len(lt))]
lty = list(set(windds6.Data[0]))
for a in range(len(lt)):
if lty[a].month != 12:
lty[a] = datetime.datetime(lt[a].year - 1, 12, 31)
for b in range(len(windds6.Codes)):
for a in range(len(lt)):
if windds6.Data[0][b] == lt[a]:
latelyrd[a].append(windds6.Codes[b])
break
windds4 = w.wss(
latelyrd[0],
"pe,pb,pe_ttm,pcf_ocf,pcf_ocf_ttm,pcf_ncf,pcf_ncf_ttm,ps,ps_ttm,netprofit_ttm,np_belongto_parcomsh,net_cash_flows_oper_act,oper_rev,net_incr_cash_cash_equ_dm",
"tradeDate=" + timestr + ";ruleType=10;unit=1;rptDate=" + str(lty[0]) +
";rptType=1;currencyType=")
if (len(lt) > 1):
for a in range(1, len(lt)):
winddst = w.wss(
latelyrd[a],
"pe,pb,pe_ttm,pcf_ocf,pcf_ocf_ttm,pcf_ncf,pcf_ncf_ttm,ps,ps_ttm,netprofit_ttm,np_belongto_parcomsh,net_cash_flows_oper_act,oper_rev,net_incr_cash_cash_equ_dm",
"tradeDate=" + timestr + ";ruleType=10;unit=1;rptDate=" +
str(lty[a]) + ";rptType=1;currencyType=")
windds4.Codes += winddst.Codes
for b in range(len(winddst.Data)):
windds4.Data[b] += winddst.Data[b]
del winddst
lt = list(set(windds6.Data[0]))
latelyrd = [[] for x in range(len(lt))]
for b in range(len(windds6.Codes)):
for a in range(len(lt)):
if windds6.Data[0][b] == lt[a]:
latelyrd[a].append(windds6.Codes[b])
break
windds5 = w.wss(
latelyrd[0], "wgsd_bps_new,total_shares,operatecashflow_ttm2,or_ttm2",
"currencyType=;unit=1;tradeDate=" + timestr + ";rptDate=" + str(lt[0]))
if (len(lt) > 1):
for a in range(1, len(lt)):
winddst = w.wss(
latelyrd[a],
"wgsd_bps_new,total_shares,operatecashflow_ttm2,or_ttm2",
"currencyType=;unit=1;tradeDate=" + timestr + ";rptDate=" +
str(lt[a]))
windds5.Codes += winddst.Codes
for b in range(len(winddst.Data)):
windds5.Data[b] += winddst.Data[b]
del winddst
windds2 = w.wss(
windds5.Codes,
'ev,mkt_cap_float,high_per,low_per,turn,free_turn_n,total_shares,float_a_shares,free_float_shares',
"unit=1;tradeDate=" + timestr + ";currencyType=;startDate=" +
startdate + ";endDate=" + timestr + ";priceAdj=U")
windds3 = w.wss(
windds5.Codes,
"high_per,low_per,maxupordown,high_date_per,low_date_per",
"unit=1;tradeDate=" + timestr + ";currencyType=;startDate=" +
startdate + ";endDate=" + timestr + ";priceAdj=B")
windds7 = w.wss(
latelyrd[0], "wgsd_cce",
"unit=1;rptDate=" + str(lt[0]) + ";rptType=1;currencyType=")
if (len(lt) > 1):
for a in range(1, len(lt)):
winddst = w.wss(
latelyrd[a], "wgsd_cce",
"unit=1;rptDate=" + str(lt[a]) + ";rptType=1;currencyType=")
windds7.Codes += winddst.Codes
for b in range(len(winddst.Data)):
windds7.Data[b] += winddst.Data[b]
del winddst
windds8 = w.wss(
latelyrd[0], "wgsd_cce", "unit=1;rptDate=" +
str(datetime.datetime(lt[0].year - 1, lt[0].month, lt[0].day)) +
";rptType=1;currencyType=")
if (len(lt) > 1):
for a in range(1, len(lt)):
winddst = w.wss(
latelyrd[a], "wgsd_cce", "unit=1;rptDate=" +
str(datetime.datetime(lt[a].year - 1, lt[a].month,
lt[a].day)) + ";rptType=1;currencyType=")
windds8.Codes += winddst.Codes
for b in range(len(winddst.Data)):
windds8.Data[b] += winddst.Data[b]
del winddst
dc1={'Ticker':windds5.Codes,'Date':[timestr]*len(datac),'MarketValue':windds2.Data[0], \
'MarketValueFree':windds2.Data[1],'High52w':windds2.Data[2],'Low52w':windds2.Data[3] \
,'TurnOver':windds2.Data[4],'TurnOverFree':windds2.Data[5],'TotalShares':windds2.Data[6],
'TotalSharesLT':windds2.Data[7],'TotalSharesFree':windds2.Data[8]}
lhs = []
for i in range(len(datac)):
lhs.append(
int(date == windds3.Data[3][i] or date == windds3.Data[4][i]))
dc2 = {
'AdjHigh52w': windds3.Data[0],
'AdjLow52w': windds3.Data[1],
'UpDownLimitStatus': windds3.Data[2],
'LowestHighestStatus': lhs
}
dc3={'PE':windds4.Data[0],'PB':windds4.Data[1],'PE_ttm':windds4.Data[2],'PCF_ocf':windds4.Data[3] \
,'PCF_ocfttm':windds4.Data[4],'PCF_ncf':windds4.Data[5],'PCF_ncfttm':windds4.Data[6], \
'PS':windds4.Data[7],'PS_ttm':windds4.Data[8],'NetProfit_parent_comp_ttm':windds4.Data[9], \
'NetProfit_parent_comp_lyr':windds4.Data[10],'NetCashFlow_oper_act_lyr':windds4.Data[11], \
'OperRev_lyr':windds4.Data[12],'NetIncrCash_cash_equ_lyr':windds4.Data[13]}
lhs = []
for i in range(len(datac)):
lhs.append(windds5.Data[0][i] * windds5.Data[1][i])
dc4 = {
'NetAsset': lhs,
'NetCashFlow_oper_act_ttm': windds5.Data[2],
'OperRev_ttm': windds5.Data[3]
}
lhs = []
for i in range(len(datac)):
if windds7.Data[0][i] == None or windds8.Data[0][i] == None:
lhs.append(None)
else:
lhs.append(windds7.Data[0][i] - windds8.Data[0][i])
dc5 = {'NetIncrCash_cash_equ_ttm': lhs}
df1 = pd.DataFrame(dc1)
df2 = pd.DataFrame(dc2)
df3 = pd.DataFrame(dc3)
df4 = pd.DataFrame(dc4)
df5 = pd.DataFrame(dc5)
df = pd.concat([df1, df2], axis=1)
df = pd.concat([df, df3], axis=1)
df = pd.concat([df, df4], axis=1)
df = pd.concat([df, df5], axis=1)
df = df[[
'Ticker', 'Date', 'MarketValue', 'MarketValueFree', 'High52w',
'Low52w', 'TurnOver', 'TurnOverFree', 'TotalShares', 'TotalSharesLT',
'TotalSharesFree', 'AdjHigh52w', 'AdjLow52w', 'UpDownLimitStatus',
'LowestHighestStatus', 'PE', 'PB', 'PE_ttm', 'PCF_ocf', 'PCF_ocfttm',
'PCF_ncf', 'PCF_ncfttm', 'PS', 'PS_ttm', 'NetProfit_parent_comp_ttm',
'NetProfit_parent_comp_lyr', 'NetAsset', 'NetCashFlow_oper_act_ttm',
'NetCashFlow_oper_act_lyr', 'OperRev_ttm', 'OperRev_lyr',
'NetIncrCash_cash_equ_ttm', 'NetIncrCash_cash_equ_lyr'
]]
df = df.sort_values(by="Ticker")
print(df)
w.stop()
return df
def __del__(self):
w.stop()
def getWsd(security):
w.start()
data = w.wsd(security, "open,close,low,high,amt", 'LQ1')
w.stop()
return data
def post(self):
"""
json str:{"codes": "600008.SH, "fields": "ask1,bid1,asize1,bsize1,volume,amt,pre_close,open,high,low,last",
"begin_time": "2017-01-04", "end_time": "2017-02-28", "options": ""}
:return: 返回万得返回数据dict
"""
args = receive_wst_parser.parse_args()
logger.info('/wst/ args:%s', args)
# codes = args['codes']
# fields = args['fields']
# begin_time = args['begin_time']
# end_time = args['end_time']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.wst(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('wst(%s) ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
# 将 Data数据中所有 datetime date 类型的数据转换为 string
data_len = len(ret_data.Data)
for n_data in range(data_len):
data = ret_data.Data[n_data]
data_len2 = len(data)
if data_len2 > 0:
# 取出第一个部位None的数据
item_check = None
for item_check in data:
if item_check is not None:
break
# 进行类型检查，如果发现是 datetime, date 类型之一，则进行类型转换
if item_check is not None and type(item_check) in (datetime,
date):
ret_data.Data[n_data] = [
format_2_datetime_str(dt) for dt in data
]
logger.info('%d column["%s"] date to str', n_data,
ret_data.Fields[n_data])
# 组成 DataFrame
ret_df = pd.DataFrame(
ret_data.Data,
index=ret_data.Fields,
columns=[format_2_datetime_str(dt) for dt in ret_data.Times])
# print(ret_df)
ret_dic = ret_df.to_dict()
# print('ret_dic:\n', ret_dic)
return ret_dic
MinuteMAFlag, NumAct, monthlist)
if ActFlag != 0: #Act标识可以进行操作
ActualAmount = checkAmount(ActFlag, amount, LivePX, TodayMoney,
TodayPos) #计算真实应该交易的数量
if ActualAmount != 0:
# TodayMoney, TodayPos = Trade(ActFlag, ActualAmount, TaxRate, LivePX, TodayMoney, TodayPos)
TodayMoney, TodayPos = SimulateTrade(Kind, AccountInfo, ActFlag,
ActualAmount, TaxRate, LivePX,
TodayMoney, TodayPos) #模拟交易
NumAct += 1 #当天交易次数加1
LastACT = ActFlag #更新ACT标识
OriginMAFlag = MinuteMAFlag #更新均线交叉标识
PrintTradeLog(ActFlag, ActualAmount, TodayMoney, TodayPos, LogFile)
time.sleep(interval) #根据计算得到的时间间隔，暂停程序运行
ClosePX = w.wsd(Kind, "close", "ED0TD", date.today(),
"Fill=Previous").Data[0][0] # 获取获取今日收盘价
profit = calc_profit(Money, Pos, LastPX, TodayMoney, TodayPos, ClosePX,
OriginAsset) # 计算资金/持仓变化
print("%s Summary of %s %s" % ('-' * 10, TodayDate, '-' * 10)) #分割线
DayFile = CheckDayFile(DayFileName, TodayDate) #
PrintDayRecord(TodayDate, TodayMoney, TodayPos, ClosePX, profit, LastACT,
DayFile)
#Money = TodayMoney
#Pos = TodayPos
LogFile.close()
DayFile.close()
w.stop() #关闭Wind-Python接口
Plot_Live_MAProfit(LogFileName, DayFileName) #根据记录的Day、Log文件绘图
for i in range(0,len(arrOpen)):
po=arrOpen[i]
if po==None or numpy.isnan(po): po=0
ph=arrHigh[i]
if ph==None or numpy.isnan(ph): ph=0
pl=arrLow[i]
if pl==None or numpy.isnan(pl): pl=0
pc=arrClose[i]
if pc==None or numpy.isnan(pc): pc=0
pv=arrVolume[i]
if pv==None or numpy.isnan(pv): pv=0
strO=str(po)
strH=str(ph)
strL=str(pl)
strC=str(pc)
strV=str(pv)
message="insert into dp values(%s,\"%s\",%s,%s,%s,%s,%s)"%(stock.split(".")[0],str(datei),strO,strH,strL,strC,strV)
#print ("meswsage="+message)
cur.execute(message)
datei=datei+oneday
#for i
#for j
#if connect
conn.commit()
#close conenction
w.stop()
def download_marketData(self, indexNameList=[]):
'''
从WIND获取目标数据
:param: startDate: 数据下载时间起点
:return:数据csv文件
'''
# 内置参数
cnMarketDict = {
u'open': u'开盘价',
'high': u'最高价',
'low': u'最低价',
'close': u'收盘价',
'volume': u'成交量',
'amt': u'成交额',
'vwap': u'均价',
'turn': u'换手率',
'vol_ratio': u'量比',
'pct_chg': '涨跌幅',
'mkt_cap_ard': u'总市值',
'mkt_cap_float': '流通市值',
u'maxupordown': u'涨跌停',
u'PB_LF': u'PB_LF',
'PE_TTM': u'PE_TTM',
'PS_TTM': u'PS_TTM',
u'pb': u'pb'
}
mktFileNameDict = {
u'open': u'open',
'high': u'high',
'low': u'low',
'close': u'close',
'volume': u'volume',
'amt': u'amt',
'vwap': u'vwap',
'turn': u'turn',
'vol_ratio': 'vol_ratio',
u'pct_chg': 'pct_chg',
'mkt_cap_ard': u'SZ_Total',
'mkt_cap_float': u'SZ_LT',
u'maxupordown': u'maxupordown',
'PB_LF': u'PB_LF',
'PE_TTM': u'PE_TTM',
'PS_TTM': u'PS_TTM',
u'pb': u'pb'
}
startDate = copy.deepcopy(self.startDate)
lastTradeday = copy.deepcopy(self.lastTradeday)
# 设置全市场数据参数
w.start()
defaultIndexNameList = [
u'open', u'high', u'low', u'close', u'volume', u'turn'
]
if indexNameList == []:
indexNameList = defaultIndexNameList
stockCodelist = copy.deepcopy(self.stockCodelist)
joinStockCode = u','.join(stockCodelist)
colName = copy.deepcopy(stockCodelist)
colName.insert(0, u'Date')
# 下载数据
for i in range(len(indexNameList)):
if indexNameList[i] in ['vol_ratio']:
periodType = 'D'
else:
periodType = 'A'
dataSet = {}
dateRangeDict = {}
uniPeriodList = self.uniPeriodListDcit[periodType]
periodList = self.periodListDcit[periodType]
for p in range(len(uniPeriodList)): # 周期迭代
period = uniPeriodList[p]
periodLocSer = pd.Series(periodList.loc[period])
tempSlice = periodList.index.get_loc(period)
if len(periodLocSer) >= 2:
stLoc = tempSlice.start
enLoc = tempSlice.stop - 1
else:
stLoc = tempSlice
enLoc = tempSlice
dateRange = [self.strDayList[stLoc], self.strDayList[enLoc]]
# 下载数据
downLoadFormula = 'w.wsd(joinStockCode, indexNameList[i], dateRange[0], dateRange[-1],'
if indexNameList[i] == u"mkt_cap_float":
result = eval(downLoadFormula +
'"unit=1;currencyType=;PriceAdj=F")')
elif indexNameList[i] == u"mkt_cap_ard":
result = eval(downLoadFormula + '"unit=1;PriceAdj=F")')
elif indexNameList[i] in [u"pb"]:
result = eval(downLoadFormula + '"ruleType=3;PriceAdj=F")')
elif indexNameList[i] in ['vol_ratio']: # 逐天下载
result = eval(downLoadFormula +
'"VolumeRatio_N=5;PriceAdj=F")')
else:
result = eval(downLoadFormula +
'"PriceAdj=F")') # 普通行情数据，例如开盘价，换手率
dataSet[period] = result
dateRangeDict[period] = [stLoc, enLoc]
print(cnMarketDict[indexNameList[i]] + u'全时间段下载完毕') # 保存数据至csv文件
strFilename = os.getcwd() + u'/DownloadedData/OriginalData/' + cnMarketDict[indexNameList[i]] + \
startDate + u'-' + str(lastTradeday) + u'.csv'
self.processedFilenameDict[mktFileNameDict[
indexNameList[i]]] = strFilename
csvFile = open(strFilename, u'w', newline=u'')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName) # 第一行股票代码
for key in uniPeriodList: # 周期
st = dateRangeDict[key][0]
end = dateRangeDict[key][-1]
csvTradedayList = self.strDayList[st:end + 1]
for j in range(len(csvTradedayList)): # 不同交易日
row = [i[j] for i in dataSet[key].Data] # j 时间段所有股票代码数据
row.insert(0, csvTradedayList[j])
csvWriter.writerow(row)
csvFile.close()
w.stop()
print(u'所有全市场数据下载完毕')
def get_min_data(filename):
file_folder = 'C:\\dev\\src\\ktlib\\data3\\'
reader = csv.reader(file(filename, 'rb'))
contList = []
fields = 'open,close,high,low,volume,oi'
contStart = []
contEnd = []
for line in reader:
contList.append(line[0])
contStart.append(line[1])
contEnd.append(line[2])
#w.start()
for cont, startD, endD in zip(contList, contStart, contEnd):
if cont[1].isalpha(): key = cont[:2]
else: key = cont[:1]
ex = 'SH'
for exch in product_code.keys():
if key in product_code[exch]:
ex = exch
ticker = cont + '.' + ex
startmin = ""
endMin = ""
if ex == 'CFE':
startMin = " 09:14:00"
endMin = " 15:15:00"
else:
startMin = " 08:59:00"
endMin = " 15:00:00"
if ex == 'CFE':
startMin = " 09:14:00"
endMin = " 15:15:00"
else:
startMin = " 08:59:00"
endMin = " 15:00:00"
try:
sdate = datetime.datetime.strptime(startD, "%Y/%m/%d")
edate = datetime.datetime.strptime(endD, "%Y/%m/%d")
sMin = datetime.datetime.strptime(startD + startMin, "%Y/%m/%d %H:%M:%S")
eMin = datetime.datetime.strptime(endD + endMin, "%Y/%m/%d %H:%M:%S")
print ticker, sMin, eMin
if edate > datetime.date(2009,1,1):
raw_data = w.wsi(ticker,fields,sMin,eMin)
if len(raw_data.Data)>1:
outfile = file_folder + cont+'_min.csv'
output={'datetime':raw_data.Times,
'open':raw_data.Data[0],
'close':raw_data.Data[1],
'high':raw_data.Data[2],
'low':raw_data.Data[3],
'volume':raw_data.Data[4],
'openInterest':raw_data.Data[5]}
dump2csvfile(output,outfile)
else:
print "no min data obtained for ticker=%s" % ticker
raw_data = w.wsd(ticker,fields,sdate,edate)
if len(raw_data.Data)>1:
outfile = file_folder + cont+'_daily.csv'
output={'datetime':raw_data.Times,
'open':raw_data.Data[0],
'close':raw_data.Data[1],
'high':raw_data.Data[2],
'low':raw_data.Data[3],
'volume':raw_data.Data[4],
'openInterest':raw_data.Data[5]}
dump2csvfile(output,outfile)
else:
print "no daily data obtained for ticker=%s" % ticker
except ValueError:
pass
w.stop()
return True
def download_indexData(self, indexCodeList=[]):
# 内置参数
startDate = copy.deepcopy(self.startDate)
endDate = copy.deepcopy(self.endDate)
lastTradeday = copy.deepcopy(self.lastTradeday)
cnIndexDict = {
u"000016.SH": u'上证50指数',
u"000905.SH": u'中证500指数',
u"000001.SH": u'上证指数',
u"000300.SH": u'沪深300指数',
u"399005.SZ": u'中小板指数',
u'000852.SH': u'中证1000',
u"399001.SZ": u'深圳成指',
u"399006.SZ": u'创业板',
"399902.SZ": '中证流通',
"SPBCNCOT.SPI": '标普中国债券指数',
"NH0100.NHF": '南华商品指数',
"049.CS": '中债银行间债券总财富(总值)指数',
"884100.WI": '新三板指数',
"885006.WI": u'混合债券型一级基金指数',
"885007.WI": '长期纯债型基金指数',
"885008.WI": '混合债券型二级基金指数'
}
indexFileNameDict = {
u"000016.SH": u'sz50',
u"000905.SH": u'zz500',
u"000001.SH": u'szzs',
u"000300.SH": u'hs300',
u"399005.SZ": u'zxb',
u'000852.SH': u'zz1000',
u"399001.SZ": u'szcz',
u"399006.SZ": u'cyb'
}
# 设置指数数据
w.start()
if indexCodeList == []:
indexCodeList = [
u"000016.SH", u"000905.SH", u"000001.SH", u"000300.SH",
u"399005.SZ", u'000852.SH', u"399001.SZ", u"399006.SZ"
]
# 指数数据下载
fieldName = [u"open", u"high", u"low", u"close", u"volume"]
# fieldName = [u"open", u"close"]
joinFieldName = ','.join(fieldName)
fieldCNName = [u'开盘', u'最高', u'最低', u'收盘', u'成交量']
colName2 = copy.deepcopy(fieldCNName)
colName2.insert(0, u'Date')
for i in range(len(indexCodeList)):
dataSet = {}
# 按年下载数据
yearNum = int(startDate.split(u'-')[0])
yearTradedaySet = {}
tempStartDate = startDate
latestYearNum, lastEndDate = int(endDate.split(u'-')[0]), int(
endDate.split(u'-')[2])
while yearNum <= latestYearNum:
if yearNum == int(latestYearNum):
tempEndDate = endDate
else:
tempEndDate = str(yearNum) + u'-12-31'
yearTradedayList = w.tdays(tempStartDate, tempEndDate,
u"").Data[0]
yearFirstTradeday = yearTradedayList[0].date()
yearLastTradeday = yearTradedayList[-1].date()
# 下载数据
indexResult = w.wsd(indexCodeList[i], joinFieldName,
str(yearFirstTradeday),
str(yearLastTradeday), u"PriceAdj=F")
dataSet[str(yearNum)] = indexResult
yearTradedaySet[str(yearNum)] = yearTradedayList
yearNum += int(1)
tempStartDate = str(yearNum) + u'-01-01'
print(cnIndexDict[indexCodeList[i]] + u'全时间段下载完毕')
# 创建文件
tempDf = pd.DataFrame()
strFilename = os.getcwd() + u'/DownloadedData/OriginalData/' + cnIndexDict[indexCodeList[i]] + \
startDate + r'-' + str(lastTradeday) + u'.csv'
self.processedFilenameDict[indexFileNameDict[
indexCodeList[i]]] = strFilename
csvFile = open(strFilename, u'w', newline=u'')
csvWriter = csv.writer(csvFile)
csvWriter.writerow(colName2) # 第一行股票代码
for key in yearTradedaySet: # 不同年
csvTradedayList = yearTradedaySet[key]
for j in range(len(csvTradedayList)):
row = [i[j] for i in dataSet[key].Data] # j 时间段所有股票代码数据
row.insert(0, csvTradedayList[j].date())
csvWriter.writerow(row)
tempDf = tempDf.append([row])
csvFile.close()
w.stop()
print(u'所有指数数据下载完毕')
"""
Created on Fri May 8 12:20:47 2020
@author: -
"""
import pandas as pd # 用于数据处理
from WindPy import w # 用于打开wind
import matplotlib.pyplot as plt # 用于画图
#pd.read_excel("draw.xlsx",sheet_name='')
w.start() # 用于打开wind
df1 = w.wsd("000001.SH", "close", "2019-01-01", "2019-12-31", usedf=True)[1]
df2 = w.wsd("000001.SH", "close", "2020-01-01", "2020-04-30", usedf=True)[1]
df = w.wsd("000001.SH", "close", "2019-01-01", "2020-04-30", usedf=True)[1]
#fig = plt.figure()
plt.plot(df1.index, df1.CLOSE)
plt.plot(df2.index, df2.CLOSE)
plt.plot(df.index[:-len(df2)], df.CLOSE.iloc[:-len(df2)])
plt.plot(df.index[len(df1):], df.CLOSE.iloc[len(df1):])
plt.show()
#plt.figure(figsize=(200, 80))
w.stop() # 用于关闭wind
def __init__(self,
stockCodelist=[],
marketGroupCode=[],
startDate=u'2017-09-08',
endDate=[],
*args):
'''
初始化参数
'''
self.startDate = startDate
if startDate == []:
self.startDate = str(datetime.date.today() -
datetime.timedelta(days=1))
self.endDate = endDate
if self.endDate == []:
self.endDate = str(datetime.date.today())
# self.endDate = datetime.date.today() - datetime.timedelta(days=1)
self.marketGroupCode = marketGroupCode
if self.marketGroupCode == []:
self.marketGroupCode = 'a001010100000000'
# 交易日数据获取
w.start()
self.tradedayList = w.tdays(self.startDate, self.endDate, u"").Data[0]
self.strDayList = [str(i.date()) for i in self.tradedayList]
tradedayDf = pd.DataFrame(self.tradedayList, index=self.tradedayList)
tradedayDf.to_csv('tradedayList.csv')
# 周期序列字典
self.periodTypeList = ['D', 'W', 'M', 'A']
self.uniPeriodListDcit = {
i: toPeriodList(periodType=i, dailyList=self.tradedayList)[0]
for i in self.periodTypeList
}
self.periodListDcit = {
i: toPeriodList(periodType=i, dailyList=self.tradedayList)[1]
for i in self.periodTypeList
}
self.lastTradeday = self.tradedayList[-1].date()
self.processedFilenameDict = {}
# stockCodelist 获取
self.stockCodelist = stockCodelist
if self.stockCodelist == []:
# 取所有时间段股票代码
periodType = 'A' # 按年循环
uniPeriodList = self.uniPeriodListDcit[periodType]
periodList = self.periodListDcit[periodType]
for p in range(len(uniPeriodList)): # 周期迭代
period = uniPeriodList[p]
periodLocSer = pd.Series(periodList.loc[period])
tempSlice = periodList.index.get_loc(period)
if len(periodLocSer) >= 2:
stLoc = tempSlice.start
enLoc = tempSlice.stop - 1
else:
stLoc = tempSlice
enLoc = tempSlice
dateRange = [self.strDayList[stLoc], self.strDayList[enLoc]]
yearStockCodelist = w.wset(
u"sectorconstituent",
u"date=" + dateRange[-1] + u";sectorid=" +
self.marketGroupCode + u";field=wind_code").Data[0] # 年末
if not p: # 第一期
fianlStockCodelist = list(yearStockCodelist)
else:
fianlStockCodelist = list(
set(fianlStockCodelist).union(set(yearStockCodelist)))
fianlStockCodelist.sort()
self.stockCodelist = fianlStockCodelist
# csvFile1 = open(u'StockCodes' + self.strDayList[-1] + '.csv', u'w', newline=u'')
# csvWriter = csv.writer(csvFile1)
# csvWriter.writerow(self.stockCodelist) # 第一行股票代码
# csvFile1.close()
w.stop()
def post(self):
"""
json str:{"codes": "M0017126,M0017127,M0017128",
"begin_time": "2016-11-10", "end_time": "2017-11-10", "options": "Fill=Previous"}
:return: 返回万得返回数据dict
"""
args = receive_edb_parser.parse_args()
logger.info('/edb/ args:%s', args)
codes = args['codes']
# begin_time = args['begin_time']
# end_time = args['end_time']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.edb(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('wst(%s) ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
# 将 Data数据中所有 datetime date 类型的数据转换为 string
data_len = len(ret_data.Data)
for n_data in range(data_len):
data = ret_data.Data[n_data]
data_len2 = len(data)
if data_len2 > 0:
# 取出第一个部位None的数据
item_check = None
for item_check in data:
if item_check is not None:
break
# 进行类型检查，如果发现是 datetime, date 类型之一，则进行类型转换
if item_check is not None and type(item_check) in (datetime,
date):
ret_data.Data[n_data] = [
format_2_date_str(dt) for dt in data
]
logger.info('%d column["%s"] date to str', n_data,
ret_data.Fields[n_data])
# 组成 DataFrame
ret_df = pd.DataFrame(
ret_data.Data,
index=[xx.strip() for xx in codes.split(',')],
columns=[format_2_date_str(dt) for dt in ret_data.Times])
# print(ret_df)
ret_dic = ret_df.to_dict()
# print('ret_dic:\n', ret_dic)
return ret_dic
def post(self):
"""
json str:{"tablename": "sectorconstituent", "options": "date=2017-03-21;sectorid=1000023121000000"}
:return: 返回万得返回数据dict
"""
args = receive_wset_parser.parse_args()
logger.info('/wset/ args:%s' % args)
# print('args:%s' % args)
# table_name = args['table_name']
# options = args['options']
if args['options'] == "":
args['options'] = None
if not w.isconnected():
w.start()
ret_data = None
for nth in range(2):
ret_data = w.wset(**args)
error_code = ret_data.ErrorCode
if error_code != 0:
if nth == 0 and error_code == -40521010:
w.stop()
w.start()
logger.warning('尝试重新登陆成功，再次调用函数')
continue
msg = ERROR_CODE_MSG_DIC.setdefault(error_code, "")
logger.error('wset(%s) ErrorCode=%d %s' %
(args, error_code, msg))
raise RequestError(msg, None, error_code)
else:
break
else:
if ret_data is None:
msg = 'wst(%s) ret_data is None' % args
logger.error(msg)
raise RequestError(msg, None, 0)
data_count = len(ret_data.Data)
# if data_count > 0:
# print('ret_data.Fields\n', ret_data.Fields)
# ret_data.Data[0] = [format_2_date_str(dt) for dt in ret_data.Data[0]]
# print('ret_data.Data\n', ret_data.Data)
for n_data in range(data_count):
data_list = ret_data.Data[n_data]
data_list_len = len(data_list)
if data_list_len > 0:
# 取出第一个不为None的数据
item_check = None
for item_check in data_list:
if item_check is not None:
break
# 进行类型检查，如果发现是 datetime, date 类型之一，则进行类型转换
if item_check is not None and type(item_check) in (datetime,
date):
ret_data.Data[n_data] = [
format_2_date_str(dt) for dt in data_list
]
logger.debug('Data[%d] column["%s"] date to str', n_data,
ret_data.Fields[n_data])
ret_df = pd.DataFrame(ret_data.Data,
index=ret_data.Fields,
columns=ret_data.Codes)
# print('ret_df\n', ret_df)
ret_dic = ret_df.to_dict()
# print('ret_dic:\n', ret_dic)
return ret_dic
index = [
'000300.SH', '716567.CSI', 'SPCLLHCP.SPI', 'h30089.CSI', '931079.CSI',
'930606.CSI', '399998.SZ'
]
if __name__ == '__main__':
date_str = datetime.date.today().strftime('%Y-%m-%d')
return_df = pd.DataFrame()
w.start()
for idx in index:
index_list = w.wset("indexconstituent",
"date=" + date_str + "windcode=" + idx)
if index_list.ErrorCode == 0:
tmp_df = pd.DataFrame(index_list.Data,
index=index_list.Fields,
columns=index_list.Codes).T
tmp_df['index'] = idx
return_df = return_df.append(tmp_df)
else:
print(index_list.Data)
sys.exit(0)
w.stop()
print(return_df)
writer = pd.ExcelWriter('indexlist.xlsx')
return_df.to_excel(writer, sheet_name='indexlist')
writer.save()
def closeWind():
'''
关闭wind接口
'''
w.stop()
