def _parsing_dayprice_json(self, pageNum=1):
"""
处理当日行情分页数据，格式为json
Parameters
------
pageNum:页码
return
-------
DataFrame 当日所有股票交易数据(DataFrame)
"""
consts._write_console()
request = Request(consts.SINA_DAY_PRICE_URL%(consts.P_TYPE['http'], consts.DOMAINS['vsf'],
consts.PAGES['jv'], pageNum))
text = urlopen(request, timeout=10).read()
if text == 'null':
return None
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text.decode('gbk'))
text = text.replace('"{symbol', '{"symbol')
text = text.replace('{symbol', '{"symbol"')
jstr = json.dumps(text)
js = json.loads(jstr)
df = pandas.DataFrame(pandas.read_json(js, dtype={'code':object}),
columns=consts.DAY_TRADING_COLUMNS)
df = df.drop('symbol', axis=1)
# df = df.ix[df.volume > 0]
return df
def getRehabilitationData(self, code, start=None, end=None, autype='qfq',
index=False, retry_count=3, pause=0.001, drop_factor=True):
'''
获取历史复权数据
Parameters
------
code:string
股票代码 e.g. 600848
start:string
开始日期 format：YYYY-MM-DD 为空时取当前日期
end:string
结束日期 format：YYYY-MM-DD 为空时取去年今日
autype:string
复权类型，qfq-前复权 hfq-后复权 None-不复权，默认为qfq
retry_count : int, 默认 3
如遇网络等问题重复执行的次数
pause : int, 默认 0
重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
drop_factor : bool, 默认 True
是否移除复权因子，在分析过程中可能复权因子意义不大，但是如需要先储存到数据库之后再分析的话，有该项目会更加灵活
return
-------
DataFrame
date 交易日期 (index)
open 开盘价
high 最高价
close 收盘价
low 最低价
volume 成交量
amount 成交金额
'''
#if no give start ,set start last year
start = DataUtils.getDefaultLastYear() if start is None else start
end = DataUtils.getStrToday() if end is None else end
qs = DataUtils.getQuarters(start, end)
qt = qs[0]
consts._write_head()
data = self._getRehabilitationByQuarter(self._get_index_url(index, code, qt), index,
retry_count, pause)
if len(qs)>1:
for d in range(1, len(qs)):
qt = qs[d]
consts._write_console()
df = self._getRehabilitationByQuarter(self._get_index_url(index, code, qt), index,
retry_count, pause)
data = data.append(df, ignore_index=True)
if len(data) == 0 or len(data[(data.date>=start)&(data.date<=end)]) == 0:
return None
data = data.drop_duplicates('date')
if index:
data = data[(data.date>=start) & (data.date<=end)]
data = data.set_index('date')
data = data.sort_index(ascending=False)
return data
if autype == 'hfq':
if drop_factor:
data = data.drop('factor', axis=1)
data = data[(data.date>=start) & (data.date<=end)]
for label in ['open', 'high', 'close', 'low']:
data[label] = data[label].map(consts.FORMAT)
data[label] = data[label].astype(float)
data = data.set_index('date')
data = data.sort_index(ascending = False)
return data
else:
if autype == 'qfq':
if drop_factor:
data = data.drop('factor', axis=1)
df = self._parase_fq_factor(code, start, end)
df = df.drop_duplicates('date')
df = df.sort_values('date', ascending=False)
firstDate = data.head(1)['date']
frow = df[df.date == firstDate[0]]
rt = self.get_realtime_quotes(code)
if rt is None:
return None
if ((float(rt['high']) == 0) & (float(rt['low']) == 0)):
preClose = float(rt['pre_close'])
else:
if DataUtils.isHoliday(DataUtils.getStrToday()):
preClose = float(rt['price'])
else:
if (DataUtils.getTodayHour() > 9) & (DataUtils.getTodayHour() < 18):
preClose = float(rt['pre_close'])
else:
preClose = float(rt['price'])
rate = float(frow['factor']) / preClose
data = data[(data.date >= start) & (data.date <= end)]
for label in ['open', 'high', 'low', 'close']:
data[label] = data[label] / rate
data[label] = data[label].map(consts.FORMAT)
data[label] = data[label].astype(float)
data = data.set_index('date')
data = data.sort_index(ascending = False)
return data
else:
for label in ['open', 'high', 'close', 'low']:
data[label] = data[label] / data['factor']
if drop_factor:
data = data.drop('factor', axis=1)
data = data[(data.date>=start) & (data.date<=end)]
for label in ['open', 'high', 'close', 'low']:
data[label] = data[label].map(consts.FORMAT)
data = data.set_index('date')
data = data.sort_index(ascending = False)
data = data.astype(float)
return data
