def test_trade_date_sse(self):
qs = Stocktradedate.importList()
day = datetime.datetime.now()
sd = Stocktradedate()
self.assertTrue(
qs.filter(tradedate__lte=day).count() == sd.trade_date_sse.count(),
'2018年之前的交易日共：{}'.format(qs.filter(tradedate__lte=day).count()))
def getNearestTradedate(date=datetime.datetime.now().date(), days=0):
""" 获取离date最近的交易日期
只能获取到前一交易日的数据qa.QA_fetch_index_day_adv
:param date:
:return:
"""
from stocks.models import Stocktradedate
if not date:
# 如果date为空，则赋值当天
date = datetime.datetime.now().date()
date = convertToDate(date)
tradedate = Stocktradedate.get_real_date(date, -1).date()
if date == tradedate:
if date == datetime.datetime.now().date():
# 当天并且是交易日
tradedate = Stocktradedate.preTradeday(tradedate)
if days == 0:
return tradedate
elif days < 0:
for day in range(abs(days)):
tradedate = Stocktradedate.preTradeday(tradedate)
else:
for day in range(days):
tradedate = Stocktradedate.nextTradeday(tradedate)
return tradedate
def test_everydayCount(self):
Stocktradedate.importList()
for d in [
d[0] for d in set(
list(HSGTCGHold.getlist().values_list('tradedate')))
]:
dcount = HSGTCGHold.getlist(d).count()
print(d, dcount)
from stocks.models import HSGTCG
tdate = HSGTCGHold.getNearestTradedate()
while tdate > convertToDate('2018-5-1'):
dcount = HSGTCGHold.getlist(tdate).count()
dcount1 = HSGTCG.getlist().filter(tradedate=tdate,
hamount__gte=8000).count()
print(tdate, dcount, dcount1, dcount - dcount1)
tdate = HSGTCGHold.getNearestTradedate(tdate -
datetime.timedelta(1))
from stocks.models import HSGTCG
tdate = HSGTCGHold.getNearestTradedate()
while tdate > convertToDate('2018-5-1'):
df1 = pd.DataFrame(list(HSGTCGHold.getlist(tdate).values('code')))
dcount = len(df1)
df2 = pd.DataFrame(
list(HSGTCG.getlist().filter(
tradedate=tdate, hamount__gte=8000).values('code')))
dcount1 = len(df2)
print(tdate, dcount, dcount1, dcount - dcount1)
if dcount - dcount1 != 0 and tdate > convertToDate('2018-6-1'):
print('数据不一致：', end='')
print(list(HSGTCGHold.dfNotInAnotherdf(df1, df2)['code']))
tdate = HSGTCGHold.getNearestTradedate(tdate -
datetime.timedelta(1))
def test_newcomingin(self):
""" 新加入市值八千万的个股
:return:
"""
Stocktradedate.importList()
from stocks.models import HSGTCG
# 2018 - 06 - 04 新增北向持股金额大于八千万
list1 = [
'603658', '600460', '002812', '002557', '600188', '000690',
'600329'
]
tdate = HSGTCGHold.getNearestTradedate()
# tdate = HSGTCGHold.getNearestTradedate('2018-6-4')
tdate1 = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1))
hsg = HSGTCGHold.getlist(tdate)
hsg1 = HSGTCGHold.getlist(tdate1)
list2 = []
for c in hsg.exclude(code__in=hsg1.values_list('code')):
list2.append(c.code)
# 验证是否前一天市值小于八千万
for code in list2:
df = pd.DataFrame(
list(
HSGTCG.getlist(code).filter(tradedate__gte=tdate1).values(
).order_by('-tradedate')))
data1 = float(df.iloc[-2].hamount)
data2 = float(df.iloc[-1].hamount)
if not (data1 >= 8000 and data2 < 8000):
print('不是新增持股金额大于八千万：{} 持股金额：{} {}'.format(code, data1, data2))
tdate = HSGTCGHold.getNearestTradedate()
while tdate > convertToDate('2018-5-2'):
tdate1 = HSGTCGHold.getNearestTradedate(tdate -
datetime.timedelta(1))
hsg = HSGTCGHold.getlist(tdate)
hsg1 = HSGTCGHold.getlist(tdate1)
list2 = []
for c in hsg.exclude(code__in=hsg1.values_list('code')):
list2.append(c.code)
# 验证是否前一天市值小于八千万
for code in list2:
df = pd.DataFrame(
list(
HSGTCG.getlist(code).filter(tradedate__gte=tdate1).
values().order_by('-tradedate')))
if len(df) > 1 and HSGTCG.getlist().filter(
tradedate=tdate1).count() > 0:
# if len(df)> 1 :
data1 = float(df.iloc[-2].hamount)
data2 = float(df.iloc[-1].hamount)
if not (data1 >= 8000 and data2 < 8000):
print('{} 不是新增持股金额大于八千万：{} 持股金额：{} {}'.format(
tdate, code, data1, data2))
tdate = HSGTCGHold.getNearestTradedate(tdate -
datetime.timedelta(1))
def test_getRecentlist(self):
Stocktradedate.importList()
n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate')))
self.assertTrue(len(n) <= 1, '交易日:{}'.format(n))
days = 3
n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate')))
self.assertTrue(len(n) <= days, '交易日:{}'.format(n))
days = 10
n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate')))
self.assertTrue(len(n) <= days, '交易日:{}'.format(n))
def test_get_real_date(self):
qs = Stocktradedate.importList()
date = '2018-5-21'
day = datetime.datetime.strptime(date, '%Y-%m-%d')
self.assertTrue(
Stocktradedate.get_real_date(day) == day,
'({}) != ({})'.format(Stocktradedate.get_real_date(day), day))
date = '2018-5-20'
day = datetime.datetime.strptime(date, '%Y-%m-%d')
self.assertFalse(
Stocktradedate.get_real_date(day) == day,
'({}) != ({})'.format(Stocktradedate.get_real_date(day), day))
def importWebdriverList(cls, firefoxHeadless=False):
""" 导入市值大于指定值的列表
网址： http://data.eastmoney.com/hsgtcg/StockStatistics.aspx
不建议用此方法，经常丢失数据
:param firefoxHeadless: 是否显示浏览器界面：
True 不显示界面
False 显示界面
默认不显示浏览器界面
:return: 最近交易日期的列表
"""
hsgh = HSGTCGHold.getlist(tradedate=cls.getNearestTradedate())
if hsgh.count() > 0:
return hsgh
browser = cls.getBrowser(firefoxHeadless)
url = 'http://data.eastmoney.com/hsgtcg/StockStatistics.aspx'
df = cls.scrap(url, browser)
df = df[['code', 'tradedate']]
df = df[df['tradedate'].apply(lambda x: Stocktradedate.if_tradeday(x))]
# 去除重复数据
df = df[~df.duplicated()]
cls.savedf(df[['code', 'tradedate']])
# pandas dataframe save to model
# HSGTCGHold.objects.bulk_create(
# HSGTCGHold(**vals) for vals in df[['code', 'tradedate']].to_dict('records')
# )
return cls.getlist(tradedate=cls.getNearestTradedate())
def getStockHdStatistics(cls, code, browser, retryCount=3):
""" 抓取持股统计
:param code: 股票代码
:param browser: webdriver浏览器
:return:
"""
url = 'http://data.eastmoney.com/hsgtcg/StockHdStatistics.aspx?stock={}'.format(
code)
for i in range(retryCount):
df = cls.scrap(url, browser)
if len(df) > 0:
# 修复持股数量
df['hvol'] = df['hvol'].apply(
lambda x: HSGTCG.hz2Num(x)).astype(float)
df['hamount'] = df['hamount'].apply(
lambda x: HSGTCG.hz2Num(x)).astype(float)
df['close'] = df['close'].astype(float)
df['tradedate'] = df['tradedate'].apply(
lambda x: convertToDate(x)).astype(datetime.date)
df = df[df['tradedate'].apply(
lambda x: Stocktradedate.if_tradeday(
x))] # 删除不是交易日的数据。这是东方财富网页版的bug
df.index = pd.RangeIndex(len(df.index))
break
else:
pass
return df
def test_get_real_datelist(self):
qs = Stocktradedate.importList()
day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日
day2 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日
start, end = Stocktradedate.get_real_date_start_end(day1, day2)
self.assertIsNone(start)
day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日
day2 = datetime.datetime.strptime('2018-5-15', '%Y-%m-%d').date() # 周日
start, end = Stocktradedate.get_real_date_start_end(day1, day2)
self.assertIsInstance(
start, datetime.date,
'返回值为datetime.date类型， 实际返回{}'.format(type(start)))
day3 = datetime.datetime.strptime('2018-5-14', '%Y-%m-%d').date() # 周一
self.assertTrue(start == day3,
'周日应该返回下一工作日:{}, 实际返回:{}'.format(day3, start))
def test_nextTradeday(self):
qs = Stocktradedate.importList()
day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日
day = Stocktradedate.nextTradeday(day1)
self.assertTrue(
'wrong date' == day,
'非交易日 {} !={}'.format(day1 + datetime.timedelta(1), day))
day1 = datetime.datetime.strptime('2018-5-14', '%Y-%m-%d').date() # 周一
day = Stocktradedate.nextTradeday(day1)
self.assertTrue(day1 + datetime.timedelta(1) == day,
'{} !={}'.format(day1 + datetime.timedelta(1), day))
n = 5
day = Stocktradedate.nextTradeday(day1, n) #
# print(type('day type: {}, date type:{}'.format(day, day1)))
self.assertTrue(day1 + datetime.timedelta(n) < day,
'{} !={}'.format(day1 + datetime.timedelta(n), day))
def importjsonList(cls, enddate=None, days=5):
""" 导入市值大于指定值的列表
网址： http://data.eastmoney.com/hsgtcg/StockStatistics.aspx
直接下载json文件转换，效率比importList高
:param firefoxHeadless: 是否显示浏览器界面：
True 不显示界面
False 显示界面
默认不显示浏览器界面
:return: 最近交易日期的列表jsonname
"""
if enddate:
end = convertToDate(enddate)
else:
end = cls.getNearestTradedate()
hsgh = cls.getlist(tradedate=end)
if hsgh.count() > 0:
return hsgh
pagesize = 300 # 每页数据量
page = 1
sortRule = -1 # sortRule-1 按照市值降序排序； 1 按照市值升序排序
start = end - datetime.timedelta(days)
jsname = cls.getRandomStr('letter')
for page in range(1, 100):
if page > 1:
sr = -1
# st=SHAREHOLDPRICE 按照持股市值排序; st sortType
url = 'http://dcfm.eastmoney.com//em_mutisvcexpandinterface/api/js/' \
'get?type=HSGTHDSTA&token=70f12f2f4f091e459a279469fe49eca5&st=SHAREHOLDPRICE&sr={sortRule}' \
'&p={page}&ps={pagesize}&js=var%20{jsname}={pages:(tp),data:(x)}&filter=(MARKET%20in%20(%27001%27,%27003%27))' \
'(HDDATE%3E=^{start}^%20and%20HDDATE%3C=^{end}^)&rt=50950960' \
.replace('{start}', str(start)).replace('{end}', str(end)) \
.replace('{sortRule}', str(sortRule)) \
.replace('{pagesize}', str(pagesize)) \
.replace('{page}', str(page)) \
.replace('{jsname}', str(jsname))
df = cls.scrapjson(url)
dfn = df[df['hamount'] >= MINHAMOUNT]
if len(dfn) > 0:
dfn = dfn[dfn['tradedate'].apply(
lambda x: Stocktradedate.if_tradeday(x))]
# 去除重复数据
dfn = dfn[~dfn.duplicated()]
cls.savedf(dfn[['code', 'tradedate']])
print('page: {}'.format(page))
if len(df[df['hamount'] < MINHAMOUNT]):
# 持股金额小于
break
return cls.getlist(tradedate=cls.getNearestTradedate())
def test_caculateRPS(self):
# Listing.importIndexListing()
# rpspreparelist = RPSprepare.importIndexListing()
qs = RPSprepare.getlist('index')
assert qs.count() > 0, "RPSprepare.getlist('index') 返回个数应大于零"
start, _ = Stocktradedate.get_real_datelist('2018-1-1', '2018-2-1')
# qs = RPSprepare.getlist('index').filter(tradedate=convertToDate('2018-1-4'))
qs = RPSprepare.getlist('index').filter(tradedate=start)
df = pd.DataFrame(list(qs.values()))
# self.assertTrue(len(df) > 0, "RPSprepare.getlist('index') 返回个数应大于零, {}".format(len(df)))
data = RPS.caculateRPS(df)
self.assertTrue(
len(data) / len(data[data['rps120'] == data['rps250']]) > 2,
'rps120等于rps250的几率很小'.format(
len(data), len(data[data['rps120'] == data['rps250']])))
def test_importList(self):
qs = Stocktradedate.importList()
day = datetime.datetime.strptime('1991-12-31', '%Y-%m-%d')
self.assertTrue(
qs.filter(tradedate__lte=day).count() == 264,
'1991年之前的交易日共：{}'.format(qs.filter(tradedate__lte=day).count()))
day = datetime.datetime.strptime('2017-12-31', '%Y-%m-%d')
self.assertTrue(
qs.filter(tradedate__lte=day).count() == 6613,
'2018年之前的交易日共：{}'.format(qs.filter(tradedate__lte=day).count()))
day = datetime.datetime.strptime('2018-3-31', '%Y-%m-%d')
self.assertTrue(
qs.filter(tradedate__lte=day).count() > 6613,
'2018年之前的交易日共：{}'.format(qs.filter(tradedate__lte=day).count()))
def getRecentlist(cls, days=1):
""" 返回最近days天列表
:param days: 之前几个交易
:return: .objects.all().filter(tradedate__gte=tdate)
"""
if days == 0:
return cls.objects.all()
else:
tdate = cls.getNearestTradedate()
if days > 1:
realdatelist = Stocktradedate.get_real_datelisting(
tdate - datetime.timedelta(days * 2 + 10), tdate)
tdate = pd.DataFrame(
list(realdatelist.order_by('id').values_list(
'tradedate'))).iloc[-days][0]
# 返回所有代码
return cls.objects.all().filter(tradedate__gte=tdate)
def importIndexListing(cls, start='2006-1-1', end=None):
"""
插入所有股票指数
:return:
"""
if end is None:
# 获取当天
end = datetime.datetime.now().date()
qs = RPSprepare.getlist('index')
# todo 如果已经插入，则判断是否有更新
try:
# 批量创建对象，减少SQL查询次数
querysetlist = []
delisted = [] # quantaxis中无数据时，保存到delisted
for v in Stocktradedate.get_real_datelisting(start, end).values('tradedate'):
print(v)
# 获取 v['tradedate']对应的指数列表
qsday = qs.filter(tradedate=v['tradedate'])
df = pd.DataFrame(list(qsday.values()))
data = RPS.caculateRPS(df)
aaa
# code = Listing.objects.get(code=v.code.code, category=11)
if len(data) > 0:
df = pd.DataFrame(data.close)
df['rps120'] = df.close / df.close.shift(120)
df['rps250'] = df.close / df.close.shift(250)
del df['close']
df = df[120:]
for a, b in df.values:
querysetlist.append(
RPS(code=code, rps120=b, rps250=b))
else:
# quantaxis中无数据
delisted.append(a)
print('delisted count {} :\n {}'.format(len(delisted), delisted))
RPSprepare.objects.bulk_create(querysetlist)
except Exception as e:
print(e.args)
return cls.getlist('index')
def setUpTestData(cls):
Stocktradedate.importList()
def setUpTestData(cls):
Stocktradedate.importList()
Listing.importIndexListing()
rpspreparelist = RPSprepare.importIndexListing()
def test_deleteNotTradedate(self):
# 删除不是交易日的数据
for d in set(list(HSGTCG.objects.all().values_list('tradedate'))):
if not Stocktradedate.if_tradeday(d[0]):
HSGTCG.objects.all().filter(tradedate=d[0]).delete()
print(d[0])
def test_if_trade(self):
qs = Stocktradedate.importList()
day = datetime.datetime.strptime('2018-5-21', '%Y-%m-%d')
self.assertTrue(Stocktradedate.if_tradeday(day), '{} 是交易日'.format(day))
def setUp(self):
Stocktradedate.importList()
def importStockListing(cls, start=None, end=None, ignoreSaved=True):
""" 插入所有股票RPS
:param start: 计算的开始日期
:param end:
:param ignoreSaved: Boolean 是否重新计算已保存的RPS
:return:
"""
"""
:return:
"""
stocktype = 'stock'
if start is None:
# 数据库中最大的已计算日期
latest = cls.getlist('stock').aggregate(
Max('tradedate'))['tradedate__max']
if latest:
start = cls.getNearestTradedate(latest, -5)
else:
start = '2014-1-1'
if end is None:
# 获取当天
end = datetime.datetime.now().date()
qs = RPSprepare.getlist(stocktype)
try:
# 批量创建对象，减少SQL查询次数
querysetlist = []
delisted = [] # quantaxis中无数据时，保存到delisted
qssaved = []
for v in Stocktradedate.get_real_datelisting(
start, end).values('tradedate'):
print('dealing ... {}'.format(v['tradedate']))
# 获取 v['tradedate']对应的股票RPS预计算列表
qsday = qs.filter(tradedate=v['tradedate'])
if ignoreSaved:
if qsday.count() == cls.getlist(stocktype).filter(
tradedate=v['tradedate']).count():
# RPS预计算数量和已保存的RPS数量相同，则跳过
# print('pass')
continue
df = pd.DataFrame(list(qsday.values()))
data = RPS.caculateRPS(df)
del data['id']
# code = Listing.objects.get(code=v.code.code, category=11)
if len(data) > 0:
df, dfsaved = cls.dfNotInModel(data, v['tradedate'])
if len(df) == 0 and len(dfsaved) > 0:
# 当天所有数据都要更新
cls.getlist(stocktype).filter(
tradedate=v['tradedate']).delete()
cls.savedf(data, isBuckCreate=True)
continue
if len(df) > 0:
cls.savedf(df, isBuckCreate=True)
print('saved:{}'.format(len(df)))
if len(dfsaved) > 0:
# 日期在原来保存区间的数据
qssaved.append(dfsaved)
print('append to later save:{}'.format(len(dfsaved)))
else:
# quantaxis中无数据
delisted.append(v['tradedate'])
print('delisted count {} :\n {}'.format(len(delisted), delisted))
cls.updateSaved(qssaved)
print('保存过的数据更新数量 {} :\n {}'.format(len(qssaved), qssaved))
except Exception as e:
print(e.args)
return cls.getlist(stocktype)
