def test_trade_date_sse(self): qs = Stocktradedate.importList() day = datetime.datetime.now() sd = Stocktradedate() self.assertTrue( qs.filter(tradedate__lte=day).count() == sd.trade_date_sse.count(), '2018年之前的交易日共:{}'.format(qs.filter(tradedate__lte=day).count()))
def getNearestTradedate(date=datetime.datetime.now().date(), days=0): """ 获取离date最近的交易日期 只能获取到前一交易日的数据qa.QA_fetch_index_day_adv :param date: :return: """ from stocks.models import Stocktradedate if not date: # 如果date为空,则赋值当天 date = datetime.datetime.now().date() date = convertToDate(date) tradedate = Stocktradedate.get_real_date(date, -1).date() if date == tradedate: if date == datetime.datetime.now().date(): # 当天并且是交易日 tradedate = Stocktradedate.preTradeday(tradedate) if days == 0: return tradedate elif days < 0: for day in range(abs(days)): tradedate = Stocktradedate.preTradeday(tradedate) else: for day in range(days): tradedate = Stocktradedate.nextTradeday(tradedate) return tradedate
def test_everydayCount(self): Stocktradedate.importList() for d in [ d[0] for d in set( list(HSGTCGHold.getlist().values_list('tradedate'))) ]: dcount = HSGTCGHold.getlist(d).count() print(d, dcount) from stocks.models import HSGTCG tdate = HSGTCGHold.getNearestTradedate() while tdate > convertToDate('2018-5-1'): dcount = HSGTCGHold.getlist(tdate).count() dcount1 = HSGTCG.getlist().filter(tradedate=tdate, hamount__gte=8000).count() print(tdate, dcount, dcount1, dcount - dcount1) tdate = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1)) from stocks.models import HSGTCG tdate = HSGTCGHold.getNearestTradedate() while tdate > convertToDate('2018-5-1'): df1 = pd.DataFrame(list(HSGTCGHold.getlist(tdate).values('code'))) dcount = len(df1) df2 = pd.DataFrame( list(HSGTCG.getlist().filter( tradedate=tdate, hamount__gte=8000).values('code'))) dcount1 = len(df2) print(tdate, dcount, dcount1, dcount - dcount1) if dcount - dcount1 != 0 and tdate > convertToDate('2018-6-1'): print('数据不一致:', end='') print(list(HSGTCGHold.dfNotInAnotherdf(df1, df2)['code'])) tdate = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1))
def test_newcomingin(self): """ 新加入市值八千万的个股 :return: """ Stocktradedate.importList() from stocks.models import HSGTCG # 2018 - 06 - 04 新增北向持股金额大于八千万 list1 = [ '603658', '600460', '002812', '002557', '600188', '000690', '600329' ] tdate = HSGTCGHold.getNearestTradedate() # tdate = HSGTCGHold.getNearestTradedate('2018-6-4') tdate1 = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1)) hsg = HSGTCGHold.getlist(tdate) hsg1 = HSGTCGHold.getlist(tdate1) list2 = [] for c in hsg.exclude(code__in=hsg1.values_list('code')): list2.append(c.code) # 验证是否前一天市值小于八千万 for code in list2: df = pd.DataFrame( list( HSGTCG.getlist(code).filter(tradedate__gte=tdate1).values( ).order_by('-tradedate'))) data1 = float(df.iloc[-2].hamount) data2 = float(df.iloc[-1].hamount) if not (data1 >= 8000 and data2 < 8000): print('不是新增持股金额大于八千万:{} 持股金额:{} {}'.format(code, data1, data2)) tdate = HSGTCGHold.getNearestTradedate() while tdate > convertToDate('2018-5-2'): tdate1 = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1)) hsg = HSGTCGHold.getlist(tdate) hsg1 = HSGTCGHold.getlist(tdate1) list2 = [] for c in hsg.exclude(code__in=hsg1.values_list('code')): list2.append(c.code) # 验证是否前一天市值小于八千万 for code in list2: df = pd.DataFrame( list( HSGTCG.getlist(code).filter(tradedate__gte=tdate1). values().order_by('-tradedate'))) if len(df) > 1 and HSGTCG.getlist().filter( tradedate=tdate1).count() > 0: # if len(df)> 1 : data1 = float(df.iloc[-2].hamount) data2 = float(df.iloc[-1].hamount) if not (data1 >= 8000 and data2 < 8000): print('{} 不是新增持股金额大于八千万:{} 持股金额:{} {}'.format( tdate, code, data1, data2)) tdate = HSGTCGHold.getNearestTradedate(tdate - datetime.timedelta(1))
def test_getRecentlist(self): Stocktradedate.importList() n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate'))) self.assertTrue(len(n) <= 1, '交易日:{}'.format(n)) days = 3 n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate'))) self.assertTrue(len(n) <= days, '交易日:{}'.format(n)) days = 10 n = set(list(HSGTCGHold.getRecentlist().values_list('tradedate'))) self.assertTrue(len(n) <= days, '交易日:{}'.format(n))
def test_get_real_date(self): qs = Stocktradedate.importList() date = '2018-5-21' day = datetime.datetime.strptime(date, '%Y-%m-%d') self.assertTrue( Stocktradedate.get_real_date(day) == day, '({}) != ({})'.format(Stocktradedate.get_real_date(day), day)) date = '2018-5-20' day = datetime.datetime.strptime(date, '%Y-%m-%d') self.assertFalse( Stocktradedate.get_real_date(day) == day, '({}) != ({})'.format(Stocktradedate.get_real_date(day), day))
def importWebdriverList(cls, firefoxHeadless=False): """ 导入市值大于指定值的列表 网址: http://data.eastmoney.com/hsgtcg/StockStatistics.aspx 不建议用此方法,经常丢失数据 :param firefoxHeadless: 是否显示浏览器界面: True 不显示界面 False 显示界面 默认不显示浏览器界面 :return: 最近交易日期的列表 """ hsgh = HSGTCGHold.getlist(tradedate=cls.getNearestTradedate()) if hsgh.count() > 0: return hsgh browser = cls.getBrowser(firefoxHeadless) url = 'http://data.eastmoney.com/hsgtcg/StockStatistics.aspx' df = cls.scrap(url, browser) df = df[['code', 'tradedate']] df = df[df['tradedate'].apply(lambda x: Stocktradedate.if_tradeday(x))] # 去除重复数据 df = df[~df.duplicated()] cls.savedf(df[['code', 'tradedate']]) # pandas dataframe save to model # HSGTCGHold.objects.bulk_create( # HSGTCGHold(**vals) for vals in df[['code', 'tradedate']].to_dict('records') # ) return cls.getlist(tradedate=cls.getNearestTradedate())
def getStockHdStatistics(cls, code, browser, retryCount=3): """ 抓取持股统计 :param code: 股票代码 :param browser: webdriver浏览器 :return: """ url = 'http://data.eastmoney.com/hsgtcg/StockHdStatistics.aspx?stock={}'.format( code) for i in range(retryCount): df = cls.scrap(url, browser) if len(df) > 0: # 修复持股数量 df['hvol'] = df['hvol'].apply( lambda x: HSGTCG.hz2Num(x)).astype(float) df['hamount'] = df['hamount'].apply( lambda x: HSGTCG.hz2Num(x)).astype(float) df['close'] = df['close'].astype(float) df['tradedate'] = df['tradedate'].apply( lambda x: convertToDate(x)).astype(datetime.date) df = df[df['tradedate'].apply( lambda x: Stocktradedate.if_tradeday( x))] # 删除不是交易日的数据。这是东方财富网页版的bug df.index = pd.RangeIndex(len(df.index)) break else: pass return df
def test_get_real_datelist(self): qs = Stocktradedate.importList() day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日 day2 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日 start, end = Stocktradedate.get_real_date_start_end(day1, day2) self.assertIsNone(start) day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日 day2 = datetime.datetime.strptime('2018-5-15', '%Y-%m-%d').date() # 周日 start, end = Stocktradedate.get_real_date_start_end(day1, day2) self.assertIsInstance( start, datetime.date, '返回值为datetime.date类型, 实际返回{}'.format(type(start))) day3 = datetime.datetime.strptime('2018-5-14', '%Y-%m-%d').date() # 周一 self.assertTrue(start == day3, '周日应该返回下一工作日:{}, 实际返回:{}'.format(day3, start))
def test_nextTradeday(self): qs = Stocktradedate.importList() day1 = datetime.datetime.strptime('2018-5-13', '%Y-%m-%d').date() # 周日 day = Stocktradedate.nextTradeday(day1) self.assertTrue( 'wrong date' == day, '非交易日 {} !={}'.format(day1 + datetime.timedelta(1), day)) day1 = datetime.datetime.strptime('2018-5-14', '%Y-%m-%d').date() # 周一 day = Stocktradedate.nextTradeday(day1) self.assertTrue(day1 + datetime.timedelta(1) == day, '{} !={}'.format(day1 + datetime.timedelta(1), day)) n = 5 day = Stocktradedate.nextTradeday(day1, n) # # print(type('day type: {}, date type:{}'.format(day, day1))) self.assertTrue(day1 + datetime.timedelta(n) < day, '{} !={}'.format(day1 + datetime.timedelta(n), day))
def importjsonList(cls, enddate=None, days=5): """ 导入市值大于指定值的列表 网址: http://data.eastmoney.com/hsgtcg/StockStatistics.aspx 直接下载json文件转换,效率比importList高 :param firefoxHeadless: 是否显示浏览器界面: True 不显示界面 False 显示界面 默认不显示浏览器界面 :return: 最近交易日期的列表jsonname """ if enddate: end = convertToDate(enddate) else: end = cls.getNearestTradedate() hsgh = cls.getlist(tradedate=end) if hsgh.count() > 0: return hsgh pagesize = 300 # 每页数据量 page = 1 sortRule = -1 # sortRule-1 按照市值降序排序; 1 按照市值升序排序 start = end - datetime.timedelta(days) jsname = cls.getRandomStr('letter') for page in range(1, 100): if page > 1: sr = -1 # st=SHAREHOLDPRICE 按照持股市值排序; st sortType url = 'http://dcfm.eastmoney.com//em_mutisvcexpandinterface/api/js/' \ 'get?type=HSGTHDSTA&token=70f12f2f4f091e459a279469fe49eca5&st=SHAREHOLDPRICE&sr={sortRule}' \ '&p={page}&ps={pagesize}&js=var%20{jsname}={pages:(tp),data:(x)}&filter=(MARKET%20in%20(%27001%27,%27003%27))' \ '(HDDATE%3E=^{start}^%20and%20HDDATE%3C=^{end}^)&rt=50950960' \ .replace('{start}', str(start)).replace('{end}', str(end)) \ .replace('{sortRule}', str(sortRule)) \ .replace('{pagesize}', str(pagesize)) \ .replace('{page}', str(page)) \ .replace('{jsname}', str(jsname)) df = cls.scrapjson(url) dfn = df[df['hamount'] >= MINHAMOUNT] if len(dfn) > 0: dfn = dfn[dfn['tradedate'].apply( lambda x: Stocktradedate.if_tradeday(x))] # 去除重复数据 dfn = dfn[~dfn.duplicated()] cls.savedf(dfn[['code', 'tradedate']]) print('page: {}'.format(page)) if len(df[df['hamount'] < MINHAMOUNT]): # 持股金额小于 break return cls.getlist(tradedate=cls.getNearestTradedate())
def test_caculateRPS(self): # Listing.importIndexListing() # rpspreparelist = RPSprepare.importIndexListing() qs = RPSprepare.getlist('index') assert qs.count() > 0, "RPSprepare.getlist('index') 返回个数应大于零" start, _ = Stocktradedate.get_real_datelist('2018-1-1', '2018-2-1') # qs = RPSprepare.getlist('index').filter(tradedate=convertToDate('2018-1-4')) qs = RPSprepare.getlist('index').filter(tradedate=start) df = pd.DataFrame(list(qs.values())) # self.assertTrue(len(df) > 0, "RPSprepare.getlist('index') 返回个数应大于零, {}".format(len(df))) data = RPS.caculateRPS(df) self.assertTrue( len(data) / len(data[data['rps120'] == data['rps250']]) > 2, 'rps120等于rps250的几率很小'.format( len(data), len(data[data['rps120'] == data['rps250']])))
def test_importList(self): qs = Stocktradedate.importList() day = datetime.datetime.strptime('1991-12-31', '%Y-%m-%d') self.assertTrue( qs.filter(tradedate__lte=day).count() == 264, '1991年之前的交易日共:{}'.format(qs.filter(tradedate__lte=day).count())) day = datetime.datetime.strptime('2017-12-31', '%Y-%m-%d') self.assertTrue( qs.filter(tradedate__lte=day).count() == 6613, '2018年之前的交易日共:{}'.format(qs.filter(tradedate__lte=day).count())) day = datetime.datetime.strptime('2018-3-31', '%Y-%m-%d') self.assertTrue( qs.filter(tradedate__lte=day).count() > 6613, '2018年之前的交易日共:{}'.format(qs.filter(tradedate__lte=day).count()))
def getRecentlist(cls, days=1): """ 返回最近days天列表 :param days: 之前几个交易 :return: .objects.all().filter(tradedate__gte=tdate) """ if days == 0: return cls.objects.all() else: tdate = cls.getNearestTradedate() if days > 1: realdatelist = Stocktradedate.get_real_datelisting( tdate - datetime.timedelta(days * 2 + 10), tdate) tdate = pd.DataFrame( list(realdatelist.order_by('id').values_list( 'tradedate'))).iloc[-days][0] # 返回所有代码 return cls.objects.all().filter(tradedate__gte=tdate)
def importIndexListing(cls, start='2006-1-1', end=None): """ 插入所有股票指数 :return: """ if end is None: # 获取当天 end = datetime.datetime.now().date() qs = RPSprepare.getlist('index') # todo 如果已经插入,则判断是否有更新 try: # 批量创建对象,减少SQL查询次数 querysetlist = [] delisted = [] # quantaxis中无数据时,保存到delisted for v in Stocktradedate.get_real_datelisting(start, end).values('tradedate'): print(v) # 获取 v['tradedate']对应的指数列表 qsday = qs.filter(tradedate=v['tradedate']) df = pd.DataFrame(list(qsday.values())) data = RPS.caculateRPS(df) aaa # code = Listing.objects.get(code=v.code.code, category=11) if len(data) > 0: df = pd.DataFrame(data.close) df['rps120'] = df.close / df.close.shift(120) df['rps250'] = df.close / df.close.shift(250) del df['close'] df = df[120:] for a, b in df.values: querysetlist.append( RPS(code=code, rps120=b, rps250=b)) else: # quantaxis中无数据 delisted.append(a) print('delisted count {} :\n {}'.format(len(delisted), delisted)) RPSprepare.objects.bulk_create(querysetlist) except Exception as e: print(e.args) return cls.getlist('index')
def setUpTestData(cls): Stocktradedate.importList()
def setUpTestData(cls): Stocktradedate.importList() Listing.importIndexListing() rpspreparelist = RPSprepare.importIndexListing()
def test_deleteNotTradedate(self): # 删除不是交易日的数据 for d in set(list(HSGTCG.objects.all().values_list('tradedate'))): if not Stocktradedate.if_tradeday(d[0]): HSGTCG.objects.all().filter(tradedate=d[0]).delete() print(d[0])
def test_if_trade(self): qs = Stocktradedate.importList() day = datetime.datetime.strptime('2018-5-21', '%Y-%m-%d') self.assertTrue(Stocktradedate.if_tradeday(day), '{} 是交易日'.format(day))
def setUp(self): Stocktradedate.importList()
def importStockListing(cls, start=None, end=None, ignoreSaved=True): """ 插入所有股票RPS :param start: 计算的开始日期 :param end: :param ignoreSaved: Boolean 是否重新计算已保存的RPS :return: """ """ :return: """ stocktype = 'stock' if start is None: # 数据库中最大的已计算日期 latest = cls.getlist('stock').aggregate( Max('tradedate'))['tradedate__max'] if latest: start = cls.getNearestTradedate(latest, -5) else: start = '2014-1-1' if end is None: # 获取当天 end = datetime.datetime.now().date() qs = RPSprepare.getlist(stocktype) try: # 批量创建对象,减少SQL查询次数 querysetlist = [] delisted = [] # quantaxis中无数据时,保存到delisted qssaved = [] for v in Stocktradedate.get_real_datelisting( start, end).values('tradedate'): print('dealing ... {}'.format(v['tradedate'])) # 获取 v['tradedate']对应的股票RPS预计算列表 qsday = qs.filter(tradedate=v['tradedate']) if ignoreSaved: if qsday.count() == cls.getlist(stocktype).filter( tradedate=v['tradedate']).count(): # RPS预计算数量和已保存的RPS数量相同,则跳过 # print('pass') continue df = pd.DataFrame(list(qsday.values())) data = RPS.caculateRPS(df) del data['id'] # code = Listing.objects.get(code=v.code.code, category=11) if len(data) > 0: df, dfsaved = cls.dfNotInModel(data, v['tradedate']) if len(df) == 0 and len(dfsaved) > 0: # 当天所有数据都要更新 cls.getlist(stocktype).filter( tradedate=v['tradedate']).delete() cls.savedf(data, isBuckCreate=True) continue if len(df) > 0: cls.savedf(df, isBuckCreate=True) print('saved:{}'.format(len(df))) if len(dfsaved) > 0: # 日期在原来保存区间的数据 qssaved.append(dfsaved) print('append to later save:{}'.format(len(dfsaved))) else: # quantaxis中无数据 delisted.append(v['tradedate']) print('delisted count {} :\n {}'.format(len(delisted), delisted)) cls.updateSaved(qssaved) print('保存过的数据更新数量 {} :\n {}'.format(len(qssaved), qssaved)) except Exception as e: print(e.args) return cls.getlist(stocktype)