def fetchStockTradeInfoFromInternet(stockBaseModel, date = None): """从东方财富网抓取一只股票当天所有的成交信息""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) date = date + ' ' try: url = 'http://hqdigi2.eastmoney.com/EM_Quote2010NumericApplication/CompatiblePage.aspx?Type=OB&stk=%s&Reference=xml&limit=0&page=%s' page = 1 totalPages = 0 while True: response = urllib2.urlopen(urllib2.Request(url % (stockBaseModel.getEastMoneyStockCode(), page))).read() pattern = re.compile(r'var jsTimeSharingData=\{pages\:(\d+),data\:\[(.*)\]\};') matches = pattern.match(response) if matches and matches.group(2): totalPages = int(matches.group(1)) rows = matches.group(2)[1:len(matches.group(2))-1].split('","') for row in rows: if row: metas = row.split(',') stockTradeModel = StockTradeModel(None, stockBaseModel.id, date + metas[0], float(metas[1]), int(metas[2]), int(metas[3])) StockStorage.saveStockTradeModel(stockTradeModel) if page < totalPages: page = page + 1 else: break except urllib2.HTTPError, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchStockTradeInfoFromInternet(stockBaseModel, date = None): """从东方财富网抓取一只股票当天所有的成交信息""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) date = date + ' ' try: url = 'http://hqdigi2.eastmoney.com/EM_Quote2010NumericApplication/CompatiblePage.aspx?Type=OB&stk=%s&Reference=xml&limit=0&page=%s' page = 1 totalPages = 0 while True: response = urllib2.urlopen(urllib2.Request(url % (stockBaseModel.getEastMoneyStockCode(), page))).read() pattern = re.compile(r'var jsTimeSharingData=\{pages\:(\d+),data\:\[(.*)\]\};') matches = pattern.match(response) if matches and matches.group(2): totalPages = int(matches.group(1)) rows = matches.group(2)[1:len(matches.group(2))-1].split('","') tradeModels = [] for row in rows: if row: metas = row.split(',') stockTradeModel = StockTradeModel(None, stockBaseModel.id, date + metas[0], float(metas[1]), int(metas[2]), int(metas[3])) tradeModels.append(stockTradeModel) StockStorage.saveStockTradeModels(tradeModels) logger.info('stock %s daily trade model at page %d/%d fetched successfully' % (stockBaseModel.stockCode, page, totalPages)) if page < totalPages: page = page + 1 else: break except urllib2.HTTPError, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchStockCategoryRelationsFromInternet(stockBaseModel): """从东方财富网抓取一只股票所属的板块""" l = [] try: url = 'http://hqchart.eastmoney.com/hq20/js/%s.js' % stockBaseModel.stockCode response = urllib2.urlopen(urllib2.Request(url)).read() pattern = re.compile(r'var zjlx_bk=\[(.*)\];var zjlx_rank') matches = pattern.search(response) if matches: rows = matches.group(1)[1:len(matches.group(1))-1].split('],[') for row in rows: if row: categoryName = row.split(',')[0] categoryName = categoryName[1:len(categoryName)-1] l.append(categoryName) except Exception, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchAllStockCategoryModelsFromInternet(): """从东方财富网抓取所有股票板块信息""" try: categories = ['trade', 'notion', 'area'] for i in range(0,3): response = urllib2.urlopen(urllib2.Request('http://quote.eastmoney.com/hq2data/bk/data/%s.js' % categories[i])).read() pattern = re.compile(r'var BKCache = \{(Trade|Notion|Area)\:\[\[(.*)\]\]\};') matches = pattern.match(response) if matches: result = matches.group(2) rows = result[1:len(result)-1].split('","') for j in range(0, (len(rows)+1)/2): row = rows[j] l = row.split(',') stockCategoryModel = StockCategoryModel(None, l[0].decode('gbk').encode('utf-8'), i+1) StockStorage.saveStockCategoryModel(stockCategoryModel) except Exception, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchDailyStockModelFromInternet(stockBaseModel, date=None): """从东方财富网抓取一只股票当天的行情数据""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) try: url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode()) response = urllib2.urlopen(urllib2.Request(url)).read() pattern = re.compile(r'\(\["(.*)"\]\)') matches = pattern.match(response) if matches: stockModel = StockModel.stockModelFromList( matches.group(1).split(',')) stockModel.stockId = stockBaseModel.id stockModel.updateDate = date try: url2 = 'http://hqchart.eastmoney.com/hq20/js/%s.js?202224' % ( stockBaseModel.stockCode) response2 = urllib2.urlopen(urllib2.Request(url2)).read() pattern2 = re.compile( r'var zjlx_detail=\{data\:\"(.*)\",update.*') matches2 = pattern2.search(response2) if matches2: list = matches2.group(1).split(',') stockModel.superFlowIn = float(list[12]) stockModel.superFlowOut = float(list[13]) stockModel.bigFlowIn = float(list[14]) stockModel.bigFlowOut = float(list[15]) stockModel.middleFlowIn = float(list[16]) stockModel.middleFlowOut = float(list[17]) stockModel.littleFlowIn = float(list[18]) stockModel.littleFlowOut = float(list[19]) StockStorage.saveStockModel(stockModel) logger.info( 'stock %s daily model fetched successfully' % stockBaseModel.stockCode) except urllib2.HTTPError, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e)) except Exception, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchDailyStockModelFromInternet(stockBaseModel, date = None): """从东方财富网抓取一只股票当天的行情数据""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) try: url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode()) response = urllib2.urlopen(urllib2.Request(url)).read() pattern = re.compile(r'\(\["(.*)"\]\)') matches = pattern.match(response) if matches: stockModel = StockModel.stockModelFromList(matches.group(1).split(',')) stockModel.stockId = stockBaseModel.id stockModel.tradeDate = date try: url2 = 'http://nufm.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&cmd=%s&sty=DCFF&st=z&sr=&p=&ps=&cb=&js=var%%20zjlx_detail=(x)&token=7bc05d0d4c3c22ef9fca8c2a912d779c' % (stockBaseModel.getEastMoneyStockCode()) response2 = urllib2.urlopen(urllib2.Request(url2)).read() pattern2 = re.compile(r'var zjlx_detail="(.*)"') matches2 = pattern2.search(response2) if matches2: list = matches2.group(1).split(',') stockModel.superFlowIn = float(list[4]) stockModel.superFlowOut = float(list[5]) stockModel.bigFlowIn = float(list[6]) stockModel.bigFlowOut = float(list[7]) stockModel.middleFlowIn = float(list[8]) stockModel.middleFlowOut = float(list[9]) stockModel.littleFlowIn = float(list[10]) stockModel.littleFlowOut = float(list[11]) StockStorage.saveStockModel(stockModel) logger.info('stock %s daily model fetched successfully' % stockBaseModel.stockCode) except urllib2.HTTPError, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e)) except Exception, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
def fetchDailyStockModelFromInternet(stockBaseModel, date = None): """从东方财富网抓取一只股票当天的行情数据""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) try: url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode()) response = urllib2.urlopen(urllib2.Request(url)).read() pattern = re.compile(r'\(\["(.*)"\]\)') matches = pattern.match(response) if matches: stockModel = StockModel.stockModelFromList(matches.group(1).split(',')) stockModel.stockId = stockBaseModel.id stockModel.updateDate = date try: url2 = 'http://hqchart.eastmoney.com/hq20/js/%s.js?202224' % (stockBaseModel.stockCode) response2 = urllib2.urlopen(urllib2.Request(url2)).read() pattern2 = re.compile(r'var zjlx_detail=\{data\:\"(.*)\",update.*') matches2 = pattern2.search(response2) if matches2: list = matches2.group(1).split(',') stockModel.superFlowIn = float(list[12]) stockModel.superFlowOut = float(list[13]) stockModel.bigFlowIn = float(list[14]) stockModel.bigFlowOut = float(list[15]) stockModel.middleFlowIn = float(list[16]) stockModel.middleFlowOut = float(list[17]) stockModel.littleFlowIn = float(list[18]) stockModel.littleFlowOut = float(list[19]) StockStorage.saveStockModel(stockModel) logger.info('stock %s daily model fetched successfully' % stockBaseModel.stockCode) except urllib2.HTTPError, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e)) except Exception, e: logger.error('%s - %s' % (stockBaseModel.stockCode, e))
if matches: # 去年基金、证券等非股票代码 stockCode = macthes.group(3).decode('gbk') if stockCode.startswith('600') or stockCode.startswith('601') or stockCode.startswith('603') \ or stockCode.startswith('002') or stockCode.startswith('300') or stockCode.startswith('000'): stockBaseModel = StockBaseModel(None, matches.group(3), matches.group(2).decode('gbk').encode('utf-8'), matches.group(1)) try: StockStorage.saveStockBaseModel(stockBaseModel) logger.info('%s(%s) - new stockCode fetched' % (stockBaseModel.stockName, stockBaseModel.stockCode)) StockCrawlerService.fetchStockCategoryRelationsFromInternet(stockBaseModel) except Exception, e: pass finally: pass except urllib2.HTTPError, e: logger.error('%s' % (e)) except Exception, e: logger.error('%s' % (e)) finally: pass @staticmethod def fetchDailyStockModelFromInternet(stockBaseModel, date = None): """从东方财富网抓取一只股票当天的行情数据""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) try: url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode()) response = urllib2.urlopen(urllib2.Request(url)).read()
None, matches.group(3), matches.group(2).decode('gbk').encode('utf-8'), matches.group(1)) try: StockStorage.saveStockBaseModel(stockBaseModel) logger.info('%s(%s) - new stockCode fetched' % (stockBaseModel.stockName, stockBaseModel.stockCode)) StockCrawlerService.fetchStockCategoryRelationsFromInternet( stockBaseModel) except Exception, e: pass finally: pass except urllib2.HTTPError, e: logger.error('%s' % (e)) except Exception, e: logger.error('%s' % (e)) finally: pass @staticmethod def fetchDailyStockModelFromInternet(stockBaseModel, date=None): """从东方财富网抓取一只股票当天的行情数据""" if not date: date = time.strftime('%Y-%m-%d', time.localtime()) try: url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \ 'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode()) response = urllib2.urlopen(urllib2.Request(url)).read()