Exemplo n.º 1
0
	def fetchStockTradeInfoFromInternet(stockBaseModel, date = None):
		"""从东方财富网抓取一只股票当天所有的成交信息"""
		if not date:
			date = time.strftime('%Y-%m-%d', time.localtime())

		date = date + ' '

		try:
			url = 'http://hqdigi2.eastmoney.com/EM_Quote2010NumericApplication/CompatiblePage.aspx?Type=OB&stk=%s&Reference=xml&limit=0&page=%s'
			page = 1
			totalPages = 0
			while True:
				response = urllib2.urlopen(urllib2.Request(url % (stockBaseModel.getEastMoneyStockCode(), page))).read()
				pattern = re.compile(r'var jsTimeSharingData=\{pages\:(\d+),data\:\[(.*)\]\};')
				matches = pattern.match(response)

				if matches and matches.group(2):
					totalPages = int(matches.group(1))
					rows = matches.group(2)[1:len(matches.group(2))-1].split('","')
					tradeModels = []
					for row in rows:
						if row:
							metas = row.split(',')
							stockTradeModel = StockTradeModel(None, stockBaseModel.id, date + metas[0], float(metas[1]), int(metas[2]), int(metas[3]))
							tradeModels.append(stockTradeModel)

					StockStorage.saveStockTradeModels(tradeModels)

				logger.info('stock %s daily trade model at page %d/%d fetched successfully' % (stockBaseModel.stockCode, page, totalPages))
				if page < totalPages:
					page = page + 1
				else:
					break
		except urllib2.HTTPError, e:
			logger.error('%s - %s' % (stockBaseModel.stockCode, e))
Exemplo n.º 2
0
	def fetchAllStockCategoryRelationsFromInternet():
		"""从东方财富网抓取所有股票所属的板块"""
		dict = {}
		categories = StockService.getAllStockCategoryModels()
		for category in categories:
			dict[category.categoryName.encode('utf-8')] = category

		stocks = StockService.getAllStockBaseModels()
		for stock in stocks:
			l = StockCrawlerService.fetchStockCategoryRelationsFromInternet(stock)
			for s in l:
				category = dict.get(s)
				if category:
					StockStorage.saveStockCategoryRelation(stock, category)
					pass
				else:
					logger.info('%s no category fetched' % (stock.stockCode))
Exemplo n.º 3
0
	def fetchAllStockCategoryRelationsFromInternet():
		"""从东方财富网抓取所有股票所属的板块"""
		dict = {}
		categories = StockService.getAllStockCategoryModels()
		for category in categories:
			dict[category.categoryName.encode('utf-8')] = category

		stocks = StockService.getAllStockBaseModels()
		for stock in stocks:
			l = StockCrawlerService.fetchStockCategoryRelationsFromInternet(stock)
			for s in l:
				category = dict.get(s)
				if category:
					StockStorage.saveStockCategoryRelation(stock, category)
					pass
				else:
					logger.info('%s no category fetched' % (stock.stockCode))
Exemplo n.º 4
0
    def fetchDailyStockModelFromInternet(stockBaseModel, date=None):
        """从东方财富网抓取一只股票当天的行情数据"""
        if not date:
            date = time.strftime('%Y-%m-%d', time.localtime())

        try:
            url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \
             'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode())
            response = urllib2.urlopen(urllib2.Request(url)).read()
            pattern = re.compile(r'\(\["(.*)"\]\)')
            matches = pattern.match(response)
            if matches:
                stockModel = StockModel.stockModelFromList(
                    matches.group(1).split(','))
                stockModel.stockId = stockBaseModel.id
                stockModel.updateDate = date

                try:
                    url2 = 'http://hqchart.eastmoney.com/hq20/js/%s.js?202224' % (
                        stockBaseModel.stockCode)
                    response2 = urllib2.urlopen(urllib2.Request(url2)).read()
                    pattern2 = re.compile(
                        r'var zjlx_detail=\{data\:\"(.*)\",update.*')
                    matches2 = pattern2.search(response2)
                    if matches2:
                        list = matches2.group(1).split(',')
                        stockModel.superFlowIn = float(list[12])
                        stockModel.superFlowOut = float(list[13])
                        stockModel.bigFlowIn = float(list[14])
                        stockModel.bigFlowOut = float(list[15])
                        stockModel.middleFlowIn = float(list[16])
                        stockModel.middleFlowOut = float(list[17])
                        stockModel.littleFlowIn = float(list[18])
                        stockModel.littleFlowOut = float(list[19])

                        StockStorage.saveStockModel(stockModel)
                        logger.info(
                            'stock %s daily model fetched successfully' %
                            stockBaseModel.stockCode)
                except urllib2.HTTPError, e:
                    logger.error('%s - %s' % (stockBaseModel.stockCode, e))
                except Exception, e:
                    logger.error('%s - %s' % (stockBaseModel.stockCode, e))
Exemplo n.º 5
0
	def fetchDailyStockModelFromInternet(stockBaseModel, date = None):
		"""从东方财富网抓取一只股票当天的行情数据"""
		if not date:
			date = time.strftime('%Y-%m-%d', time.localtime())

		try:
			url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \
				'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode())
			response = urllib2.urlopen(urllib2.Request(url)).read()
			pattern = re.compile(r'\(\["(.*)"\]\)')
			matches = pattern.match(response)
			if matches:
				stockModel = StockModel.stockModelFromList(matches.group(1).split(','))
				stockModel.stockId = stockBaseModel.id
				stockModel.tradeDate = date

				try:
					url2 = 'http://nufm.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \
						'type=CT&cmd=%s&sty=DCFF&st=z&sr=&p=&ps=&cb=&js=var%%20zjlx_detail=(x)&token=7bc05d0d4c3c22ef9fca8c2a912d779c' % (stockBaseModel.getEastMoneyStockCode())
					response2 = urllib2.urlopen(urllib2.Request(url2)).read()
					pattern2 = re.compile(r'var zjlx_detail="(.*)"')
					matches2 = pattern2.search(response2)
					if matches2:
						list = matches2.group(1).split(',')
						stockModel.superFlowIn = float(list[4])
						stockModel.superFlowOut = float(list[5])
						stockModel.bigFlowIn = float(list[6])
						stockModel.bigFlowOut = float(list[7])
						stockModel.middleFlowIn = float(list[8])
						stockModel.middleFlowOut = float(list[9])
						stockModel.littleFlowIn = float(list[10])
						stockModel.littleFlowOut = float(list[11])

						StockStorage.saveStockModel(stockModel)
						logger.info('stock %s daily model fetched successfully' % stockBaseModel.stockCode)
				except urllib2.HTTPError, e:
					logger.error('%s - %s' % (stockBaseModel.stockCode, e))
				except Exception, e:
					logger.error('%s - %s' % (stockBaseModel.stockCode, e))
Exemplo n.º 6
0
	def fetchDailyStockModelFromInternet(stockBaseModel, date = None):
		"""从东方财富网抓取一只股票当天的行情数据"""
		if not date:
			date = time.strftime('%Y-%m-%d', time.localtime())

		try:
			url = 'http://nufm2.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx?' \
				'type=CT&sty=FDT&token=beb0a0047196124721f56b0f0ff5a27c&cmd=%s' % (stockBaseModel.getEastMoneyStockCode())
			response = urllib2.urlopen(urllib2.Request(url)).read()
			pattern = re.compile(r'\(\["(.*)"\]\)')
			matches = pattern.match(response)
			if matches:
				stockModel = StockModel.stockModelFromList(matches.group(1).split(','))
				stockModel.stockId = stockBaseModel.id
				stockModel.updateDate = date

				try:
					url2 = 'http://hqchart.eastmoney.com/hq20/js/%s.js?202224' % (stockBaseModel.stockCode)
					response2 = urllib2.urlopen(urllib2.Request(url2)).read()
					pattern2 = re.compile(r'var zjlx_detail=\{data\:\"(.*)\",update.*')
					matches2 = pattern2.search(response2)
					if matches2:
						list = matches2.group(1).split(',')
						stockModel.superFlowIn = float(list[12])
						stockModel.superFlowOut = float(list[13])
						stockModel.bigFlowIn = float(list[14])
						stockModel.bigFlowOut = float(list[15])
						stockModel.middleFlowIn = float(list[16])
						stockModel.middleFlowOut = float(list[17])
						stockModel.littleFlowIn = float(list[18])
						stockModel.littleFlowOut = float(list[19])

						StockStorage.saveStockModel(stockModel)
						logger.info('stock %s daily model fetched successfully' % stockBaseModel.stockCode)
				except urllib2.HTTPError, e:
					logger.error('%s - %s' % (stockBaseModel.stockCode, e))
				except Exception, e:
					logger.error('%s - %s' % (stockBaseModel.stockCode, e))
Exemplo n.º 7
0
	def fetchAllStockBaseModelsFromInternet():
		"""从东方财富网抓取所有股票代码、名字及所属市场"""
		try:
			response = urllib2.urlopen(urllib2.Request('http://quote.eastmoney.com/stocklist.html')).read()
			pattern = re.compile(r'(<li><a target="_blank" href="http://quote\.eastmoney\.com/.+\.html">.+\(\d{6}\)</a></li>)', re.MULTILINE)
			rows = pattern.findall(response)

			pattern = re.compile(r'<li><a target="_blank" href="http://quote\.eastmoney\.com/([a-zA-Z]{2,})\d+\.html">(.+)\((\d{6})\)</a></li>')
			for row in rows:
				matches = pattern.match(row)
				if matches:
					# 去年基金、证券等非股票代码
					stockCode = macthes.group(3).decode('gbk')
					if stockCode.startswith('600') or stockCode.startswith('601') or stockCode.startswith('603') \
									or stockCode.startswith('002') or stockCode.startswith('300') or stockCode.startswith('000'):
						stockBaseModel = StockBaseModel(None, matches.group(3), matches.group(2).decode('gbk').encode('utf-8'), matches.group(1))
						try:
							StockStorage.saveStockBaseModel(stockBaseModel)
							logger.info('%s(%s) - new stockCode fetched' % (stockBaseModel.stockName, stockBaseModel.stockCode))
							StockCrawlerService.fetchStockCategoryRelationsFromInternet(stockBaseModel)
						except Exception, e:
							pass
						finally:
							pass
Exemplo n.º 8
0
    def fetchAllStockBaseModelsFromInternet():
        """从东方财富网抓取所有股票代码、名字及所属市场"""
        try:
            response = urllib2.urlopen(
                urllib2.Request(
                    'http://quote.eastmoney.com/stocklist.html')).read()
            pattern = re.compile(
                r'(<li><a target="_blank" href="http://quote\.eastmoney\.com/.+\.html">.+\(\d{6}\)</a></li>)',
                re.MULTILINE)
            rows = pattern.findall(response)

            pattern = re.compile(
                r'<li><a target="_blank" href="http://quote\.eastmoney\.com/([a-zA-Z]{2,})\d+\.html">(.+)\((\d{6})\)</a></li>'
            )
            for row in rows:
                matches = pattern.match(row)
                if matches:
                    # 去年基金、证券等非股票代码
                    stockCode = macthes.group(3).decode('gbk')
                    if stockCode.startswith('600') or stockCode.startswith('601') or stockCode.startswith('603') \
                        or stockCode.startswith('002') or stockCode.startswith('300') or stockCode.startswith('000'):
                        stockBaseModel = StockBaseModel(
                            None, matches.group(3),
                            matches.group(2).decode('gbk').encode('utf-8'),
                            matches.group(1))
                        try:
                            StockStorage.saveStockBaseModel(stockBaseModel)
                            logger.info('%s(%s) - new stockCode fetched' %
                                        (stockBaseModel.stockName,
                                         stockBaseModel.stockCode))
                            StockCrawlerService.fetchStockCategoryRelationsFromInternet(
                                stockBaseModel)
                        except Exception, e:
                            pass
                        finally:
                            pass