class EventLoop: def __init__(self, timeout_in_second: float = None): self.event = QEventLoop(None) self.timeout_in_second = timeout_in_second def quit(self): self.event.quit() def __enter__(self): return self def __exit__(self, *_): if self.timeout_in_second is not None: t = Timer() t.timeout.connect(self.quit) t.start(seconds=self.timeout_in_second) self.event.exec_()
class CFFEXSpider(QObject): spider_finished = Signal(str, bool) def __init__(self, *args, **kwargs): super(CFFEXSpider, self).__init__(*args, **kwargs) self.date = None self.event_loop = QEventLoop(self) # 用于网络请求同步事件阻塞 def set_date(self, date): self.date = datetime.strptime(date, '%Y-%m-%d') def get_daily_source_file(self): """ 获取每日行情数据源文件保存至本地 """ if self.date is None: raise DateValueError("请先使用`set_date`设置`CZCESpider`日期.") url = "http://www.cffex.com.cn/sj/hqsj/rtj/{}/{}/{}_1.csv".format( self.date.strftime('%Y%m'), self.date.strftime('%d'), self.date.strftime('%Y%m%d')) app = QApplication.instance() network_manager = getattr(app, "_network") request = QNetworkRequest(url=url) request.setHeader(QNetworkRequest.UserAgentHeader, random.choice(USER_AGENTS)) reply = network_manager.get(request) reply.finished.connect(self.daily_source_file_reply) def daily_source_file_reply(self): """ 获取日统计数据请求返回 """ reply = self.sender() if reply.error(): reply.deleteLater() self.spider_finished.emit("失败:" + str(reply.error()), True) return save_path = os.path.join( LOCAL_SPIDER_SRC, 'cffex/daily/{}.csv'.format(self.date.strftime("%Y-%m-%d"))) file_data = reply.readAll() file_obj = QFile(save_path) is_open = file_obj.open(QFile.WriteOnly) if is_open: file_obj.write(file_data) file_obj.close() reply.deleteLater() self.spider_finished.emit( "获取中金所{}日交易数据源文件成功!".format(self.date.strftime("%Y-%m-%d")), True) def get_rank_source_file(self): """ 获取日排名数据源文件 """ base_url = "http://www.cffex.com.cn/sj/ccpm/{}/{}/{}_1.csv" app = QApplication.instance() network_manager = getattr(app, "_network") for variety in VARIETY_LIST: url = base_url.format(self.date.strftime("%Y%m"), self.date.strftime("%d"), variety) self.spider_finished.emit("准备获取{}的日排名数据文件...".format(variety), False) request = QNetworkRequest(url=url) request.setHeader(QNetworkRequest.UserAgentHeader, random.choice(USER_AGENTS)) reply = network_manager.get(request) reply.finished.connect(self.rank_source_file_reply) time.sleep(1) self.event_loop.exec_() def rank_source_file_reply(self): """ 获取日排名请求返回 """ reply = self.sender() request_url = reply.request().url().url() # 解析出请求的品种 request_filename = request_url.rsplit("/", 1)[1] request_variety = request_filename.split("_")[0] if reply.error(): reply.deleteLater() self.spider_finished.emit( "获取{}排名数据文件。\n失败:{}".format(request_variety[:2], str(reply.error())), True) logger.error("获取{}排名数据文件失败了!".format(request_url[:2])) return save_path = os.path.join( LOCAL_SPIDER_SRC, 'cffex/rank/{}_{}.csv'.format(request_variety, self.date.strftime("%Y-%m-%d"))) file_data = reply.readAll() file_obj = QFile(save_path) is_open = file_obj.open(QFile.WriteOnly) if is_open: file_obj.write(file_data) file_obj.close() reply.deleteLater() tip = "获取中金所{}_{}日持仓排名数据保存到文件成功!".format( request_variety, self.date.strftime("%Y-%m-%d")) if request_variety == "T": tip = "获取中金所{}日所有品种持仓排名数据保存到文件成功!".format( self.date.strftime("%Y-%m-%d")) self.spider_finished.emit(tip, True) self.event_loop.quit()
class Page(QWebEnginePage): def __init__(self, view): super(Page, self).__init__() self.parent = view.parent self.view = view self.result = None self.fullView = QWebEngineView() self.exitFSAction = QAction(self.fullView) self.loop = None def javaScriptConsoleMessage(self, level, msg, line, sourceid): """Override javaScriptConsoleMessage to use debug log.""" if level == QWebEnginePage.InfoMessageLevel: print("JS - INFO - Ligne {} : {}".format(line, msg)) elif level == QWebEnginePage.WarningMessageLevel: print("JS - WARNING - Ligne {} : {}".format(line, msg)) else: print("JS - ERROR - Ligne {} : {}".format(line, msg)) def hittestcontent(self, pos): return WebHitTestResult(self, pos) def maptoviewport(self, pos): return QPointF(pos.x(), pos.y()) def executejavascript(self, scriptsrc): self.loop = QEventLoop() self.result = None QTimer.singleShot(250, self.loop.quit) self.runJavaScript(scriptsrc, self.callbackjs) self.loop.exec_() self.loop = None return self.result def callbackjs(self, res): if self.loop is not None and self.loop.isRunning(): self.result = res self.loop.quit() def vsource(self): if "view-source:http" in self.url().toString(): self.load(QUrl(self.url().toString().split("view-source:")[1])) else: self.triggerAction(self.ViewSource) def makefullscreen(self, request): if request.toggleOn(): self.fullView = QWebEngineView() self.exitFSAction = QAction(self.fullView) self.exitFSAction.setShortcut(Qt.Key_Escape) self.exitFSAction.triggered.connect( lambda: self.triggerAction(self.ExitFullScreen)) self.fullView.addAction(self.exitFSAction) self.setView(self.fullView) self.fullView.showFullScreen() self.fullView.raise_() else: del self.fullView self.setView(self.view) request.accept()