class EventLoop:
def __init__(self, timeout_in_second: float = None):
self.event = QEventLoop(None)
self.timeout_in_second = timeout_in_second
def quit(self):
self.event.quit()
def __enter__(self):
return self
def __exit__(self, *_):
if self.timeout_in_second is not None:
t = Timer()
t.timeout.connect(self.quit)
t.start(seconds=self.timeout_in_second)
self.event.exec_()
class CFFEXSpider(QObject):
spider_finished = Signal(str, bool)
def __init__(self, *args, **kwargs):
super(CFFEXSpider, self).__init__(*args, **kwargs)
self.date = None
self.event_loop = QEventLoop(self) # 用于网络请求同步事件阻塞
def set_date(self, date):
self.date = datetime.strptime(date, '%Y-%m-%d')
def get_daily_source_file(self):
""" 获取每日行情数据源文件保存至本地 """
if self.date is None:
raise DateValueError("请先使用`set_date`设置`CZCESpider`日期.")
url = "http://www.cffex.com.cn/sj/hqsj/rtj/{}/{}/{}_1.csv".format(
self.date.strftime('%Y%m'), self.date.strftime('%d'),
self.date.strftime('%Y%m%d'))
app = QApplication.instance()
network_manager = getattr(app, "_network")
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader,
random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.daily_source_file_reply)
def daily_source_file_reply(self):
""" 获取日统计数据请求返回 """
reply = self.sender()
if reply.error():
reply.deleteLater()
self.spider_finished.emit("失败:" + str(reply.error()), True)
return
save_path = os.path.join(
LOCAL_SPIDER_SRC,
'cffex/daily/{}.csv'.format(self.date.strftime("%Y-%m-%d")))
file_data = reply.readAll()
file_obj = QFile(save_path)
is_open = file_obj.open(QFile.WriteOnly)
if is_open:
file_obj.write(file_data)
file_obj.close()
reply.deleteLater()
self.spider_finished.emit(
"获取中金所{}日交易数据源文件成功!".format(self.date.strftime("%Y-%m-%d")), True)
def get_rank_source_file(self):
""" 获取日排名数据源文件 """
base_url = "http://www.cffex.com.cn/sj/ccpm/{}/{}/{}_1.csv"
app = QApplication.instance()
network_manager = getattr(app, "_network")
for variety in VARIETY_LIST:
url = base_url.format(self.date.strftime("%Y%m"),
self.date.strftime("%d"), variety)
self.spider_finished.emit("准备获取{}的日排名数据文件...".format(variety),
False)
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader,
random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.rank_source_file_reply)
time.sleep(1)
self.event_loop.exec_()
def rank_source_file_reply(self):
""" 获取日排名请求返回 """
reply = self.sender()
request_url = reply.request().url().url()
# 解析出请求的品种
request_filename = request_url.rsplit("/", 1)[1]
request_variety = request_filename.split("_")[0]
if reply.error():
reply.deleteLater()
self.spider_finished.emit(
"获取{}排名数据文件。\n失败:{}".format(request_variety[:2],
str(reply.error())), True)
logger.error("获取{}排名数据文件失败了!".format(request_url[:2]))
return
save_path = os.path.join(
LOCAL_SPIDER_SRC,
'cffex/rank/{}_{}.csv'.format(request_variety,
self.date.strftime("%Y-%m-%d")))
file_data = reply.readAll()
file_obj = QFile(save_path)
is_open = file_obj.open(QFile.WriteOnly)
if is_open:
file_obj.write(file_data)
file_obj.close()
reply.deleteLater()
tip = "获取中金所{}_{}日持仓排名数据保存到文件成功!".format(
request_variety, self.date.strftime("%Y-%m-%d"))
if request_variety == "T":
tip = "获取中金所{}日所有品种持仓排名数据保存到文件成功!".format(
self.date.strftime("%Y-%m-%d"))
self.spider_finished.emit(tip, True)
self.event_loop.quit()
class Page(QWebEnginePage):
def __init__(self, view):
super(Page, self).__init__()
self.parent = view.parent
self.view = view
self.result = None
self.fullView = QWebEngineView()
self.exitFSAction = QAction(self.fullView)
self.loop = None
def javaScriptConsoleMessage(self, level, msg, line, sourceid):
"""Override javaScriptConsoleMessage to use debug log."""
if level == QWebEnginePage.InfoMessageLevel:
print("JS - INFO - Ligne {} : {}".format(line, msg))
elif level == QWebEnginePage.WarningMessageLevel:
print("JS - WARNING - Ligne {} : {}".format(line, msg))
else:
print("JS - ERROR - Ligne {} : {}".format(line, msg))
def hittestcontent(self, pos):
return WebHitTestResult(self, pos)
def maptoviewport(self, pos):
return QPointF(pos.x(), pos.y())
def executejavascript(self, scriptsrc):
self.loop = QEventLoop()
self.result = None
QTimer.singleShot(250, self.loop.quit)
self.runJavaScript(scriptsrc, self.callbackjs)
self.loop.exec_()
self.loop = None
return self.result
def callbackjs(self, res):
if self.loop is not None and self.loop.isRunning():
self.result = res
self.loop.quit()
def vsource(self):
if "view-source:http" in self.url().toString():
self.load(QUrl(self.url().toString().split("view-source:")[1]))
else:
self.triggerAction(self.ViewSource)
def makefullscreen(self, request):
if request.toggleOn():
self.fullView = QWebEngineView()
self.exitFSAction = QAction(self.fullView)
self.exitFSAction.setShortcut(Qt.Key_Escape)
self.exitFSAction.triggered.connect(
lambda: self.triggerAction(self.ExitFullScreen))
self.fullView.addAction(self.exitFSAction)
self.setView(self.fullView)
self.fullView.showFullScreen()
self.fullView.raise_()
else:
del self.fullView
self.setView(self.view)
request.accept()
