def __init__(self, *args, **kargs):
super().__init__(*args, **kargs)
# lazy creation of the NetworkManager
global _manager
if _manager is None:
_manager = QNetworkAccessManager()
# open the destination file for writing
if self.dest and self.dest != '::mem::':
try:
self._dest_fp = open(self.dest, 'wb')
except OSError:
self._notify_done(False)
return
else:
self._dest_fp = None
# build and send the GET request
request = QNetworkRequest(self.url)
if self._headers is not None:
for key in self._headers:
request.setHeader(key, self._headers[key])
self._reply = _manager.get(request)
self._reply.finished.connect(self._finished_cb)
self._reply.downloadProgress.connect(self._progress_cb)
if self.dest != '::mem::':
self._reply.readyRead.connect(self._data_ready_cb)
def save_rank_server(self, source_df):
""" 保存日持仓排名到服务器 """
self.parser_finished.emit("开始保存上期所{}日持仓排名数据到服务器数据库...".format(self.date.strftime("%Y-%m-%d")), False)
data_body = source_df.to_dict(orient="records")
app = QApplication.instance()
network_manager = getattr(app, "_network")
url = SERVER + "exchange/shfe/rank/?date=" + self.date.strftime("%Y-%m-%d")
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json;charset=utf-8")
reply = network_manager.post(request, json.dumps(data_body).encode("utf-8"))
reply.finished.connect(self.save_rank_server_reply)
def get_rank_source_file(self):
""" 获取日排名数据 """
if self.date is None:
raise DateValueError("请先使用`set_date`设置`SHFESpider`日期.")
url = "http://www.shfe.com.cn/data/dailydata/kx/pm{}.dat".format(self.date.strftime('%Y%m%d'))
app = QApplication.instance()
network_manager = getattr(app, "_network")
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader, random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.rank_source_file_reply)
def commit_spot_data(self):
""" 提交数据 """
self.tip_label.setText("正在上传数据到服务器...")
app = QApplication.instance()
network_manager = getattr(app, "_network")
url = SERVER + "spot/price/?date=" + self.today_str
request = QNetworkRequest(QUrl(url))
request.setHeader(QNetworkRequest.ContentTypeHeader,
"application/json;charset=utf-8")
reply = network_manager.post(
request,
json.dumps(self.final_data).encode("utf-8"))
reply.finished.connect(self.save_spot_price_reply)
def get_daily_source_file(self):
""" 获取每日行情数据源文件保存至本地 """
if self.date is None:
raise DateValueError("请先使用`set_date`设置`CZCESpider`日期.")
url = "http://www.cffex.com.cn/sj/hqsj/rtj/{}/{}/{}_1.csv".format(
self.date.strftime('%Y%m'), self.date.strftime('%d'),
self.date.strftime('%Y%m%d'))
app = QApplication.instance()
network_manager = getattr(app, "_network")
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader,
random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.daily_source_file_reply)
def get_rank_source_file(self):
""" 获取日持仓排名数据源文件保存至本地 """
if self.date is None:
raise DateValueError("请先使用`set_date`设置`CZCESpider`日期.")
url = "http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataHolding.xls".format(
self.date.year, self.date.strftime('%Y%m%d'))
app = QApplication.instance()
network_manager = getattr(app, "_network")
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader,
random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.rank_source_file_reply)
def get_rank_source_file(self):
""" 获取日排名数据源文件 """
base_url = "http://www.cffex.com.cn/sj/ccpm/{}/{}/{}_1.csv"
app = QApplication.instance()
network_manager = getattr(app, "_network")
for variety in VARIETY_LIST:
url = base_url.format(self.date.strftime("%Y%m"),
self.date.strftime("%d"), variety)
self.spider_finished.emit("准备获取{}的日排名数据文件...".format(variety),
False)
request = QNetworkRequest(url=url)
request.setHeader(QNetworkRequest.UserAgentHeader,
random.choice(USER_AGENTS))
reply = network_manager.get(request)
reply.finished.connect(self.rank_source_file_reply)
time.sleep(1)
self.event_loop.exec_()
def modify_row_data(self):
""" 修改数据表的单元格点击 """
btn = self.sender()
row = getattr(btn, 'row_index')
# 获取组织数据
item = {
"id": int(self.modify_table.item(row, 0).text()),
"date": self.modify_table.item(row, 1).text(),
"variety_en": self.modify_table.item(row, 2).text(),
"spot_price": float(self.modify_table.item(row, 3).text()),
"price_increase": float(self.modify_table.item(row, 4).text())
}
app = QApplication.instance()
network_manager = getattr(app, "_network")
url = SERVER + "spot/price/{}/".format(item["id"])
request = QNetworkRequest(QUrl(url))
request.setHeader(QNetworkRequest.ContentTypeHeader,
"application/json;charset=utf-8")
reply = network_manager.put(request, json.dumps(item).encode("utf-8"))
reply.finished.connect(self.modify_spot_price_reply)
