def get_session(self, retry_times: int=0):
if retry_times > settings.RETRY_TIMES:
# tools.lock_
logger.error(
f'<{self.name}>: \n- '
f'访问次数 {settings.HOME_URL} 超过{settings.RETRY_TIMES}次，'
f'获取cookies失败！')
return False
else:
try:
# tools.lock_
logger.info(f'尝试访问 {settings.HOME_URL} 获得cookies')
# 由于settings.HOME_HEADERS和settings.SEARCH_HEADERS使用了fake_useragent，这里重新赋值以确保在子线程中固定值
home_headers = settings.HOME_HEADERS
self.headers = settings.SEARCH_HEADERS
self.session = requests.Session()
self.session.get(settings.HOME_URL, headers=home_headers)
return True
except Exception as e:
# tools.lock_
logger.warning(f'<{self.name}>: \n- '
f'尝试访问 {settings.HOME_URL} 获取cookies失败，将重试'
f'\n- 错误信息: {e}')
time.sleep(settings.REQUEST_GAP)
return self.get_session(retry_times + 1)
def conncet_inst(self):
GPIB_port = self.get_gpibport()
if GPIB_port != "":
self.inst = self.rm.open_resource(GPIB_port)
return True
else:
logger.warning("Keithley2400 is not connected!")
return False
def conncet_inst(self):
GPIB_port = self.get_gpibport()
if GPIB_port != "":
self.inst = self.rm.open_resource(GPIB_port)
return True
else:
logger.warning("Keithley2400 is not connected!")
return False
def BeginTxn(self, ctx):
'''Begin txn
@param ctx: Context
'''
if self.TxnValid():
logger.warning('rollback un-commited txn, startTS=%s',
self.txn.startTS)
return ErrTxnAlreadyExists
txn = self.store.Begin()
self.txn = txn # set by session
return None
def get_gpibport(self):
GPIB_port = ""
rm_list = self.rm.list_resources()
print rm_list
for io in rm_list:
if io.find('GPIB') != -1:
GPIB_port = io
if GPIB_port != "":
logger.info("Find GPIB port:%s" % GPIB_port)
else:
logger.warning("CAN NOT FOUND GPIB PORT!")
return GPIB_port
def get_gpibport(self):
GPIB_port = ""
rm_list = self.rm.list_resources()
print rm_list
for io in rm_list:
if io.find('GPIB') != -1:
GPIB_port = io
if GPIB_port != "":
logger.info("Find GPIB port:%s" % GPIB_port)
else:
logger.warning("CAN NOT FOUND GPIB PORT!")
return GPIB_port
def updateTbTask(self, taskid, state):
tasks = self.selectAllTasksById(taskid)
if tasks == None or len(tasks) == 0:
logger.warning("Warning, no task need to update status.")
return
for i in range(len(tasks)):
sql = "UPDATE tb_task SET task_status=%s WHERE task_uuid='%d'" % (
state, tasks[i].get('task_uuid'))
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
except:
logger.error("Error: unable to update call task status.")
return
def run(self):
# tools.lock_
logger.info(f'启动页面请求线程<{self.name}>')
# 获取会话并判断是否成功
if not self.get_session():
# tools.lock_
logger.error(f'<{self.name}>:\n- 获取会话失败，将退出线程')
else:
while True:
try:
urls_info = self.urls_queue.get()
# 有可能存在所有页面数据不够需求数量
if urls_info is None:
break
retry_times = urls_info['retry_times']
url = urls_info['url']
city = urls_info['city']
break_flag = self.request(url, city, retry_times)
# 全部收集完成，则退出
if break_flag == 1:
break
# 相应城市收集完，则跳过
if break_flag == 2:
continue
except Exception as e:
# tools.lock_
logger.warning(
f'<{self.name}>:'
f'\n- 请求页面 {url} 失败，后续将重试'
f'\n- 错误信息: {e}')
if retry_times <= settings.RETRY_TIMES:
self.urls_queue.put({
'url': url,
'city': city,
'retry_times': retry_times + 1})
finally:
self.urls_queue.task_done()
# 最终关闭会话
self.session.close()
# tools.lock_
logger.info(f'页面请求线程<{self.name}>结束')
def get_sys_state(self):
if Valve_Ctrl.get_pcb_state():
self.pcb_sta.setPixmap(QtGui.QPixmap(":/icon/icons/dlb.png"))
else:
QtGui.QMessageBox.warning(self, u'警告', u"电路板未连接!")
logger.warning("PCB isn't connected!")
self.pcb_sta.setPixmap(QtGui.QPixmap(":/icon/icons/nodlb.png"))
self.temprature = ai518p_api.get_now_temp()
if self.temprature != "":
self.now_T.setText(str(self.temprature))
self.ai518_sta.setPixmap(QtGui.QPixmap(":/icon/icons/wky.png"))
else:
QtGui.QMessageBox.warning(self, u'警告', u"温控仪未连接!")
logger.warning("AI518P isn't connected!")
self.ai518_sta.setPixmap(QtGui.QPixmap(":/icon/icons/nowky.png"))
instState = Keithley2400.conncet_inst()
if instState:
self.inst_sta.setPixmap(QtGui.QPixmap(":/icon/icons/yb.png"))
else:
QtGui.QMessageBox.warning(self, u'警告', u"仪表2400未连接!")
logger.warning("2400 isn't connected!")
self.inst_sta.setPixmap(QtGui.QPixmap(":/icon/icons/noyb.png"))
self.flow_port = read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'port')
self.flow1_addr = read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow1_addr')
self.flow1_range = int(read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow1_range'))
self.flow2_addr = read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow2_addr')
self.flow2_range = int(read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow2_range'))
self.flow3_addr = read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow3_addr')
self.flow3_range = int(read_config(qmdz_const.SYS_CONF_PATH, 'HMTS48', 'flow3_range'))
def cleanup(self):
'''
@type cs: ColumnStore
@return: ErrRetry if cleanup failed. else None
'''
logger.debug('[txn cleanup] startTS = %d, primary = %s, commitTS=%d' %
(self.startTS, self.us.primary, self.commitTS))
for cs in self.us.GetColumnStores():
if cs.primary is None:
logger.debug("no mutations")
continue
db = self.store.GetMvccDB(cs.col)
err = db.Rollback(cs.Keys(), self.startTS)
if err:
if not isinstance(err, ErrAlreadyCommitted):
logger.debug('cleanup err: %s', err.ERROR())
return ErrRetry
else:
logger.warning(
'cleanup txn already commited, cleanup cancel')
return err
return None
def request(self, url: str, city: str, retry_times: int):
if tools.check_db_position_count():
return 1
if tools.check_db_position_count(city):
return 2
if retry_times <= settings.RETRY_TIMES:
# tools.lock_
logger.info(f'<{self.name}>: \n- 开始下载 {url} ...')
content = self.session.get(
url=url,
headers=self.headers,
cookies=self.session.cookies,
# proxies=settings.PROXY
)
# 适当降低频率
time.sleep(settings.REQUEST_GAP)
if content.json()['state'] == 1:
# tools.lock_
logger.info(f'<{self.name}>: \n- 页面{url}下载完成！')
self.page_queue.put(content)
else:
# tools.lock_
logger.warning(
f'<{self.name}>: '
f'\n- 未获取到目标页面 {url}，正在更换cookies重试')
self.change_cookie()
self.urls_queue.put((0, {
'url': unquote_plus(content.request.url),
'city': city,
'retry_times': retry_times + 1}))
else:
# tools.lock_
logger.warning(
f'<{self.name}>:'
f'\n- 重复请求 {url} 超过{settings.RETRY_TIMES}次，'
f'放弃该页面')
return 0
from mylog import logger
logger.warning('I warn you.')
logger.info('Please be informed that this is just a test.')
logger.error('Found error on your smile.')
