def get_session(self, retry_times: int=0):
if retry_times > settings.RETRY_TIMES:
# tools.lock_
logger.error(
f'<{self.name}>: \n- '
f'访问次数 {settings.HOME_URL} 超过{settings.RETRY_TIMES}次，'
f'获取cookies失败！')
return False
else:
try:
# tools.lock_
logger.info(f'尝试访问 {settings.HOME_URL} 获得cookies')
# 由于settings.HOME_HEADERS和settings.SEARCH_HEADERS使用了fake_useragent，这里重新赋值以确保在子线程中固定值
home_headers = settings.HOME_HEADERS
self.headers = settings.SEARCH_HEADERS
self.session = requests.Session()
self.session.get(settings.HOME_URL, headers=home_headers)
return True
except Exception as e:
# tools.lock_
logger.warning(f'<{self.name}>: \n- '
f'尝试访问 {settings.HOME_URL} 获取cookies失败，将重试'
f'\n- 错误信息: {e}')
time.sleep(settings.REQUEST_GAP)
return self.get_session(retry_times + 1)
def comm_connect(self):
try:
self.ser = serial.Serial(self.com_port, self.baud, timeout=1)
except:
logger.error("can not find port:%s" % self.com_port)
self.ser = None
return self.ser
def comm_connect(self):
try:
self.ser = serial.Serial(self.com_port, self.baud, timeout=1)
except:
logger.error("can not find port:%s" % self.com_port)
self.ser = None
return self.ser
def run(self):
# tools.lock_
logger.info(f'启动数据库写入线程<{self.name}>')
while True:
try:
position_info = self.data_queue.get()
if position_info is None:
break
# 加锁，防止在判断到计数之间发生中断导致最终判断错误
with tools.db_count_lock:
break_flag = self.upsert(position_info)
# 若所有数据已满
if break_flag == 1:
break
# 如果某一个区域的目标数量已满足则跳过该条数据
if break_flag == 2:
continue
except Exception as e:
logger.error(f'<{self.name}>:\n'
f'- 插入记录{position_info}到数据库失败\n'
f'- 错误信息: {e}')
finally:
self.data_queue.task_done()
# tools.lock_
logger.info(f'结束数据库写入线程<{self.name}>')
def mainTask(self):
cpt = cpTime.calcProhibitedTime()
executor = ThreadPoolExecutor()
isFirst = True
queuedTasks = []
threadedTasks = []
count = 0
tasksInDb = self.dbConn.selectDialTask()
while (True):
if not isFirst:
tasksInDb = self.dbConn.selectDialTask()
else:
isFirst = False
logger.info("tasksInDb: %s", tasksInDb)
logger.info("queuedTasks: %s", queuedTasks)
logger.info("threadedTasks: %s", threadedTasks)
for task in tasksInDb:
if not self.isInQueuedTasks(task, queuedTasks):
queuedTasks.append(task)
if not self.isTaskTheaded(task, threadedTasks):
taskRec = self.dbConn.selectDialTaskById(
task.get('task_id'))
logger.debug("taskRec to thread: %s" % (taskRec))
if len(taskRec) == 0:
continue
prohibitSlot = taskRec.get('task_timegroup')
if cpt.isTimeSlotProhibit(prohibitSlot):
logger.info("Currently it's prohibited to dial for task %s, don't init dialThread."%\
(task.get('task_id')))
logger.info("Ths task's prohibitSlot is %s" %
(prohibitSlot))
continue
else:
logger.debug("to submit thread: %s" %
(task.get('task_id')))
executor.submit(self.handleTask, taskRec, prohibitSlot)
threadedTasks.append(task)
count = count + 1
if count == 30:
count = 0
records = self.dbConn.selectLatestTime()
for record in records:
taskid = record.get("task_id")
try:
logger.info("任务:%s,已经20分钟没有呼出成功了,先暂停了" % (taskid))
self.hc.sendMsgTimeOut(taskid)
self.dbConn.updateTbTask(taskid, 2)
except Exception as e:
logger.error("Error: sendmsg error: %s." % (e))
logger.info("wait %s seconds to take new task. count:%d\n",
self.dialInterval, count)
self.removeFinishedTasks(threadedTasks, queuedTasks)
time.sleep(self.dialInterval)
def set_valve_clen(self):
try:
self.comm_connect()
self.S48.write_bit(2, 1, 5)
self.comm_close()
except:
self.comm_close()
logger.error("set_valve_clen failed!")
def set_sta_analog(self):
try:
self.comm_connect()
self.S48.write_bit(3, 0, 5)
self.comm_close()
except:
self.comm_close()
logger.error("set_sta_analog failed!")
def set_sta_digital(self):
try:
self.comm_connect()
self.S48.write_bit(3, 1, 5)
self.comm_close()
except:
self.comm_close()
logger.error("set_sta_digital failed!")
def set_autozero_off(self):
try:
self.comm_connect()
self.S48.write_bit(4, 0, 5)
self.comm_close()
except:
self.comm_close()
logger.error("set_autozero_off failed!")
def updateCallrecordToAlloc(self, uuid):
sql = "update tb_callrecord set cr_status = 1 where cr_uuid=%d" % (
uuid)
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
except:
logger.error("Error: unable to update tb_callrecord to alloc.")
return
def __init__(self, graph=None, file_name=None):
if file_name:
path = os.path.join(os.getcwd(), 'export', '%s.json' % file_name)
try:
with open(path, 'r', encoding='utf-8') as fr:
data = json.load(fr)
graph = nx.node_link_graph(data)
except Exception as e:
logger.error(e)
nx.MultiDiGraph.__init__(self, graph)
def load_from_data(self, data):
if type(data) != dict:
logger.info('%s object can not be load as a graph, please load a dict' % str(type(data)))
return
try:
self.graph = nx.node_link_graph(data)
except Exception as e:
logger.error(e)
else:
return self.graph
def read_sta(self):
try:
self.comm_connect()
sta = self.S48.read_bit(3, 1)
self.comm_close()
return sta
except:
logger.error("read_sta failed!")
self.comm_close()
return 0
def load_from_file(self, path):
try:
with open(path, 'r', encoding='utf-8') as fr:
data = json.load(fr)
self.graph = self.load_from_data(data)
except Exception as e:
# print(e)
logger.error(e)
else:
return self.graph
def read_valve_clen(self):
try:
self.comm_connect()
state = self.S48.read_bit(2, 1)
self.comm_close()
return state
except:
logger.error("read_valve_clen failed!")
self.comm_close()
return 0
def selectAllTasksById(self, taskid):
sql = "select task_uuid from tb_task WHERE task_id='%s'" % (taskid)
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to selectAllTasksById by %s." %
(taskid))
return records
def func_read(filename,separator): line = [] try: with open(filename,'r') as f: for lines in f: if lines.strip(): line.append(tuple(lines.strip().split(separator))) #print line except Exception,e: logger.error(str(e))
def selectDialTask(self):
sql = "select distinct task_id from tb_task WHERE task_status = 1"
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to fetch dial task data.")
return records
def selectOneCallRecord(self, uuid):
sql = "select cr_uuid, cr_taskid, cr_mobile, cr_status, cr_calltime from \
tb_callrecord WHERE cr_uuid = %d" % (uuid)
record = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
record = cursor.fetchall()
except:
logger.error("Error: unable to fetch a call record data.")
return record[0]
def updateDialTime(self, rec):
state = rec.get('cr_status')
uuid = rec.get('cr_uuid')
sql = "UPDATE tb_callrecord SET cr_calltime=NOW(3), cr_status=%d WHERE cr_uuid=%d" % (
state, uuid)
logger.debug("updateDialTime sql: %s", sql)
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
except:
logger.error("Error: unable to update dial time.")
return
def get_pcb_state(self):
self.pcb.write("MEAS:*IDN\n")
try:
res = self.pcb.readline()
if res == "partulab\n":
return True
else:
logger.error("Read return message from PCB timeout!")
return False
except Exception,e:
logger.error("Read from PCB port %s error:%s" % (PORT,e))
return False
def get_pcb_state(self):
self.pcb.write("MEAS:*IDN\n")
try:
res = self.pcb.readline()
if res == "partulab\n":
return True
else:
logger.error("Read return message from PCB timeout!")
return False
except Exception, e:
logger.error("Read from PCB port %s error:%s" % (PORT, e))
return False
def selectGatewayConfig(self, taskid):
sql = "select DISTINCT task_gateway from tb_task WHERE task_id='%s'" % (
taskid)
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to fetch gateway data.")
logger.debug("selectGatewayConfig by taskid results: %s", records)
return records
def ParserLockFromErr(err):
'''
@param err: ErrLocked
@return: kvrpcpb.LockInfo
'''
if isinstance(err, ErrLocked):
return kvrpcpb.LockInfo(primary_lock=err.primary,
lock_version=err.startTS,
key=err.key,
lock_ttl=err.ttl)
else:
logger.error('type error, %s', type(err))
return None
def selectCallRecordsToDial(self, taskid, cacheNum):
sql = "select cr_uuid, cr_taskid, cr_mobile, cr_status, cr_calltime from \
tb_callrecord WHERE cr_taskid = '%s' and cr_status = 0 \
order by cr_uuid limit %d" % (taskid, cacheNum)
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to fetch call records data.")
return records
def selectDialTaskStatus(self, task_id):
sql = "select task_status from tb_task WHERE task_id='%s'" % (task_id)
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to fetch dial task data.")
if records != None and len(records) > 0:
return records[0].get('task_status')
else:
return None
def handle_prewrite_errs(self, errs):
''' handle mvcc Prewrite errs to txn's err.
@param errs: BaseError. one of those errs:
ErrKeyAlreadyExist : when op is Insert and key already exist.
ErrLocked: wait to resolve lock
ErrRetryable: restart txn
None: success
@return: err, locked_errs.
err should be one of those case:
None: if all errs is None
ErrKeyExists: if any of errs is ErrKeyAlreadyExist
ErrRetryable: if any of errs is ErrRetryable, and no err is ErrKeyAlreadyExist
ErrLockConflict : if any of errs is ErrLocked, and no err is ErrKeyAlreadyExist and ErrRetryable
locked_errs: errs if err is ErrLockConflict.
'''
ret = None
locked_errs = list()
for err in errs:
# Check already exists error
if isinstance(
err,
ErrKeyAlreadyExist): # : :type err: ErrKeyAlreadyExist
key = err.Key
logger.debug("key: %s already exists", key)
ret = ErrKeyExists
break # any of
if isinstance(err, ErrRetryable): # : :type err: ErrRetryable
logger.debug("2PC prewrite encounters retryable err: %s",
err.ERROR())
ret = ErrRetry
continue # any of but no ErrKeyExists err in the left errs.
if ret is None: # no other err yet.
# Extract lock from key error
if isinstance(err, ErrLocked): # : :type err: ErrLocked
logger.debug("2PC prewrite encounters lock: %s",
err.ERROR())
locked_errs.append(err)
elif err is not None:
logger.error("2PC prewrite encounters unknown err: %s",
err.ERROR())
if ret:
return ret, None
else:
if len(locked_errs):
return ErrLockConflict, locked_errs
else:
return None, None
def selectManager(self):
sql = "SELECT mi_name,mi_phone,now() \
mi_msg,mi_time FROM tb_manager_info;"
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error(
"Error: unable to fetch tb_manager_info record data.")
return records
def twoPhaseCommit(self, connID=0):
'''
@param connID: int
@return: err.
None : success
ErrInvalidTxn: txn is invalid.
ErrKeyExists: if mutation op is Insert And the key already exists.
ErrRetry: suggests that client may restart the txn again, and this txn is abort.
'''
if not self.valid:
return ErrInvalidTxn
# 0. init mutations
self.us.WalkBuffer()
if self.us.primary is None:
logger.debug("con:%d 2PC, no primary" % connID)
return None
# 1. prewrite mutations
err = self.prewrite(connID)
if err != nil:
logger.debug("con:%d 2PC failed on prewrite: tid: %d, err:%s" %
(connID, self.startTS, err))
# prewrite only ErrKeyExists cann't retry now.
if err != ErrKeyExists:
err = ErrRetry
return err
# 2 commit mutations
commitTS = self.store.GetTimestamp()
# check commitTS
if commitTS <= self.startTS:
err = "con:%d Invalid transaction tso with start_ts=%d while commit_ts=%d" % (
connID, self.startTS, commitTS)
logger.error(err)
return ErrInvalidTSO
self.commitTS = commitTS
if self.store.meta.IsExpired(self.startTS, MaxTxnTimeUse):
logger.error(
"con:%d txn takes too much time, start: %d, commit: %d" %
(connID, self.startTS, self.commitTS))
return ErrRetry
err = self.commit(connID)
if err != nil:
logger.debug("con:%d 2PC failed on commit, tid: %d, err:%s" %
(connID, self.startTS, err))
return err
return nil
def selectPhoneStatus(self, Phone):
sql = "select cr_uuid, cr_taskid, cr_mobile, cr_status, cr_calltime \
from tb_callrecord where cr_mobile='%s' and cr_status=2;" % (
Phone)
records = None
with self.pool.cursor() as cursor:
try:
cursor.execute(sql)
records = cursor.fetchall()
except:
logger.error("Error: unable to fetch dial task data.")
if records != None and len(records) > 0:
return records
else:
return None
def para_convert(self,para_value):
point_num = self.get_point()
if point_num >= 128:
para_value = para_value/10
point_num = point_num - 128
if point_num == 0:
real_para = para_value
elif point_num == 1:
real_para = para_value / 10.0
elif point_num == 2:
real_para = para_value / 100.0
elif point_num == 3:
real_para = para_value / 1000.0
else:
logger.error("point_num error!")
return real_para
#print line
except Exception,e:
logger.error(str(e))
finally:
return line
if task_type == 'single':
file_name = config.get('single_file_conf','in_file')
if file_name is '' :
print 'please config in_file name!'
logger.error('conf error: in_file name is null')
sys.exit(1)
in_file = data_dir+'/'+file_name
print 'in_file path:'+in_file
logger.info('in_file path:'+in_file)
print func_read(in_file,separator)
if task_type == 'multi':
if file_pattern is '' :
print 'please config file_pattern!'
logger.error('conf error: pattern error')
sys.exit(1)
all_files = [i for i in os.listdir(data_dir)]
file_pattern = '^'+file_pattern+'$'
def set_stop(self):
try:
self.write_para('RUNSTA',12) #HOLD仪表暂停运行
except:
logger.error("stop ai518p failed!")
from mylog import logger
logger.warning('I warn you.')
logger.info('Please be informed that this is just a test.')
logger.error('Found error on your smile.')
def close_inst(self):
try:
self.inst.close()
except Exception,e:
logger.error(str(e))