def _udp_listen_thread(self): rx_error = False while self.status and not rx_error: try: ready_to_read, ready_to_write, in_error = select.select( [self.__connSocket], [], []) if ready_to_read: data, addr = self.__connSocket.recvfrom(RX_BUFFER_SIZE) #print "Message received %s" % data if data: rx_bytes = bytearray(data) self.dataReadCallback(rx_bytes) #sleep(0.001) elif ready_to_write: continue else: # Connection closed rx_error = True except socket.error, v: rx_error = True logger.error('Exception receiving data new connection, ' + v.strerror)
def __init__(self): threading.Thread.__init__(self) if OfflineAntiVirusThread.idx < OfflineAntiVirusThread.maxOffAVT: OfflineAntiVirusThread.idx += 1 else: err = 'Offline AntiVirus resource limited!!!Maximum number %i reached!' logger.error(err,OfflineAntiVirusThread.maxOffAVT)
def get_pid(self, uid): """ :param uid: :return: corresponding pid """ fetcher = self.fetchers[self.main_fetcher] url = 'http://www.weibo.com/u/%s' % (uid,) while True: html = open_url(fetcher, url) parsed_uid = self.parser.parse_uid(html) if parsed_uid == -1: self.ban_account() continue elif self.parser.is_visitor(html) is True: self.reset_account() continue # make sure that the html is correct. is_enterprise = self.parser.parse_is_enterprise(html) if is_enterprise is True: return -1 # -1 denotes this user is an enterprise pid = self.parser.parse_pid(html) if pid is not None: return pid else: log.error('Cannot get pid for uid:%s' % (uid,)) time.sleep(random.randint(Config.SLEEP_WHEN_EXCEPTION, 2*Config.SLEEP_WHEN_EXCEPTION))
def __init__(self): threading.Thread.__init__(self) if OfflineAntiVirusThread.idx < OfflineAntiVirusThread.maxOffAVT: OfflineAntiVirusThread.idx += 1 else: err = 'Offline AntiVirus resource limited!!!Maximum number %i reached!' logger.error(err, OfflineAntiVirusThread.maxOffAVT)
def cancel(self, event_id): """Remove an event from the queue. This must be presented the ID as returned by enter(). If the event is not in the queue, this raises ValueError. """ i = 0 deleted = False global logger print 'event ', event_id, ' to be canceled\n' while i < len(scheduler._queue): #print i,'th event in queue is',scheduler._queue[i][4] if scheduler._queue[i][4] == int(event_id): print 'event ', event_id, ' is queuing\n' #time.sleep(3) #might cause a problem of inconsistency #should try and catch scheduler._queue.remove(scheduler._queue[i]) deleted = True break i = i + 1 if deleted: msg = 'Successfully canceld event %s' logger.debug(msg, event_id) scheduler.print_queue() heapq.heapify(scheduler._queue) else: msg = 'fail to cancel event %s' logger.error(msg, event_id) print msg % event_id return deleted
def _answerCIPSTARTHandler(self, answer): if answer.ok: # Serial port self.__serialPort.rawMode = True try: localPort = int(self.localport_text.GetValue()) downPort = int(self.local_down_port_text.GetValue()) type = self.conntype_combo.GetValue() # Check correct values if self.__localSocket.start(localPort, downPort, type): # Serial and local socket are both connected # now just enable redirection txt = '\nWaiting for local socket connection' logger.info('Waiting for local socket connection') else: txt = '\nError opening local ports' logger.error('Error opening local ports') txt += '\nRemote connection open' logger.info('Remote connection open') except Exception, v: txt = '\nWrong port number' logger.error('Wrong port number(s) ' + v.strerror)
def try_execute(f, *args, **kwargs): try: result = f(*args, **kwargs) return True, result except Exception as e: logger.error(str(e)) return False, str(e)
def checkResponse(self, resStr): res = None try: # Remove # resStr = resStr.replace(self.sentString, '') # Only if we have echo activated if resStr.find('ERROR') != -1: res = ATresponse(True, self.cmd, [resStr]) elif resStr.find('OK') != -1: resStr = (resStr.replace('\n', '')).replace('\r', '') logger.debug("[%s] search %s " % (resStr, commandsList[self.cmd][2])) # Capture values res = self._parseValues(resStr) elif resStr.find('\r\n'): if commandsList[self.cmd][2] != '': res = self._parseValues(resStr) #searchObj = re.search(commandsList[self.cmd][2], resStr, re.S | re.M) #if searchObj: # res = ATresponse(True, self.cmd, searchObj.groups()) except Exception as e: logger.error("Exception parse response, %s" % e) return res
def _store_data(self, week, day, date, session): db = Modle.get_db() if db is None: print("Error, please check the log") exit(1) cur = db.cursor() if cur is None: print("Error, please check the log") exit(1) data = self.get_empty_classroom(week, day + 1, self.session_list.get(session)) now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") insert_sql = """ insert into `empty_classroom`(`date`, `week`, `day`, `session`, `data`, `updated_at`, `campus`) values (str_to_date('{}', '{}'), {}, {}, '{}', '{}', '{}', '{}') """.format(date.strftime('%Y-%m-%d'), '%Y-%m-%d', week, day, session, str(data).replace("\'", "\"", -1), now, Config.basicInfo.get("xqh_id")) try: with self.Lock: cur.execute(insert_sql) db.commit() print("{} {}, {} 校区的数据已被插入".format( date.strftime('%Y-%m-%d'), session, Config.basicInfo.get("xqh_id"))) except Exception as e: logger.error(e) print('A failure happened. Please check logging.log') db.rollback()
def check_var_info(self, id, name): var_info = self.get_var_info(id, name) if var_info != '': return var_info else: logger.error('根据id:{},name:{},未找到对应变量'.format(id, name)) raise Exception('根据id:{},name:{},未找到对应变量'.format(id, name))
def urgent(self, event_id): global logger msg = "Try to elevate event %s to urgent" i = 0 while i < len(scheduler._queue): if scheduler._queue[i][4] == int(event_id): break else: i += 1 if i == len(scheduler._queue): err = 'Can not find event %s' logger.error(err, event_id) print err % event_id return False else: (tmp, tmp, handle, argument, uid, tmp) = scheduler._queue[i] scheduler._queue.remove(scheduler._queue[i]) event = Event(0, 0, handle, argument, uid, 'URG') heapq.heappush(scheduler._queue, event) if Wakeup_Lock().IsLocked(): Wakeup_Lock().OpenLock() print 'after urgent insert\n' i = 0 while i < len(scheduler._queue): print i, 'th event in queue is', scheduler._queue[i][4] i += 1 #time.sleep(5) logger.debug(msg, event_id) print msg % event_id return True
def get_pid(self, uid): """ :param uid: :return: corresponding pid """ fetcher = self.fetchers[self.main_fetcher] url = 'http://www.weibo.com/u/%s' % (uid, ) while True: html = open_url(fetcher, url) parsed_uid = self.parser.parse_uid(html) if parsed_uid == -1: self.ban_account() continue elif self.parser.is_visitor(html) is True: self.reset_account() continue # make sure that the html is correct. is_enterprise = self.parser.parse_is_enterprise(html) if is_enterprise is True: return -1 # -1 denotes this user is an enterprise pid = self.parser.parse_pid(html) if pid is not None: return pid else: log.error('Cannot get pid for uid:%s' % (uid, )) time.sleep( random.randint(Config.SLEEP_WHEN_EXCEPTION, 2 * Config.SLEEP_WHEN_EXCEPTION))
def wrapper(*args, **kwargs): for try_number in range(num_of_tries): success, result = try_execute(func, *args, **kwargs) if success: return result logger.error( f'{func.__name__} failed in try number {try_number + 1}') raise Exception(f"Failed after {num_of_tries} times!")
def _listen_thread(self): if not self.status: return try: # Listen for incoming connections self.__listenSocket.listen(1) except socket.error, v: logger.error('Socket error listening, ' + v.strerror)
def rxDataCallback(self, data): if self.__socketRedirection: #print "socket wr " + data #print "." self.__localSocket.writeData(data) else: try: wx.CallAfter(pub.sendMessage, "AppendLogText", text=data) except Exception, v: logger.error("Exception writing to log_text " + v.strerror)
def get_empty_classroom(self, week, day, session): global r codes = { '综合楼': '13', '教三楼': '05', '教一楼': '06', '教二楼': '04', '东教楼': '15', '公共教学楼二': '22', '公共教学楼一': '21', } old_campus_buildings = ['综合楼', '教三楼', '教二楼', '教一楼', '东教楼'] new_campus_buildings = ['公共教学楼一', '公共教学楼二'] result = {} buildings = (old_campus_buildings if Config.basicInfo.get("xqh_id") == '1' else new_campus_buildings) # 对每栋楼进行抓取 for building in buildings: result[building] = [] data = EmptyClassroomSpider.get_data(week, day, session, codes.get(building)) try: self.session.headers.update({ 'User-Agent': str(self.UA.random), 'Accept-Language': 'zh-CN,zh;q=0.9' }) r = self.session.post(data=data, url=self.get_empty_classroom_url, timeout=100) except Exception as e: logger.error(e, str(traceback.format_exc())) print('Failure. Please check logging.log') # exit(1) r = self.session.post(data=data, url=self.get_empty_classroom_url, timeout=100) if '用户登录' in r.text: logger.error("未进行单点登录") print('Failure. Please check logging.log') exit(1) res = r.json() for item in res.get('items'): if building != "综合楼": result[building].append( item.get('cdmc').replace(building, "").replace( '公教1-', "").replace('公教2-', "")) else: result[building].append( item.get('cdmc').replace("北综楼", "")) return result
def run(self): thread_pool = [] self.log_in() db = Modle.get_db() cur = db.cursor() # del_table_sql = """drop table if exists empty_classroom""" create_table_sql = """ create table if not exists empty_classroom ( id int unsigned primary key auto_increment, date date not null, campus tinyint not null , day int, week int, session varchar(50), data longtext, updated_at datetime) """ try: # cur.execute(del_table_sql) cur.execute(create_table_sql) except Exception as e: logger.error(e, str(traceback.format_exc())) print('Failure. Please check logging.log') exit(1) date = datetime.datetime.today() + datetime.timedelta(days=1) print("working... Just be patient~") for week in range(self.start_week, self.end_week + 1): if week == self.start_week: days = date.weekday() date = date + datetime.timedelta(days=-1) else: days = 0 for day in range(days, 7): date = date + datetime.timedelta(days=1) for session in self.session_list: th = threading.Thread(target=self._store_data, args=( week, day, date, session, )) thread_pool.append(th) # 降低速度防止被封 time.sleep(1) th.start() for t in thread_pool: t.join() print('Complete!')
def writeData(self, tx_data): #print "Sending message %s length %d" % (tx_data, len(tx_data)) if self.status: try: if self.__socketType == 'TCP': self.__connSocket.sendall(tx_data) else: self.__udpDownSocket.sendto(tx_data, self.__remoteAddress) except socket.error, v: logger.error("Error sending mesage " + v.strerror)
def writeDirect(self, data): try: if self.status: if self.rawMode: self.__serial.write(data) else: logger.debug("tx " + data) self.rawData(data) self.__serial.write(self.__tx_decoder.encode(data)) except Exception, v: logger.error("Exception writeDirect")
def reconnect(self, port, baudrate): res = True try: if self.status: self.__serial.close() self.status = False self.__serial.port = port self.__serial.baudrate = baudrate self.__serial.open() self.status = True except: res = False logger.error("Error reconnecting") return res
def get_db(): global conn if conn is None: with lock: if conn is None: try: conn = pymysql.connect(db_config.get("host"), db_config.get("username"), db_config.get("password"), db_config.get("database")) except Exception as e: logger.error( "fail to connect to the database: {}".format(e)) exit(1) return conn
def __init__(self, cmdId, setMode, params): self.cmd = cmdId self.setOperation = setMode self.params = params self.sentString = '' try: self.paramString = commandsList[self.cmd][1] self.answer = '' if self.paramString != '': if self.paramString.find('%val1%') != -1 and len(params) > 0: self.paramString = self.paramString.replace('%val1%', params[0]) if self.paramString.find('%val2%') != -1 and len(params) > 1: self.paramString = self.paramString.replace('%val2%', params[1]) if self.paramString.find('%val3%') != -1 and len(params) > 2: self.paramString = self.paramString.replace('%val3%', params[2]) except: logger.error("Exception creating command")
def _timeoutHandler(self): try: logger.debug("Command response timeout, command %s" % self.__currCommand.cmd) answer = None self.__cmdMutex.acquire() if self.__waitingAnswer: self.__waitingAnswer = False answer = ATresponse(False, self.__currCommand.cmd, ["Timeout"]) self.__cmdMutex.release() if answer is not None: self.receivedAnswer(answer) except: logger.error("Timeout handler exception") self.__cmdProcesssedEvent.set()
def stop(self): if self.status: self.status = False self.connOpen = False if self.__connSocket is not None: try: self.__connSocket.shutdown(socket.SHUT_RDWR) self.__connSocket.close() except socket.error, v: logger.error('Error closing Socket Port' + v.strerror) if self.__listenSocket is not None: try: self.__listenSocket.shutdown(socket.SHUT_RDWR) self.__listenSocket.close() except socket.error, v: logger.error('Error closing Socket Port, ' + v.strerror)
def start(self, port, downPort, socketType): result = True if self.status and socketType not in ['TCP', 'UDP']: result = False else: try: if socketType == 'TCP': self.__listenSocket = socket.socket( socket.AF_INET, socket.SOCK_STREAM) # Bind the socket to the port self.__remoteAddress = ( "", port) # to connect from outside the machine self.__listenSocket.bind(self.__remoteAddress) logger.info('Starting request port on port ' + str(port)) target = self._listen_thread self.status = True elif socketType == 'UDP': # Uplink socket (read) self.__connSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) uplinkAddr = ("127.0.0.1", port) self.__connSocket.bind(uplinkAddr) self.__connSocket.setblocking(0) self.__connSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Downlink socket (write) self.__remoteAddress = ("127.0.0.1", downPort) self.__udpDownSocket = socket.socket( socket.AF_INET, socket.SOCK_DGRAM) self.__udpDownSocket.setblocking(0) logger.info('Open socket on port ' + str(port)) target = self._udp_listen_thread self.status = True self.connOpenCallback() except socket.error, v: logger.error('Error starting local socket Port ' + v.strerror) result = False
def start(self, port, baudrate): result = True if self.status: result = False if result: try: self.status = True self.__serial = serial.Serial() self.__serial.port = port self.__serial.baudrate = baudrate if not hasattr(self.__serial, 'cancel_read'): # enable timeout for alive flag polling if cancel_read is not available self.__serial.timeout = 1 self.__serial.open() if self.__serial.in_waiting > 0: self.__serial.timeout = 1 self.__serial.read(self.__serial.in_waiting) self.__serial.timeout = 0 except serial.SerialException as e: logger.error('could not open port {!r}: {}\n'.format(port, e)) self.status = False result = False if result: self.__readThread = threading.Thread(target=self._reader, args=()) self.__readThread.daemon = True self.__writeThread = threading.Thread(target=self._writer, args=()) self.__writeThread.daemon = True # start thread self.__readThread.start() self.__writeThread.start() return result
def log_in(self): global res key = b'neusofteducationplatform' iv = '01234567' k = triple_des(key, CBC, iv, pad=None, padmode=PAD_PKCS5) student_no_encrypt = k.encrypt(self.username).hex() md5 = hashlib.md5() md5.update(self.password.encode('utf-8')) password_encrypt = md5.hexdigest().upper() self.session.headers.update({ 'source': 'neumobile', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Host': 'xyfw.cug.edu.cn', 'User-Agent': str(self.UA.random), 'Upgrade-Insecure-Requests': '1', 'Accept-Language': 'zh-cn', 'id_number': student_no_encrypt, 'enp': password_encrypt }) try: res = self.session.get(self.login_url, timeout=1000) except Exception as e: logger.error(e, str(traceback.format_exc())) print('Failure. Please check logging.log') exit(1) if '错误' in res.text and 'sfrz' in res.url: logger.error("{} 的账号密码错误".format(self.username)) print('Failure. Please check logging.log') exit(1) elif '身份验证失败' in res.text and 'ssoserver' in res.url: logger.error("学校SSO系统故障") print('Failure. Please check logging.log') exit(1) try: self.session.get(self.login_classroom_system_url, timeout=1000) except Exception as e: logger.error(e, str(traceback.format_exc())) print('Failure. Please check logging.log') exit(1) return True
def parse_class_vars(self): class_var_decs = self.soup.find_all('classVarDec') static_index = 0 field_index = 0 for class_var_dec in class_var_decs: all_elements = class_var_dec.find_all() var_kind = all_elements[0].string var_type = all_elements[1].string # 前面两个标签是变量定义,故去除 all_elements = all_elements[2:len(all_elements) - 1] for element in all_elements: # 变量定义一定是identifier标签 if element.name != 'identifier': continue if var_kind == 'static': self.class_table.append({ 'id': self.class_name, 'name': element.string, 'type': var_type, 'kind': 'static', 'index': static_index }) static_index += 1 elif var_kind == 'field': self.class_table.append({ 'id': self.class_name, 'name': element.string, 'type': var_type, 'kind': 'field', 'index': field_index }) field_index += 1 else: logger.error('解析类变量出错,未知的变量类型{}'.format(element)) raise Exception('解析类变量出错,未知的变量类型{}'.format(element))
def _writer(self): self.__cmdProcesssedEvent.clear() while self.status: try: logger.debug("[W]queue Get") newCmd = self.__cmdQueue.get(True) if newCmd is None: #Exit from queue block raise Exception("Exit thread") logger.debug("[W]new cmd " + newCmd.cmd) #self.__cmdMutex.acquire() # Check if we are waiting for an answer self.__waitingAnswer = True self.__currCommand = newCmd #self.__cmdMutex.release() self.writeDirect(self.__currCommand.getString()) # Start timeout logger.debug("[W]Start timeout timer") self.__TOTimer = threading.Timer(5, self._timeoutHandler) self.__TOTimer.start() logger.debug("[W]Wait event") self.__cmdProcesssedEvent.wait() self.__cmdProcesssedEvent.clear() logger.debug("[W]event rx") except Exception: logger.error("[W]Exception write thread ") logger.debug("exit wr")
def compile_term(self, term): # logger.debug(term) name = term.find().name text = term.find().text length = len(term.find_all(recursive=False)) if length > 1: next_text = term.find_all(recursive=False)[1].text # 如果是前缀操作符号 if name == 'symbol' and length > 1 and text in ['~', '-']: self.handle_symbol(term, text) # 如果是数字常量 elif name == 'integerConstant': self.vm_code.append('push constant {}'.format(text)) # 如果是字符串常量 elif name == 'stringConstant': logger.info('=======stringConstant========{}'.format(text)) self.handle_string(text) # 如果是复合表达式 elif text == '(': expression = term.find('expression', recursive=False) self.compile_expression(expression) # 如果是方法调用 elif name == 'identifier' and length > 1 and (next_text == '(' or next_text == '.'): self.handle_subroutine_call(next_text, term) # 如果是变量 elif name == 'identifier' and length == 1: self.deal_single_var(text) # todo 如果是数组变量 elif name == 'identifier' and length > 1 and next_text == '[': self.handle_array(term) elif name == 'keyword' and text in ['true', 'false', 'null', 'this']: self.headle_keyword(text) else: logger.error('无法识别此表达式{}'.format(term)) raise Exception('无法识别此表达式{}'.format(term))
while True: # in case of connection lost try: spider.save() break except Exception as e: print e.message, uid if 'Lost connection to MySQL server during query' in e.message: continue else: break crawled_list.append(uid) spider.end_time = datetime.now() duration = spider.end_time - spider.start_time if duration.seconds > ACCOUNT_CHANGE_TIME: spider.main_fetcher = loop_increase( spider.main_fetcher, len(spider.fetchers)) spider.start_time = datetime.now() emphasis_print('Account changed!!!') print 'Complete a batch of tasks!' print 'Getting new tasks...' uid_list = get_tasks(TASK_NUM) if len(uid_list) == 0: print 'No tasks to proceed!' exit(-1) except Exception as e: print e.message log.error('Problematic UID: %s' % (uid, )) finally: reset(user_list, uid_list, crawled_list) # reset
while True: # in case of connection lost try: spider.save() break except Exception as e: print e.message, uid if 'Lost connection to MySQL server during query' in e.message: continue else: break crawled_list.append(uid) spider.end_time = datetime.now() duration = spider.end_time - spider.start_time if duration.seconds > ACCOUNT_CHANGE_TIME: spider.main_fetcher = loop_increase(spider.main_fetcher, len(spider.fetchers)) spider.start_time = datetime.now() emphasis_print('Account changed!!!') print 'Complete a batch of tasks!' print 'Getting new tasks...' uid_list = get_tasks(TASK_NUM) if len(uid_list) == 0: print 'No tasks to proceed!' exit(-1) except Exception as e: print e.message log.error('Problematic UID: %s' % (uid, )) finally: reset(user_list, uid_list, crawled_list) # reset
try: logger.info('Waiting for new connection') # Wait for a connection self.__connSocket, client_address = self.__listenSocket.accept( ) logger.debug('New Connection from ' + client_address[0]) self.connOpen = True self.connOpenCallback() if not self._receive_data(): logger.info('Connection closed') self.connClosedCallback() except socket.error: logger.error('Exception accepting new connection') self.stop() def _receive_data(self): rx_error = False while self.status and not rx_error: try: rx_bytes = bytearray(self.__connSocket.recv(RX_BUFFER_SIZE)) if rx_bytes is not None: self.dataReadCallback(rx_bytes) else: # Connection closed rx_error = True except socket.error: