def _udp_listen_thread(self):
rx_error = False
while self.status and not rx_error:
try:
ready_to_read, ready_to_write, in_error = select.select(
[self.__connSocket], [], [])
if ready_to_read:
data, addr = self.__connSocket.recvfrom(RX_BUFFER_SIZE)
#print "Message received %s" % data
if data:
rx_bytes = bytearray(data)
self.dataReadCallback(rx_bytes)
#sleep(0.001)
elif ready_to_write:
continue
else:
# Connection closed
rx_error = True
except socket.error, v:
rx_error = True
logger.error('Exception receiving data new connection, ' +
v.strerror)
def __init__(self):
threading.Thread.__init__(self)
if OfflineAntiVirusThread.idx < OfflineAntiVirusThread.maxOffAVT:
OfflineAntiVirusThread.idx += 1
else:
err = 'Offline AntiVirus resource limited!!!Maximum number %i reached!'
logger.error(err,OfflineAntiVirusThread.maxOffAVT)
def get_pid(self, uid):
"""
:param uid:
:return: corresponding pid
"""
fetcher = self.fetchers[self.main_fetcher]
url = 'http://www.weibo.com/u/%s' % (uid,)
while True:
html = open_url(fetcher, url)
parsed_uid = self.parser.parse_uid(html)
if parsed_uid == -1:
self.ban_account()
continue
elif self.parser.is_visitor(html) is True:
self.reset_account()
continue # make sure that the html is correct.
is_enterprise = self.parser.parse_is_enterprise(html)
if is_enterprise is True:
return -1 # -1 denotes this user is an enterprise
pid = self.parser.parse_pid(html)
if pid is not None:
return pid
else:
log.error('Cannot get pid for uid:%s' % (uid,))
time.sleep(random.randint(Config.SLEEP_WHEN_EXCEPTION, 2*Config.SLEEP_WHEN_EXCEPTION))
def __init__(self):
threading.Thread.__init__(self)
if OfflineAntiVirusThread.idx < OfflineAntiVirusThread.maxOffAVT:
OfflineAntiVirusThread.idx += 1
else:
err = 'Offline AntiVirus resource limited!!!Maximum number %i reached!'
logger.error(err, OfflineAntiVirusThread.maxOffAVT)
def cancel(self, event_id):
"""Remove an event from the queue.
This must be presented the ID as returned by enter().
If the event is not in the queue, this raises ValueError.
"""
i = 0
deleted = False
global logger
print 'event ', event_id, ' to be canceled\n'
while i < len(scheduler._queue):
#print i,'th event in queue is',scheduler._queue[i][4]
if scheduler._queue[i][4] == int(event_id):
print 'event ', event_id, ' is queuing\n'
#time.sleep(3) #might cause a problem of inconsistency
#should try and catch
scheduler._queue.remove(scheduler._queue[i])
deleted = True
break
i = i + 1
if deleted:
msg = 'Successfully canceld event %s'
logger.debug(msg, event_id)
scheduler.print_queue()
heapq.heapify(scheduler._queue)
else:
msg = 'fail to cancel event %s'
logger.error(msg, event_id)
print msg % event_id
return deleted
def _answerCIPSTARTHandler(self, answer):
if answer.ok:
# Serial port
self.__serialPort.rawMode = True
try:
localPort = int(self.localport_text.GetValue())
downPort = int(self.local_down_port_text.GetValue())
type = self.conntype_combo.GetValue()
# Check correct values
if self.__localSocket.start(localPort, downPort, type):
# Serial and local socket are both connected
# now just enable redirection
txt = '\nWaiting for local socket connection'
logger.info('Waiting for local socket connection')
else:
txt = '\nError opening local ports'
logger.error('Error opening local ports')
txt += '\nRemote connection open'
logger.info('Remote connection open')
except Exception, v:
txt = '\nWrong port number'
logger.error('Wrong port number(s) ' + v.strerror)
def try_execute(f, *args, **kwargs):
try:
result = f(*args, **kwargs)
return True, result
except Exception as e:
logger.error(str(e))
return False, str(e)
def checkResponse(self, resStr):
res = None
try:
# Remove
# resStr = resStr.replace(self.sentString, '') # Only if we have echo activated
if resStr.find('ERROR') != -1:
res = ATresponse(True, self.cmd, [resStr])
elif resStr.find('OK') != -1:
resStr = (resStr.replace('\n', '')).replace('\r', '')
logger.debug("[%s] search %s " %
(resStr, commandsList[self.cmd][2]))
# Capture values
res = self._parseValues(resStr)
elif resStr.find('\r\n'):
if commandsList[self.cmd][2] != '':
res = self._parseValues(resStr)
#searchObj = re.search(commandsList[self.cmd][2], resStr, re.S | re.M)
#if searchObj:
# res = ATresponse(True, self.cmd, searchObj.groups())
except Exception as e:
logger.error("Exception parse response, %s" % e)
return res
def _store_data(self, week, day, date, session):
db = Modle.get_db()
if db is None:
print("Error, please check the log")
exit(1)
cur = db.cursor()
if cur is None:
print("Error, please check the log")
exit(1)
data = self.get_empty_classroom(week, day + 1,
self.session_list.get(session))
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
insert_sql = """
insert into `empty_classroom`(`date`, `week`, `day`, `session`, `data`, `updated_at`, `campus`) values
(str_to_date('{}', '{}'), {}, {}, '{}', '{}', '{}', '{}')
""".format(date.strftime('%Y-%m-%d'), '%Y-%m-%d', week, day,
session,
str(data).replace("\'", "\"", -1), now,
Config.basicInfo.get("xqh_id"))
try:
with self.Lock:
cur.execute(insert_sql)
db.commit()
print("{} {}, {} 校区的数据已被插入".format(
date.strftime('%Y-%m-%d'), session,
Config.basicInfo.get("xqh_id")))
except Exception as e:
logger.error(e)
print('A failure happened. Please check logging.log')
db.rollback()
def check_var_info(self, id, name):
var_info = self.get_var_info(id, name)
if var_info != '':
return var_info
else:
logger.error('根据id：{}，name：{}，未找到对应变量'.format(id, name))
raise Exception('根据id：{}，name：{}，未找到对应变量'.format(id, name))
def urgent(self, event_id):
global logger
msg = "Try to elevate event %s to urgent"
i = 0
while i < len(scheduler._queue):
if scheduler._queue[i][4] == int(event_id):
break
else:
i += 1
if i == len(scheduler._queue):
err = 'Can not find event %s'
logger.error(err, event_id)
print err % event_id
return False
else:
(tmp, tmp, handle, argument, uid, tmp) = scheduler._queue[i]
scheduler._queue.remove(scheduler._queue[i])
event = Event(0, 0, handle, argument, uid, 'URG')
heapq.heappush(scheduler._queue, event)
if Wakeup_Lock().IsLocked():
Wakeup_Lock().OpenLock()
print 'after urgent insert\n'
i = 0
while i < len(scheduler._queue):
print i, 'th event in queue is', scheduler._queue[i][4]
i += 1
#time.sleep(5)
logger.debug(msg, event_id)
print msg % event_id
return True
def get_pid(self, uid):
"""
:param uid:
:return: corresponding pid
"""
fetcher = self.fetchers[self.main_fetcher]
url = 'http://www.weibo.com/u/%s' % (uid, )
while True:
html = open_url(fetcher, url)
parsed_uid = self.parser.parse_uid(html)
if parsed_uid == -1:
self.ban_account()
continue
elif self.parser.is_visitor(html) is True:
self.reset_account()
continue # make sure that the html is correct.
is_enterprise = self.parser.parse_is_enterprise(html)
if is_enterprise is True:
return -1 # -1 denotes this user is an enterprise
pid = self.parser.parse_pid(html)
if pid is not None:
return pid
else:
log.error('Cannot get pid for uid:%s' % (uid, ))
time.sleep(
random.randint(Config.SLEEP_WHEN_EXCEPTION,
2 * Config.SLEEP_WHEN_EXCEPTION))
def wrapper(*args, **kwargs):
for try_number in range(num_of_tries):
success, result = try_execute(func, *args, **kwargs)
if success:
return result
logger.error(
f'{func.__name__} failed in try number {try_number + 1}')
raise Exception(f"Failed after {num_of_tries} times!")
def _listen_thread(self):
if not self.status:
return
try:
# Listen for incoming connections
self.__listenSocket.listen(1)
except socket.error, v:
logger.error('Socket error listening, ' + v.strerror)
def rxDataCallback(self, data):
if self.__socketRedirection:
#print "socket wr " + data
#print "."
self.__localSocket.writeData(data)
else:
try:
wx.CallAfter(pub.sendMessage, "AppendLogText", text=data)
except Exception, v:
logger.error("Exception writing to log_text " + v.strerror)
def get_empty_classroom(self, week, day, session):
global r
codes = {
'综合楼': '13',
'教三楼': '05',
'教一楼': '06',
'教二楼': '04',
'东教楼': '15',
'公共教学楼二': '22',
'公共教学楼一': '21',
}
old_campus_buildings = ['综合楼', '教三楼', '教二楼', '教一楼', '东教楼']
new_campus_buildings = ['公共教学楼一', '公共教学楼二']
result = {}
buildings = (old_campus_buildings if Config.basicInfo.get("xqh_id")
== '1' else new_campus_buildings)
# 对每栋楼进行抓取
for building in buildings:
result[building] = []
data = EmptyClassroomSpider.get_data(week, day, session,
codes.get(building))
try:
self.session.headers.update({
'User-Agent': str(self.UA.random),
'Accept-Language': 'zh-CN,zh;q=0.9'
})
r = self.session.post(data=data,
url=self.get_empty_classroom_url,
timeout=100)
except Exception as e:
logger.error(e, str(traceback.format_exc()))
print('Failure. Please check logging.log')
# exit(1)
r = self.session.post(data=data,
url=self.get_empty_classroom_url,
timeout=100)
if '用户登录' in r.text:
logger.error("未进行单点登录")
print('Failure. Please check logging.log')
exit(1)
res = r.json()
for item in res.get('items'):
if building != "综合楼":
result[building].append(
item.get('cdmc').replace(building, "").replace(
'公教1-', "").replace('公教2-', ""))
else:
result[building].append(
item.get('cdmc').replace("北综楼", ""))
return result
def run(self):
thread_pool = []
self.log_in()
db = Modle.get_db()
cur = db.cursor()
# del_table_sql = """drop table if exists empty_classroom"""
create_table_sql = """
create table if not exists empty_classroom (
id int unsigned primary key auto_increment,
date date not null,
campus tinyint not null ,
day int,
week int,
session varchar(50),
data longtext,
updated_at datetime)
"""
try:
# cur.execute(del_table_sql)
cur.execute(create_table_sql)
except Exception as e:
logger.error(e, str(traceback.format_exc()))
print('Failure. Please check logging.log')
exit(1)
date = datetime.datetime.today() + datetime.timedelta(days=1)
print("working... Just be patient~")
for week in range(self.start_week, self.end_week + 1):
if week == self.start_week:
days = date.weekday()
date = date + datetime.timedelta(days=-1)
else:
days = 0
for day in range(days, 7):
date = date + datetime.timedelta(days=1)
for session in self.session_list:
th = threading.Thread(target=self._store_data,
args=(
week,
day,
date,
session,
))
thread_pool.append(th)
# 降低速度防止被封
time.sleep(1)
th.start()
for t in thread_pool:
t.join()
print('Complete!')
def writeData(self, tx_data):
#print "Sending message %s length %d" % (tx_data, len(tx_data))
if self.status:
try:
if self.__socketType == 'TCP':
self.__connSocket.sendall(tx_data)
else:
self.__udpDownSocket.sendto(tx_data, self.__remoteAddress)
except socket.error, v:
logger.error("Error sending mesage " + v.strerror)
def writeDirect(self, data):
try:
if self.status:
if self.rawMode:
self.__serial.write(data)
else:
logger.debug("tx " + data)
self.rawData(data)
self.__serial.write(self.__tx_decoder.encode(data))
except Exception, v:
logger.error("Exception writeDirect")
def reconnect(self, port, baudrate):
res = True
try:
if self.status:
self.__serial.close()
self.status = False
self.__serial.port = port
self.__serial.baudrate = baudrate
self.__serial.open()
self.status = True
except:
res = False
logger.error("Error reconnecting")
return res
def get_db():
global conn
if conn is None:
with lock:
if conn is None:
try:
conn = pymysql.connect(db_config.get("host"),
db_config.get("username"),
db_config.get("password"),
db_config.get("database"))
except Exception as e:
logger.error(
"fail to connect to the database: {}".format(e))
exit(1)
return conn
def __init__(self, cmdId, setMode, params):
self.cmd = cmdId
self.setOperation = setMode
self.params = params
self.sentString = ''
try:
self.paramString = commandsList[self.cmd][1]
self.answer = ''
if self.paramString != '':
if self.paramString.find('%val1%') != -1 and len(params) > 0:
self.paramString = self.paramString.replace('%val1%', params[0])
if self.paramString.find('%val2%') != -1 and len(params) > 1:
self.paramString = self.paramString.replace('%val2%', params[1])
if self.paramString.find('%val3%') != -1 and len(params) > 2:
self.paramString = self.paramString.replace('%val3%', params[2])
except:
logger.error("Exception creating command")
def _timeoutHandler(self):
try:
logger.debug("Command response timeout, command %s" %
self.__currCommand.cmd)
answer = None
self.__cmdMutex.acquire()
if self.__waitingAnswer:
self.__waitingAnswer = False
answer = ATresponse(False, self.__currCommand.cmd, ["Timeout"])
self.__cmdMutex.release()
if answer is not None:
self.receivedAnswer(answer)
except:
logger.error("Timeout handler exception")
self.__cmdProcesssedEvent.set()
def stop(self):
if self.status:
self.status = False
self.connOpen = False
if self.__connSocket is not None:
try:
self.__connSocket.shutdown(socket.SHUT_RDWR)
self.__connSocket.close()
except socket.error, v:
logger.error('Error closing Socket Port' + v.strerror)
if self.__listenSocket is not None:
try:
self.__listenSocket.shutdown(socket.SHUT_RDWR)
self.__listenSocket.close()
except socket.error, v:
logger.error('Error closing Socket Port, ' + v.strerror)
def start(self, port, downPort, socketType):
result = True
if self.status and socketType not in ['TCP', 'UDP']:
result = False
else:
try:
if socketType == 'TCP':
self.__listenSocket = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
self.__remoteAddress = (
"", port) # to connect from outside the machine
self.__listenSocket.bind(self.__remoteAddress)
logger.info('Starting request port on port ' + str(port))
target = self._listen_thread
self.status = True
elif socketType == 'UDP':
# Uplink socket (read)
self.__connSocket = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
uplinkAddr = ("127.0.0.1", port)
self.__connSocket.bind(uplinkAddr)
self.__connSocket.setblocking(0)
self.__connSocket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
# Downlink socket (write)
self.__remoteAddress = ("127.0.0.1", downPort)
self.__udpDownSocket = socket.socket(
socket.AF_INET, socket.SOCK_DGRAM)
self.__udpDownSocket.setblocking(0)
logger.info('Open socket on port ' + str(port))
target = self._udp_listen_thread
self.status = True
self.connOpenCallback()
except socket.error, v:
logger.error('Error starting local socket Port ' + v.strerror)
result = False
def start(self, port, baudrate):
result = True
if self.status:
result = False
if result:
try:
self.status = True
self.__serial = serial.Serial()
self.__serial.port = port
self.__serial.baudrate = baudrate
if not hasattr(self.__serial, 'cancel_read'):
# enable timeout for alive flag polling if cancel_read is not available
self.__serial.timeout = 1
self.__serial.open()
if self.__serial.in_waiting > 0:
self.__serial.timeout = 1
self.__serial.read(self.__serial.in_waiting)
self.__serial.timeout = 0
except serial.SerialException as e:
logger.error('could not open port {!r}: {}\n'.format(port, e))
self.status = False
result = False
if result:
self.__readThread = threading.Thread(target=self._reader,
args=())
self.__readThread.daemon = True
self.__writeThread = threading.Thread(target=self._writer,
args=())
self.__writeThread.daemon = True
# start thread
self.__readThread.start()
self.__writeThread.start()
return result
def log_in(self):
global res
key = b'neusofteducationplatform'
iv = '01234567'
k = triple_des(key, CBC, iv, pad=None, padmode=PAD_PKCS5)
student_no_encrypt = k.encrypt(self.username).hex()
md5 = hashlib.md5()
md5.update(self.password.encode('utf-8'))
password_encrypt = md5.hexdigest().upper()
self.session.headers.update({
'source': 'neumobile',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Connection': 'keep-alive',
'Accept-Encoding': 'gzip, deflate',
'Host': 'xyfw.cug.edu.cn',
'User-Agent': str(self.UA.random),
'Upgrade-Insecure-Requests': '1',
'Accept-Language': 'zh-cn',
'id_number': student_no_encrypt,
'enp': password_encrypt
})
try:
res = self.session.get(self.login_url, timeout=1000)
except Exception as e:
logger.error(e, str(traceback.format_exc()))
print('Failure. Please check logging.log')
exit(1)
if '错误' in res.text and 'sfrz' in res.url:
logger.error("{} 的账号密码错误".format(self.username))
print('Failure. Please check logging.log')
exit(1)
elif '身份验证失败' in res.text and 'ssoserver' in res.url:
logger.error("学校SSO系统故障")
print('Failure. Please check logging.log')
exit(1)
try:
self.session.get(self.login_classroom_system_url, timeout=1000)
except Exception as e:
logger.error(e, str(traceback.format_exc()))
print('Failure. Please check logging.log')
exit(1)
return True
def parse_class_vars(self):
class_var_decs = self.soup.find_all('classVarDec')
static_index = 0
field_index = 0
for class_var_dec in class_var_decs:
all_elements = class_var_dec.find_all()
var_kind = all_elements[0].string
var_type = all_elements[1].string
# 前面两个标签是变量定义，故去除
all_elements = all_elements[2:len(all_elements) - 1]
for element in all_elements:
# 变量定义一定是identifier标签
if element.name != 'identifier':
continue
if var_kind == 'static':
self.class_table.append({
'id': self.class_name,
'name': element.string,
'type': var_type,
'kind': 'static',
'index': static_index
})
static_index += 1
elif var_kind == 'field':
self.class_table.append({
'id': self.class_name,
'name': element.string,
'type': var_type,
'kind': 'field',
'index': field_index
})
field_index += 1
else:
logger.error('解析类变量出错，未知的变量类型{}'.format(element))
raise Exception('解析类变量出错，未知的变量类型{}'.format(element))
def _writer(self):
self.__cmdProcesssedEvent.clear()
while self.status:
try:
logger.debug("[W]queue Get")
newCmd = self.__cmdQueue.get(True)
if newCmd is None:
#Exit from queue block
raise Exception("Exit thread")
logger.debug("[W]new cmd " + newCmd.cmd)
#self.__cmdMutex.acquire()
# Check if we are waiting for an answer
self.__waitingAnswer = True
self.__currCommand = newCmd
#self.__cmdMutex.release()
self.writeDirect(self.__currCommand.getString())
# Start timeout
logger.debug("[W]Start timeout timer")
self.__TOTimer = threading.Timer(5, self._timeoutHandler)
self.__TOTimer.start()
logger.debug("[W]Wait event")
self.__cmdProcesssedEvent.wait()
self.__cmdProcesssedEvent.clear()
logger.debug("[W]event rx")
except Exception:
logger.error("[W]Exception write thread ")
logger.debug("exit wr")
def compile_term(self, term):
# logger.debug(term)
name = term.find().name
text = term.find().text
length = len(term.find_all(recursive=False))
if length > 1:
next_text = term.find_all(recursive=False)[1].text
# 如果是前缀操作符号
if name == 'symbol' and length > 1 and text in ['~', '-']:
self.handle_symbol(term, text)
# 如果是数字常量
elif name == 'integerConstant':
self.vm_code.append('push constant {}'.format(text))
# 如果是字符串常量
elif name == 'stringConstant':
logger.info('=======stringConstant========{}'.format(text))
self.handle_string(text)
# 如果是复合表达式
elif text == '(':
expression = term.find('expression', recursive=False)
self.compile_expression(expression)
# 如果是方法调用
elif name == 'identifier' and length > 1 and (next_text == '('
or next_text == '.'):
self.handle_subroutine_call(next_text, term)
# 如果是变量
elif name == 'identifier' and length == 1:
self.deal_single_var(text)
# todo 如果是数组变量
elif name == 'identifier' and length > 1 and next_text == '[':
self.handle_array(term)
elif name == 'keyword' and text in ['true', 'false', 'null', 'this']:
self.headle_keyword(text)
else:
logger.error('无法识别此表达式{}'.format(term))
raise Exception('无法识别此表达式{}'.format(term))
while True: # in case of connection lost
try:
spider.save()
break
except Exception as e:
print e.message, uid
if 'Lost connection to MySQL server during query' in e.message:
continue
else:
break
crawled_list.append(uid)
spider.end_time = datetime.now()
duration = spider.end_time - spider.start_time
if duration.seconds > ACCOUNT_CHANGE_TIME:
spider.main_fetcher = loop_increase(
spider.main_fetcher, len(spider.fetchers))
spider.start_time = datetime.now()
emphasis_print('Account changed!!!')
print 'Complete a batch of tasks!'
print 'Getting new tasks...'
uid_list = get_tasks(TASK_NUM)
if len(uid_list) == 0:
print 'No tasks to proceed!'
exit(-1)
except Exception as e:
print e.message
log.error('Problematic UID: %s' % (uid, ))
finally:
reset(user_list, uid_list, crawled_list) # reset
while True: # in case of connection lost
try:
spider.save()
break
except Exception as e:
print e.message, uid
if 'Lost connection to MySQL server during query' in e.message:
continue
else:
break
crawled_list.append(uid)
spider.end_time = datetime.now()
duration = spider.end_time - spider.start_time
if duration.seconds > ACCOUNT_CHANGE_TIME:
spider.main_fetcher = loop_increase(spider.main_fetcher, len(spider.fetchers))
spider.start_time = datetime.now()
emphasis_print('Account changed!!!')
print 'Complete a batch of tasks!'
print 'Getting new tasks...'
uid_list = get_tasks(TASK_NUM)
if len(uid_list) == 0:
print 'No tasks to proceed!'
exit(-1)
except Exception as e:
print e.message
log.error('Problematic UID: %s' % (uid, ))
finally:
reset(user_list, uid_list, crawled_list) # reset
try:
logger.info('Waiting for new connection')
# Wait for a connection
self.__connSocket, client_address = self.__listenSocket.accept(
)
logger.debug('New Connection from ' + client_address[0])
self.connOpen = True
self.connOpenCallback()
if not self._receive_data():
logger.info('Connection closed')
self.connClosedCallback()
except socket.error:
logger.error('Exception accepting new connection')
self.stop()
def _receive_data(self):
rx_error = False
while self.status and not rx_error:
try:
rx_bytes = bytearray(self.__connSocket.recv(RX_BUFFER_SIZE))
if rx_bytes is not None:
self.dataReadCallback(rx_bytes)
else:
# Connection closed
rx_error = True
except socket.error:
