def tuling_robot(msg):
#使用图灵接口获取自动回复信息
data = {
'reqType': 0,
'perception': {
"inputText": {
"text": msg.raw.content
},
},
'userInfo': {
"apiKey": TULING_KEY,
"userId": Util.GetMd5(msg.from_id.id)
}
}
try:
robot_ret = eval(
Util.post(TULING_HOST, TULING_API, json.dumps(data)).decode())
logger.debug('tuling api 返回:{}'.format(robot_ret))
#自动回消息
interface.new_send_msg(
msg.from_id.id,
robot_ret['results'][0]['values']['text'].encode(encoding="utf-8"))
except:
logger.info('tuling api 调用异常!')
return
def post(self, test_name):
import tornado.httpclient
mongo=MongoDB()
test=mongo.db.test.find_one({"test_user_id": self.current_user_id,"test_name":test_name})
if "testset" not in test:
self.flash("error","No testset assign for this test.")
self.redirect("/#"+test_name)
return
xml=self.readfile(test["execution_config_file"])
t=template.Template(xml)
xml=t.generate(testset_ax_id=test["testset"])
try:
body=self.send_ax_request("execution","POST",retry=1, content_type="text/xml",body=xml)
from lxml import etree
logger.debug(body)
doc=etree.fromstring(body)
exec_dict={}
exec_dict.update(doc.attrib)
exec_dict['test_name']=test_name
exec_dict['user_id']=self.current_user_id
exec_dict['test']=str(test['_id'])
mongo.db.execution.insert(exec_dict)
self.flash("success","The test kicks off successfully")
except Exception as e:
try:
doc=etree.fromstring(e.message)
error=doc.attrib["error"]
except:
error=e.message
self.flash("error",error)
self.redirect("/#"+test_name)
def traffic_feature_anomaly_detect_by_port(path, direction):
ANOMALY_SCORE_COL_NAME = 'anomaly_score'
ANOMALY_EVENT_TYPE = 'abnormal_event_type'
normal_data, data = read_data(path)
final_data = pd.DataFrame()
final_data['time'] = data.time
final_data['ip'] = data.host_ip
final_data['dest_port'] = data['dest_port']
final_data['cor_ip_list'] = data['cor_ip_list']
final_data[ANOMALY_SCORE_COL_NAME] = 0
final_data['direction'] = direction
final_data['raw_values'] = data.ANF
final_data[ANOMALY_EVENT_TYPE] = -1
port_list = list(pd.unique(data.prot, data.dest_port))
for port in port_list:
logger.debug('##################################')
logger.debug('anomaly for ' + direction + ' data ' + str(port))
select_indices = list(np.where(data.dest_port == port)[0])
local_normal_data = normal_data.iloc[select_indices]
local_data = local_normal_data
index_col, score = feature_anomaly_score_computation(
local_normal_data, local_data)
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_SCORE_COL_NAME)] = score
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_EVENT_TYPE)] = index_col
return final_data
def UnPack(src,key = b''):
if len(src) < 0x20:
raise RuntimeError('Unpack Error!Please check mm protocol!') #协议需要更新
return b''
if not key:
key = Util.sessionKey
#解析包头
nCur= 0
if src[nCur] == struct.unpack('>B',b'\xbf')[0]:
nCur += 1 #跳过协议标志位
nLenHeader = src[nCur] >> 2 #包头长度
bUseCompressed = (src[nCur] & 0x3 == 1) #包体是否使用压缩算法:01使用,02不使用
nCur += 1
nDecryptType = src[nCur] >> 4 #解密算法(固定为AES解密): 05 aes解密 / 07 rsa解密
nLenCookie = src[nCur] & 0xf #cookie长度
nCur += 1
nCur += 4 #服务器版本(当前固定返回4字节0)
uin= struct.unpack('>i',src[nCur:nCur+4])[0] #uin
nCur += 4
cookie_temp = src[nCur:nCur+nLenCookie] #cookie
if cookie_temp and not(cookie_temp == Util.cookie):
Util.cookie = cookie_temp #刷新cookie
nCur += nLenCookie
(nCgi,nCur) = decoder._DecodeVarint(src,nCur) #cgi type
(nLenProtobuf,nCur) = decoder._DecodeVarint(src,nCur) #压缩前protobuf长度
(nLenCompressed,nCur) = decoder._DecodeVarint(src,nCur) #压缩后protobuf长度
logger.debug('包头长度:{}\n是否使用压缩算法:{}\n解密算法:{}\ncookie长度:{}\nuin:{}\ncookie:{}\ncgi type:{}\nprotobuf长度:{}\n压缩后protobuf长度:{}'.format(nLenHeader, bUseCompressed, nDecryptType, nLenCookie, uin, str(Util.cookie), nCgi, nLenProtobuf, nLenCompressed))
#对包体aes解密解压缩
body = src[nLenHeader:] #取包体数据
if bUseCompressed:
protobufData = Util.decompress_and_aesDecrypt(body,key)
else:
protobufData = Util.aesDecrypt(body,key)
logger.debug('解密后数据:%s' % str(protobufData))
return protobufData
def pack(src,cgi_type,use_compress = 0):
#必要参数合法性判定
if not Util.cookie or not Util.uin or not Util.sessionKey:
return b''
#压缩加密
len_proto_compressed = len(src)
if use_compress:
(body,len_proto_compressed) = Util.compress_and_aes(src,Util.sessionKey)
else:
body = Util.aes(src,Util.sessionKey)
logger.debug("cgi:{},protobuf数据:{}\n加密后数据:{}".format(cgi_type,Util.b2hex(src),Util.b2hex(body)))
#封包包头
header = bytearray(0)
header += b'\xbf' #标志位(可忽略该字节)
header += bytes([0]) #最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算 #
header += bytes([((0x5<<4) + 0xf)]) #05:AES加密算法 0xf:cookie长度(默认使用15字节长的cookie)
header += struct.pack(">I",define.__CLIENT_VERSION__) #客户端版本号 网络字节序
header += struct.pack(">i",Util.uin) #uin
header += Util.cookie #coockie
header += encoder._VarintBytes(cgi_type) #cgi type
header += encoder._VarintBytes(len(src)) #body proto压缩前长度
header += encoder._VarintBytes(len_proto_compressed) #body proto压缩后长度
header += bytes([0]*15) #3个未知变长整数参数,共15字节
header[1] = (len(header)<<2) + (1 if use_compress else 2) #包头长度
logger.debug("包头数据:{}".format(Util.b2hex(header)))
#组包
senddata = header + body
return senddata
def traffic_feature_anomaly_detect_by_host_prot_port(path, direction):
global host2GmmModel
host2GmmModel = {}
ANOMALY_SCORE_COL_NAME = 'anomaly_score'
ANOMALY_EVENT_TYPE = 'abnormal_event_type'
RAW_VALUE_COL_NAME = 'raw_values'
GMM_COL_NAME = 'gmm_params'
EVT_COL_NAME = 'evt_value'
normal_data, data = read_data(path)
final_data = pd.DataFrame()
final_data['time'] = data.time
final_data['ip'] = data.host_ip
final_data['prot'] = data.prot
final_data['dest_port'] = data['dest_port']
final_data['cor_ip_list'] = data['cor_ip_list']
final_data[ANOMALY_SCORE_COL_NAME] = 0
final_data['direction'] = direction
final_data[RAW_VALUE_COL_NAME] = 0
final_data[ANOMALY_EVENT_TYPE] = -1
final_data[GMM_COL_NAME] = 'None'
final_data[EVT_COL_NAME] = 0
ip_prot_port_list = list(
pd.unique(zip(data.host_ip, data.prot, data.dest_port)))
bar = Bar('Loading',
fill='@',
suffix='%(percent)d%% %(avg).3fs',
max=len(ip_prot_port_list))
for ip, prot, port in ip_prot_port_list:
logger.debug('##################################')
logger.debug('anomaly for ' + direction + ' data ' + ip + ':' + prot +
' -> ' + str(port))
#local_normal_data,best_model,local_data,select_indices = slice_data_by_ip_prot_port_padding(normal_data,data,ip,prot,port)
local_normal_data, best_model, local_data, select_indices = slice_data_by_ip_prot_port(
normal_data, data, ip, prot, port)
#index_col,score,value_vector = feature_anomaly_score_computation_compact(ip,local_normal_data,local_data)
index_col, score, value_vector, gmm_params, pv_vector = feature_anomaly_score_computation_given_model_save_p_v(
ip, prot, port, local_normal_data, best_model, local_data)
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_SCORE_COL_NAME)] = score
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_EVENT_TYPE)] = index_col.values
final_data.iloc[
select_indices,
final_data.columns.get_loc(RAW_VALUE_COL_NAME)] = value_vector
final_data.iloc[select_indices,
final_data.columns.get_loc(GMM_COL_NAME)] = gmm_params
final_data.iloc[select_indices,
final_data.columns.get_loc(EVT_COL_NAME)] = pv_vector
bar.next()
bar.finish()
return final_data
def new_send_msg(to_wxid, msg_content, msg_type=1):
#组包
send_data = business.new_send_msg_req2buf(to_wxid, msg_content, msg_type)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newsendmsg', send_data)
logger.debug('new_send_msg返回数据:' + Util.b2hex(ret_bytes))
#解包
return business.new_send_msg_buf2resp(ret_bytes)
def Login(name, password):
#组包
(senddata, login_aes_key) = business.login_req2buf(name, password)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/manualauth', senddata)
logger.debug('返回数据:' + str(ret_bytes))
#解包
return business.login_buf2Resp(ret_bytes, login_aes_key)
def new_sync():
#组包
send_data = business.new_sync_req2buf()
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newsync', send_data)
logger.debug('new_sync返回数据:' + str(ret_bytes))
#解包
return business.new_sync_buf2resp(ret_bytes)
def writeDataToRecord(self, dict, type='boeData'):
with open(self._getRecordingPath, "r", encoding="UTF-8") as f:
rData = json.load(f)
logger.info('{} 的值为：{}'.format(type, rData.get(type)))
if rData.get(type) == None:
rData.update({type:{}})
rData[type].update(dict)
data = rData
with open(self._getRecordingPath, "w", encoding="UTF-8") as f:
json.dump(data, f)
logger.debug('记录文件信息, 文件信息为 -> {} : {}'.format(type, dict))
def new_send_msg_buf2resp(buf):
#解包
res = mm_pb2.new_send_msg_resp()
res.ParseFromString(UnPack(buf))
#消息发送结果
if res.res.code:
logger.info('消息发送失败,错误码:{}'.format(res.res.code)) #-44被删好友,-22被拉黑;具体提示系统会发type=10000的通知
else:
logger.debug('消息发送成功,svrid:{}'.format(res.res.svrid))
return (res.res.code,res.res.svrid)
def new_init():
#组包
send_data = business.new_init_req2buf()
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newinit',send_data)
logger.debug('返回数据:' + str(ret_bytes))
#解包
business.new_init_buf2resp(ret_bytes)
return
def send_app_msg(wxid, title, des, link_url, thumb_url=''):
#组包
send_data = business.send_app_msg_req2buf(wxid, title, des, link_url,
thumb_url)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/sendappmsg', send_data)
logger.debug('send_app_msg返回数据:' + Util.b2hex(ret_bytes))
#解包
return business.send_app_msg_buf2resp(ret_bytes)
def sync_done_req2buf():
#取sync key
sync_key = mm_pb2.SyncKey()
sync_key.ParseFromString(Util.get_sync_key())
#包体
body = sync_key.msgkey.SerializeToString()
#包头:固定8字节,网络字节序;4字节本地时间与上次newsync时间差(us);4字节protobuf长度
header = b'\x00\x00\x00\xff' + struct.pack(">I",len(body))
#组包
send_data = header + body
logger.debug('report kv数据:{}'.format(Util.b2hex(send_data)))
return send_data
def new_init():
continue_flag = True
cur = max = b''
while continue_flag:
#组包
send_data = business.new_init_req2buf(cur, max)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newinit', send_data)
logger.debug('new_init返回数据:' + str(ret_bytes))
#解包
(continue_flag, cur, max) = business.new_init_buf2resp(ret_bytes)
return
def new_send_msg(to_wxid, msg_content, msg_type=1):
#组包
send_data = business.new_send_msg_req2buf(to_wxid, msg_content, msg_type)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newsendmsg', send_data)
logger.debug('new_send_msg返回数据:' + Util.b2hex(ret_bytes))
#解包
(ret_code, svrid) = business.new_send_msg_buf2resp(ret_bytes)
#消息记录存入数据库
Util.insert_msg_to_db(svrid, Util.get_utc(), Util.wxid, to_wxid, msg_type,
msg_content.decode())
#返回发送消息结果
return ret_code
def switchWindow(self):
num = 0
while True:
if len(self.getWindowHandles()) > 1:
break
if num > 1000:
break
num = num + 1
logger.debug("所有窗口为：{}".format(self.getWindowHandles()))
windowsList = self.getWindowHandles()
index = 0
for i in range(len(self.getWindowHandles())):
if len(self.getWindowHandles()) == 1:
break
if self.getCurrentWindowHandle() != self.getWindowHandles()[i]:
index = i
self.switchToWin(windowsList[index])
logger.debug("当前窗口为：{}".format(self.getCurrentWindowHandle()))
def traffic_feature_anomaly_detect_by_prot_port(path, direction):
ANOMALY_SCORE_COL_NAME = 'anomaly_score'
ANOMALY_EVENT_TYPE = 'abnormal_event_type'
RAW_VALUE_COL_NAME = 'raw_values'
normal_data, data = read_data(path)
final_data = pd.DataFrame()
final_data['time'] = data.time
final_data['ip'] = data.host_ip
final_data['prot'] = data.prot
final_data['dest_port'] = data['dest_port']
final_data['cor_ip_list'] = data['cor_ip_list']
final_data[ANOMALY_SCORE_COL_NAME] = 0
final_data['direction'] = direction
final_data[RAW_VALUE_COL_NAME] = 0
final_data[ANOMALY_EVENT_TYPE] = -1
prot_port_list = list(pd.unique(zip(data.prot, data.dest_port)))
for prot, port in prot_port_list:
logger.debug('##################################')
logger.debug('anomaly for ' + direction + ' data ' + prot + ' -> ' +
str(port))
select_indices = list(
np.where((data.dest_port == port) & (data.prot == prot))[0])
select_normal_indices = list(
np.where((normal_data.dest_port == port)
& (normal_data.prot == prot))[0])
local_normal_data = normal_data.iloc[select_normal_indices]
if local_normal_data.shape[0] == 1:
local_normal_data = normal_data
local_data = data.iloc[select_indices]
index_col, score, value_vector = feature_anomaly_score_computation(
local_normal_data, local_data)
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_SCORE_COL_NAME)] = score
final_data.iloc[
select_indices,
final_data.columns.get_loc(ANOMALY_EVENT_TYPE)] = index_col.values
final_data.iloc[
select_indices,
final_data.columns.get_loc(RAW_VALUE_COL_NAME)] = value_vector
return final_data
def send_app_msg(to_wxid, title, des, link_url, thumb_url=''):
#组包
(send_data,
msg_content) = business.send_app_msg_req2buf(to_wxid, title, des,
link_url, thumb_url)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/sendappmsg', send_data)
logger.debug('send_app_msg返回数据:' + Util.b2hex(ret_bytes))
#解包
(ret_code, svrid) = business.send_app_msg_buf2resp(ret_bytes)
#消息记录存入数据库
Util.insert_msg_to_db(svrid, Util.get_utc(), Util.wxid, to_wxid, 5,
msg_content)
#返回发送消息结果
return ret_code
def run(name, password):
#连接服务器
longlink.settimeout(HEARTBEAT_TIMEOUT + 1)
longlink.connect((Util.ip['longip'], 443))
#发送心跳
if send_heartbeat():
recv_data = longlink.recv(BUFFSIZE)
logger.info('服务器返回心跳数据:{}'.format(Util.b2hex(recv_data)))
#登录
global login_aes_key
(login_buf, login_aes_key) = business.login_req2buf(name, password)
send_data = pack(CMDID_MANUALAUTH_REQ, login_buf)
longlink.send(send_data)
#死循环recv
recv_data = b''
while True:
try:
recv_data += longlink.recv(BUFFSIZE)
except Exception as e:
if str(e) == 'timed out':
#发送心跳
send_heartbeat()
continue
else:
raise RuntimeError('recv Error!!!')
logger.debug('收到数据:{}'.format(Util.b2hex(recv_data)))
(ret, buf) = unpack(recv_data)
if UNPACK_OK == ret:
(ret, buf) = unpack(buf)
while UNPACK_OK == ret:
(ret, buf) = unpack(buf)
recv_data = buf
#刷新心跳
send_heartbeat()
elif UNPACK_CONTINUE == ret:
pass
longlink.close()
return
def new_init_buf2resp(buf):
#解包
res = mm_pb2.NewInitResponse()
res.ParseFromString(UnPack(buf))
#newinit后保存sync key
Util.set_sync_key(res.sync_key_cur) #newinit结束前不要异步调用newsync
logger.debug('newinit sync_key_cur len:{}\ndata:{}'.format(len(res.sync_key_cur), Util.b2hex(res.sync_key_cur)))
logger.debug('newinit sync_key_max len:{}\ndata:{}'.format(len(res.sync_key_max), Util.b2hex(res.sync_key_max)))
#初始化数据
logger.info('newinit cmd数量:{},是否需要继续初始化:{}'.format(res.cntList,res.continue_flag))
#初始化
for i in range(res.cntList):
if 5 == res.tag7[i].type: #未读消息
msg = mm_pb2.Msg()
msg.ParseFromString(res.tag7[i].data.data)
if 10002 == msg.type or 9999 == msg.type: #过滤系统垃圾消息
continue
else:
#将消息存入数据库
Util.insert_msg_to_db(msg.serverid,msg.createTime,msg.from_id.id,msg.to_id.id,msg.type,msg.raw.content)
logger.info('收到新消息:\ncreate utc time:{}\ntype:{}\nfrom:{}\nto:{}\nraw data:{}\nxml data:{}'.format(Util.utc_to_local_time(msg.createTime), msg.type, msg.from_id.id, msg.to_id.id, msg.raw.content, msg.xmlContent))
elif 2 == res.tag7[i].type: #好友列表
friend = mm_pb2.contact_info()
friend.ParseFromString(res.tag7[i].data.data)
#过滤系统wxid
if friend.wxid.id in define.MM_DEFAULT_WXID:
logger.info('更新好友信息:跳过默认wxid[{}]'.format(friend.wxid.id))
continue
#好友分类
if friend.wxid.id.endswith('@chatroom'): #群聊
logger.info('更新好友信息:群聊名:{} 群聊wxid:{} chatroom_serverVer:{} chatroom_max_member:{} 群主:{} 群成员数量:{}'.format(friend.nickname.name,friend.wxid.id,friend.chatroom_serverVer,friend.chatroom_max_member,friend.chatroomOwnerWxid,friend.group_member_list.cnt))
elif friend.wxid.id.startswith('gh_'): #公众号
logger.info('更新好友信息:公众号:{} 公众号wxid:{} alias:{} 注册主体:{}'.format(friend.nickname.name,friend.wxid.id,friend.alias,friend.register_body if friend.register_body_type == 24 else '个人'))
else: #好友
logger.info('更新好友信息:昵称:{} 备注名:{} wxid:{} alias:{} 性别:{} 好友来源:{} 个性签名:{}'.format(friend.nickname.name,friend.remark_name.name,friend.wxid.id,friend.alias,friend.sex,Util.get_way(friend.src),friend.qianming))
#将好友信息存入数据库
Util.insert_contact_info_to_db(friend.wxid.id,friend.nickname.name,friend.remark_name.name,friend.alias,friend.avatar_big,friend.v1_name,friend.type,friend.sex,friend.country,friend.sheng,friend.shi,friend.qianming,friend.register_body,friend.src,friend.chatroomOwnerWxid,friend.chatroom_serverVer,friend.chatroom_max_member,friend.group_member_list.cnt)
return (res.continue_flag,res.sync_key_cur,res.sync_key_max)
def new_sync_buf2resp(buf):
#解包
res = mm_pb2.new_sync_resp()
res.ParseFromString(Util.UnPack(buf))
#刷新sync key
Util.set_sync_key(res.sync_key)
logger.debug('sync key len:{}\ndata:{}'.format(len(res.sync_key), Util.b2hex(res.sync_key)))
#未读消息
for i in range(res.msg.cntList):
if 5 == res.msg.tag2[i].type: #未读消息
msg = mm_pb2.Msg()
msg.ParseFromString(res.msg.tag2[i].data.data)
if 10002 == msg.type or 9999 == msg.type: #过滤系统垃圾消息
continue
else:
#将消息存入数据库
Util.insert_msg_to_db(msg.serverid,msg.createTime,msg.from_id.id,msg.to_id.id,msg.type,msg.raw.content)
logger.info('收到新消息:\ncreate utc time:{}\ntype:{}\nfrom:{}\nto:{}\nraw data:{}\nxml data:{}'.format(Util.utc_to_local_time(msg.createTime), msg.type, msg.from_id.id, msg.to_id.id, msg.raw.content, msg.xmlContent))
return
def pack(cmd_id, buf=b''):
global seq
header = bytearray(0)
header += struct.pack(">I", len(buf) + 16) #封包总长度(含包头) 4字节
header += b'\x00\x10' #包头长度 2字节 固定00 10(包头长16字节)
header += b'\x00\x01' #协议版本 固定00 01
header += struct.pack(">I", cmd_id) #cmd_id 4字节 不同cgi对应不同cmd_id
if CMDID_NOOP_REQ == cmd_id:
header += struct.pack(">I", HEARTBEAT_SEQ) #心跳包
elif CMDID_IDENTIFY_REQ == cmd_id:
header += struct.pack(
">I", IDENTIFY_SEQ
) #登录确认包(暂未实现;该请求确认成功后服务器会直接推送消息内容,否则服务器只下发推送通知,需要主动同步消息)
else:
header += struct.pack(">I", seq) #封包编号 4字节 (除心跳-1，推送0外,其余消息从1每次自增1)
#封包编号自增
seq += 1
logger.debug('长链接包头:{}'.format(Util.b2hex(header)))
return header + buf
def new_init():
#protobuf组包
new_init_request = mm_pb2.NewInitRequest(
login = mm_pb2.LoginInfo(
aesKey = Util.sessionKey,
uin = 0,
guid = define.__GUID__ + '\0', #guid以\0结尾
clientVer = define.__CLIENT_VERSION__,
androidVer = define.__ANDROID_VER__,
unknown = 1,
),
wxid = Util.wxid,
tag3 = mm_pb2.mmStr(),
tag4 = mm_pb2.mmStr(),
language = define.__LANGUAGE__,
)
#组包
send_data = Util.pack(new_init_request.SerializeToString(),139)
#发包
ret_bytes = Util.mmPost('/cgi-bin/micromsg-bin/newinit',send_data)
logger.debug('返回数据:' + str(ret_bytes))
#解包
res = mm_pb2.NewInitResponse()
res.ParseFromString(Util.UnPack(ret_bytes))
#newinit后保存sync key
Util.sync_key = res.synckeybytes.key
logger.debug('sync key len:{}\ndata:{}'.format(res.synckeybytes.len, Util.b2hex(res.synckeybytes.key)))
#初始化数据
logger.debug('tag7数量:{}'.format(res.cntList))
#未读消息
for i in range(res.cntList):
if 5 == res.tag7[i].type: #未读消息
msg = mm_pb2.Msg()
msg.ParseFromString(res.tag7[i].data.data)
if 10002 == msg.type or 9999 == msg.type: #过滤系统垃圾消息
continue
else:
logger.info('收到新消息:\ncreate utc time:{}\ntype:{}\nfrom:{}\nto:{}\nraw data:{}\nxml data:{}'.format(msg.createTime, msg.type, msg.from_id.id, msg.to_id.id, msg.raw.content, msg.xmlContent))
return
def test_login(self, publciLoginPage_testdata):
''' --------------------测试登陆时不同场景-------------------- '''
logger.info("测试用例名字为：{}".format(publciLoginPage_testdata["casename"]))
allure.title(publciLoginPage_testdata["casename"])
with allure.step("第一步，登录页面初始化"):
self.publicloginpage.refresh()
with allure.step("第二步，输入账号/密码"):
self.publicloginpage.input_account(publciLoginPage_testdata["account"])
logger.debug("输入账户为：{}".format(publciLoginPage_testdata["account"]))
self.publicloginpage.input_password(publciLoginPage_testdata["password"])
logger.debug("输入密码为：{}".format(publciLoginPage_testdata["password"]))
with allure.step("第三步，点击登录按钮"):
self.publicloginpage.click_loginbutton()
with allure.step("第四步，判断断言信息"):
try:
if (publciLoginPage_testdata["casename"] == '账号密码不匹配') or (publciLoginPage_testdata["casename"] =='当前登录用户名或密码错误'):
WebDriverWait(self.publicloginpage.driver, 5).until(
EC.visibility_of_element_located(self.publicloginpage.getMessageBox()))
msg = self.publicloginpage.get_errortext()
assert (publciLoginPage_testdata["casename"] in msg)
elif publciLoginPage_testdata["casename"] == '请输入账户名称':
WebDriverWait(self.publicloginpage.driver, 5).until(
EC.text_to_be_present_in_element(self.publicloginpage.getAccountInputErrorBox(),publciLoginPage_testdata["casename"]))
msg = self.publicloginpage.get_accounterrortext()
assert publciLoginPage_testdata["casename"] == msg
elif publciLoginPage_testdata["casename"] == '请输入密码':
WebDriverWait(self.publicloginpage.driver, 5).until(
EC.text_to_be_present_in_element(self.publicloginpage.getPasswordInputErrorBox(),publciLoginPage_testdata["casename"]))
msg = self.publicloginpage.get_passworderrortext()
assert publciLoginPage_testdata["casename"] == msg
elif publciLoginPage_testdata["casename"] == '登陆成功':
WebDriverWait(self.publicloginpage.driver, 5).until(
EC.visibility_of_element_located(self.publicloginpage.getIntoButton()))
self.publicloginpage.get_into()
self.publicloginpage.driver.refresh()
assert 1 == 1
except Exception as e:
logger.error("出现异常，异常信息为：{}".format(type(e)))
code = 'wrong'
timeNow = getNowTime()
self.publicloginpage.screenshot(code, timeNow)
allure.attach.file(getPicturePath(code,timeNow),name=timeNow + code + "screenshot",
attachment_type=allure.attachment_type.PNG)
assert 1 == 0
else:
logger.info("测试用例执行成功")
code = 'success'
timeNow = getNowTime()
self.publicloginpage.screenshot(code, timeNow)
allure.attach.file(getPicturePath(code, timeNow), name=timeNow + code + "screenshot",
attachment_type=allure.attachment_type.PNG)
def login_req2buf(name,password):
#随机生成16位登录包AesKey
login_aes_key = bytes(''.join(random.sample(string.ascii_letters + string.digits, 16)), encoding = "utf8")
#protobuf组包1
accountRequest = mm_pb2.ManualAuthAccountRequest(
aes = mm_pb2.ManualAuthAccountRequest.AesKey(
len = 16,
key = login_aes_key
),
ecdh = mm_pb2.ManualAuthAccountRequest.Ecdh(
nid = 713,
ecdhKey = mm_pb2.ManualAuthAccountRequest.Ecdh.EcdhKey(
len = len(Util.EcdhPubKey),
key = Util.EcdhPubKey
)
),
userName = name,
password1 = Util.GetMd5(password),
password2 = Util.GetMd5(password)
)
#protobuf组包2
deviceRequest = mm_pb2.ManualAuthDeviceRequest(
login = mm_pb2.LoginInfo(
aesKey = login_aes_key,
uin = 0,
guid = define.__GUID__ + '\0', #guid以\0结尾
clientVer = define.__CLIENT_VERSION__,
androidVer = define.__ANDROID_VER__,
unknown = 1,
),
tag2 = mm_pb2.ManualAuthDeviceRequest._Tag2(),
imei = define.__IMEI__,
softInfoXml = define.__SOFTINFO__.format(define.__IMEI__,define.__ANDROID_ID__, define.__MANUFACTURER__+" "+define.__MODELNAME__, define.__MOBILE_WIFI_MAC_ADDRESS__, define.__CLIENT_SEQID_SIGN__, define.__AP_BSSID__, define.__MANUFACTURER__,"taurus", define.__MODELNAME__, define.__IMEI__),
unknown5 = 0,
clientSeqID = define.__CLIENT_SEQID__,
clientSeqID_sign = define.__CLIENT_SEQID_SIGN__,
loginDeviceName = define.__MANUFACTURER__+" "+define.__MODELNAME__,
deviceInfoXml = define.__DEVICEINFO__.format(define.__MANUFACTURER__, define.__MODELNAME__),
language = define.__LANGUAGE__,
timeZone = "8.00",
unknown13 = 0,
unknown14 = 0,
deviceBrand = define.__MANUFACTURER__,
deviceModel = define.__MODELNAME__+"armeabi-v7a",
osType = define.__ANDROID_VER__,
realCountry = "cn",
unknown22 = 2, #Unknown
)
logger.debug("accountData protobuf数据:" + str(accountRequest.SerializeToString()))
logger.debug("deviceData protobuf数据:" + str(deviceRequest.SerializeToString()))
#加密
reqAccount = Util.compress_and_rsa(accountRequest.SerializeToString())
reqDevice = Util.compress_and_aes(deviceRequest.SerializeToString(),login_aes_key)
logger.debug("加密后数据长度:reqAccount={},reqDevice={}".format(len(reqAccount),len(reqDevice[0])))
logger.debug("加密后reqAccount数据:" + str(reqAccount))
logger.debug("加密后reqDevice数据:" + str(reqDevice[0]))
#封包包体
subheader = b''
subheader += struct.pack(">I",len(accountRequest.SerializeToString())) #accountData protobuf长度
subheader += struct.pack(">I",len(deviceRequest.SerializeToString())) #deviceData protobuf长度
subheader += struct.pack(">I",len(reqAccount)) #accountData RSA加密后长度
body = subheader + reqAccount + reqDevice[0] #包体由头信息、账号密码加密后数据、硬件设备信息加密后数据3部分组成
#封包包头
header = bytearray(0)
header += bytes([0]) #最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算 #
header += bytes([((0x7<<4) + 0xf)]) #07:RSA加密算法 0xf:cookie长度
header += struct.pack(">I",define.__CLIENT_VERSION__) #客户端版本号 网络字节序
header += bytes([0]*4) #uin
header += bytes([0]*15) #coockie
header += encoder._VarintBytes(701) #cgi type
header += encoder._VarintBytes(len(body)) #body 压缩前长度
header += encoder._VarintBytes(len(body)) #body 压缩后长度(登录包不需要压缩body数据)
header += struct.pack(">B",define.__LOGIN_RSA_VER__) #RSA秘钥版本
header += b'\x01\x02' #Unknown Param
header[0] = (len(header)<<2) + 2 #包头长度
#组包
logger.debug('包体数据:' + str(body))
logger.debug('包头数据:' + str(header))
senddata = header + body
return (senddata,login_aes_key)
def read_groundtruth():
path = PATH
data = read_ctu13_data(path)
data = data[data.Label.str.contains('Botnet')]
data = data[data.Proto == 'tcp']
spam_data = data[data.Dport == '25']
logger.debug('Spam Attacker IPs')
unique_attacker = Counter(spam_data.SrcAddr)
logger.debug(dict_to_sorted_list(unique_attacker))
logger.debug('Spam Victim IPs')
unique_spam_victims = Counter(spam_data.DstAddr)
logger.debug(dict_to_sorted_list(unique_spam_victims))
cf_data = data[data.Dport == '80']
logger.debug('ClickFraud Attacker IPs')
unique_cf_attacker = Counter(cf_data.SrcAddr)
logger.debug(dict_to_sorted_list(unique_cf_attacker))
logger.debug('ClickFraud Victim IPs')
unique_cf_victims = Counter(cf_data.DstAddr)
logger.debug(dict_to_sorted_list(unique_cf_victims))
irc_data = data[data.Dport == '6667']
logger.debug('IRC channel connector IPs')
unique_irc_user = Counter(irc_data.SrcAddr)
logger.debug(dict_to_sorted_list(unique_irc_user))
logger.debug('IRC channel server IPs')
unique_irc_server = Counter(irc_data.DstAddr)
logger.debug(dict_to_sorted_list(unique_irc_server))
return unique_attacker, unique_spam_victims, unique_cf_attacker, unique_cf_victims, unique_irc_user, unique_irc_server
def traffic_hosts_correlate(reduced_anomaly_list):
print 'correlating traffic'
global node_idx_map
source_data_map = reduced_anomaly_list['source_host']
dest_data_map = reduced_anomaly_list['dest_host']
source_host_list = source_data_map.keys()
dest_host_list = dest_data_map.keys()
source2dest_map_s = get_source_dest_mapping(source_data_map, 'src')
source2dest_map_d = get_source_dest_mapping(dest_data_map, 'dest')
print 'source dict'
print len(source2dest_map_s.keys())
print 'dest dict'
print len(source2dest_map_d.keys())
data_map = list(
set(source2dest_map_s.keys()) & set(source2dest_map_d.keys()))
print len(data_map)
bar = Bar('Loading',
fill='@',
suffix='%(percent)d%% %(avg).3fs seconds per loop',
max=len(data_map))
correlation_result = []
for host_a, host_b in data_map:
if host_a == host_b:
bar.next()
continue
host_a_event_list = source_data_map[host_a]
host_b_event_list = dest_data_map[host_b]
host_a_ip = host_a.split('_')[1]
host_b_ip = host_b.split('_')[1]
logger.debug('host_a_ip: ' + str(host_a_ip))
logger.debug('host_b_ip: ' + str(host_b_ip))
events_with_b_in_a = [
e for e in host_a_event_list if (host_b_ip in e['S'][3])
]
events_with_a_in_b = [
e for e in host_b_event_list if (host_a_ip in e['S'][3])
]
#events_with_b_in_a.sort(key=lambda x: time_array_to_datetime(x['T']), reverse=False)
#events_with_a_in_b.sort(key=lambda x: time_array_to_datetime(x['T']), reverse=False)
time_begin_2_index_map_a = {}
time_begin_2_index_map_b = {}
for index, elem in enumerate(events_with_b_in_a):
time_begin_2_index_map_a[get_unique_index_of_event(elem)] = index
for index, elem in enumerate(events_with_a_in_b):
time_begin_2_index_map_b[get_unique_index_of_event(elem)] = index
time_port_b_in_a = set(time_begin_2_index_map_a.keys())
time_port_a_in_b = set(time_begin_2_index_map_b.keys())
x = [time_port_b_in_a, time_port_a_in_b]
time_series_intersection = set.intersection(*x)
logger.info(str(host_a_ip) + ' -> ' + str(host_b_ip))
logger.info(
str(time_port_b_in_a) + ' <-> ' + str(time_port_a_in_b) +
' intersection: ' + str(time_series_intersection))
data_genre = 'traffic'
source_I = host_a_event_list[0]['I']
target_I = 'TD_' + str(node_idx_map[host_b])
time_series_intersection_list = list(time_series_intersection)
time_series_intersection_list.sort(
key=lambda x: time_array_to_datetime(
get_time_array_from_event_index(x)),
reverse=False)
result = []
for event_index in time_series_intersection_list:
event_a = events_with_b_in_a[time_begin_2_index_map_a[event_index]]
event_b = events_with_a_in_b[time_begin_2_index_map_b[event_index]]
source_visual_time = {
'start': timelist_to_visual_time(event_a['T'][0]),
'end': timelist_to_visual_time(event_a['T'][1])
}
dest_visual_time = {
'start': timelist_to_visual_time(event_b['T'][0]),
'end': timelist_to_visual_time(event_b['T'][1])
}
pair_start_time = timelist_to_visual_time(event_b['T'][0])
pair_end_time = timelist_to_visual_time(event_b['T'][1])
sourceTI = event_a['TimeIndex']
destTI = event_b['TimeIndex']
result.append({
'data_genre': data_genre,
'source': source_I,
'sourceTimeIndex': sourceTI,
'sourceTime': source_visual_time,
'target': target_I,
'targetTimeIndex': destTI,
'targetTime': dest_visual_time,
'temporalValue': 0,
'spatialValue': 0,
'categoricalValue': 0,
'weightAbsolute': max(event_a['A'], event_b['A']),
'weightRelative': min(event_a['A'], event_b['A']),
'TimeStart': pair_start_time,
'TimeEnd': pair_end_time
})
correlation_record = {"source": source_I, \
"target": target_I, \
'EdgeType': data_genre, \
"timeSeries": result \
}
correlation_result.append(correlation_record)
bar.next()
bar.finish()
return correlation_result
def Login(name,password):
#随机生成16位登录包AesKey
login_aes_key = bytes(''.join(random.sample(string.ascii_letters + string.digits, 16)), encoding = "utf8")
#生成ECC key
if not Util.GenEcdhKey():
logger.info('生成ECC Key失败!')
return -1
#protobuf组包1
accountRequest = mm_pb2.ManualAuthAccountRequest(
aes = mm_pb2.ManualAuthAccountRequest.AesKey(
len = 16,
key = login_aes_key
),
ecdh = mm_pb2.ManualAuthAccountRequest.Ecdh(
nid = 713,
ecdhKey = mm_pb2.ManualAuthAccountRequest.Ecdh.EcdhKey(
len = len(Util.EcdhPubKey),
key = Util.EcdhPubKey
)
),
userName = name,
password1 = Util.GetMd5(password),
password2 = Util.GetMd5(password)
)
#protobuf组包2
deviceRequest = mm_pb2.ManualAuthDeviceRequest(
login = mm_pb2.LoginInfo(
aesKey = login_aes_key,
uin = 0,
guid = define.__GUID__ + '\0', #guid以\0结尾
clientVer = define.__CLIENT_VERSION__,
androidVer = define.__ANDROID_VER__,
unknown = 1,
),
tag2 = mm_pb2.ManualAuthDeviceRequest._Tag2(),
imei = define.__IMEI__,
softInfoXml = define.__SOFTINFO__.format(define.__IMEI__,define.__ANDROID_ID__, define.__MANUFACTURER__+" "+define.__MODELNAME__, define.__MOBILE_WIFI_MAC_ADDRESS__, define.__CLIENT_SEQID_SIGN__, define.__AP_BSSID__, define.__MANUFACTURER__,"taurus", define.__MODELNAME__, define.__IMEI__),
unknown5 = 0,
clientSeqID = define.__CLIENT_SEQID__,
clientSeqID_sign = define.__CLIENT_SEQID_SIGN__,
loginDeviceName = define.__MANUFACTURER__+" "+define.__MODELNAME__,
deviceInfoXml = define.__DEVICEINFO__.format(define.__MANUFACTURER__, define.__MODELNAME__),
language = define.__LANGUAGE__,
timeZone = "8.00",
unknown13 = 0,
unknown14 = 0,
deviceBrand = define.__MANUFACTURER__,
deviceModel = define.__MODELNAME__+"armeabi-v7a",
osType = define.__ANDROID_VER__,
realCountry = "cn",
unknown22 = 2, #Unknown
)
logger.debug("accountData protobuf数据:" + str(accountRequest.SerializeToString()))
logger.debug("deviceData protobuf数据:" + str(deviceRequest.SerializeToString()))
#加密
reqAccount = Util.compress_and_rsa(accountRequest.SerializeToString())
reqDevice = Util.compress_and_aes(deviceRequest.SerializeToString(),login_aes_key)
logger.debug("加密后数据长度:reqAccount={},reqDevice={}".format(len(reqAccount),len(reqDevice[0])))
logger.debug("加密后reqAccount数据:" + str(reqAccount))
logger.debug("加密后reqDevice数据:" + str(reqDevice[0]))
#封包包体
subheader = b''
subheader += struct.pack(">I",len(accountRequest.SerializeToString())) #accountData protobuf长度
subheader += struct.pack(">I",len(deviceRequest.SerializeToString())) #deviceData protobuf长度
subheader += struct.pack(">I",len(reqAccount)) #accountData RSA加密后长度
body = subheader + reqAccount + reqDevice[0] #包体由头信息、账号密码加密后数据、硬件设备信息加密后数据3部分组成
#封包包头
header = bytearray(0)
header += bytes([0]) #最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算 #
header += bytes([((0x7<<4) + 0xf)]) #07:RSA加密算法 0xf:cookie长度
header += struct.pack(">I",define.__CLIENT_VERSION__) #客户端版本号 网络字节序
header += bytes([0]*4) #uin
header += bytes([0]*15) #coockie
header += encoder._VarintBytes(701) #cgi type
header += encoder._VarintBytes(len(body)) #body 压缩前长度
header += encoder._VarintBytes(len(body)) #body 压缩后长度(登录包不需要压缩body数据)
header += struct.pack(">B",define.__LOGIN_RSA_VER__) #RSA秘钥版本
header += b'\x01\x02' #Unknown Param
header[0] = (len(header)<<2) + 2 #包头长度
#组包
logger.debug('包体数据:' + str(body))
logger.debug('包头数据:' + str(header))
senddata = header + body
#发包
loginRetBytes = Util.mmPost('/cgi-bin/micromsg-bin/manualauth',senddata)
logger.debug('返回数据:' + str(loginRetBytes))
#解包
loginRes = mm_pb2.ManualAuthResponse()
loginRes.result.code = -1
loginRes.ParseFromString(Util.UnPack(loginRetBytes,login_aes_key))
#登录异常处理
if -301 == loginRes.result.code: #DNS解析失败,请尝试更换idc
logger.info('登陆结果:\ncode:{}\n请尝试更换DNS重新登陆!'.format(loginRes.result.code))
elif -106 == loginRes.result.code: #需要在IE浏览器中滑动操作解除环境异常/扫码、短信、好友授权(滑动解除异常后需要重新登录一次)
logger.info('登陆结果:\ncode:{}\nError msg:{}\n'.format(loginRes.result.code,loginRes.result.err_msg.msg[loginRes.result.err_msg.msg.find('<Content><![CDATA[')+len('<Content><![CDATA['):loginRes.result.err_msg.msg.find(']]></Content>')]))
#打开IE,完成授权
logger.info('请在浏览器授权后重新登陆!')
Util.OpenIE(loginRes.result.err_msg.msg[loginRes.result.err_msg.msg.find('<Url><![CDATA[')+len('<Url><![CDATA['):loginRes.result.err_msg.msg.find(']]></Url>')])
elif loginRes.result.code: #其他登录错误
logger.info('登陆结果:\ncode:{}\nError msg:{}\n'.format(loginRes.result.code,loginRes.result.err_msg.msg[loginRes.result.err_msg.msg.find('<Content><![CDATA[')+len('<Content><![CDATA['):loginRes.result.err_msg.msg.find(']]></Content>')]))
else: #登陆成功
#密钥协商
Util.sessionKey = Util.aesDecrypt(loginRes.authParam.session.key,Util.DoEcdh(loginRes.authParam.ecdh.ecdhKey.key))
#保存uin/wxid
Util.uin = loginRes.authParam.uin
Util.wxid = loginRes.accountInfo.wxId
logger.info('登陆成功!\nsession_key:{}\nuin:{}\nwxid:{}\nnickName:{}\nalias:{}'.format(Util.sessionKey,Util.uin,Util.wxid,loginRes.accountInfo.nickName,loginRes.accountInfo.Alias))
return loginRes.result.code
def input_password(self, password):
self.clear(*self._passwordInput)
self.send_text(password, *self._passwordInput)
logger.debug("输入密码，密码为：{}".format(password))
def input_account(self, account):
self.clear(*self._accountInput)
self.send_text(account, *self._accountInput)
logger.debug("输入账户，账户为：{}".format(account))
