class FlowGenerator(object):
"""
The flow generator class.
It maintains two collection spaces, one represents the currently processing flows `current_flows`,
the other is the flows finished processing `finished_flows`.
Attributes
----------
current_flows : dict of {str: Flow}
The flows which are currently processing.
finished_flows : list of Flow
The flows which are considered as finished.
flow_timeout : int
The timeout of a flow.
Parameters
----------
flow_timeout: int
The timeout of a flow.
"""
def __init__(self, flow_timeout):
self.current_flows = {}
self.finished_flows = []
self.flow_timeout = flow_timeout
self.logger = MyLogger('FlowGenerator')
def add_packet(self, packet):
"""
Add the packet to current flow.
Parameters
----------
packet: BasicPacket
The processed `BasicPacket` object.
"""
if not packet:
return
forward_packet_flow_id = packet.forward_flow_id()
backward_packet_flow_id = packet.backward_flow_id()
current_timestamp = packet.timestamp
if forward_packet_flow_id not in self.current_flows and backward_packet_flow_id not in self.current_flows:
# A new flow begins
self.current_flows[forward_packet_flow_id] = Flow(packet, self.flow_timeout)
else:
if forward_packet_flow_id in self.current_flows:
flow = self.current_flows[forward_packet_flow_id]
else:
flow = self.current_flows[backward_packet_flow_id]
# Under the status of ACTIVE:
# 1. When received a packet and it is timeout, finish current flow and genenrate a new flow.
# 2. When received a forward FIN packet, turn to FIN_WAIT_1 status.
# 3. When received a backward FIN packet, turn to CLOSE_WAIT status.
# 4. When received a RST packet, finish current flow.
# 5. When received a normal packet, add the packet to current flow.
if flow.flow_status == FlowStatus.ACTIVE:
if current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
elif packet.hasFIN:
flow.add_packet(packet)
if packet.forward_flow_id() == flow.flow_id:
flow.flow_status = FlowStatus.FIN_WAIT_1
else:
flow.flow_status = FlowStatus.CLOSE_WAIT
elif packet.hasRST:
self.logger.debug(f'Received a RST packet: {flow.flow_id}')
flow.add_packet(packet)
self._move_flow_from_current_to_finished(flow)
else:
flow.add_packet(packet)
elif flow.flow_status == FlowStatus.FIN_WAIT_1:
flow.add_packet(packet)
if packet.hasFIN:
flow.flow_status = FlowStatus.CLOSING
else:
flow.flow_status = FlowStatus.FIN_WAIT_2
elif flow.flow_status == FlowStatus.FIN_WAIT_2:
flow.add_packet(packet)
if packet.hasFIN:
flow.flow_status = FlowStatus.TIME_WAIT
elif current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
elif flow.flow_status == FlowStatus.CLOSING:
flow.add_packet(packet)
if packet.hasACK:
self._move_flow_from_current_to_finished(flow)
elif current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
elif flow.flow_status == FlowStatus.CLOSE_WAIT:
flow.add_packet(packet)
if packet.backward_flow_id() == flow.flow_id and packet.hasFIN:
flow.flow_status = FlowStatus.LAST_ACK
elif current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
elif flow.flow_status == FlowStatus.LAST_ACK:
flow.add_packet(packet)
if packet.hasACK:
self._move_flow_from_current_to_finished(flow)
elif current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
elif flow.flow_status == FlowStatus.TIME_WAIT:
if current_timestamp - flow.end_timestamp > self.flow_timeout:
self._timeout_process(flow, packet)
def _move_flow_from_current_to_finished(self, flow):
"""
Move the flow from `current_flows` to `finished_flows`.
Parameters
----------
flow: Flow
The flow to be moved.
"""
self.finished_flows.append(flow)
if flow.flow_id in self.current_flows.keys():
self.current_flows.pop(flow.flow_id)
else:
self.logger.warning("flow id %s is not in current_flows.keys()", flow.flow_id)
self.logger.warning("Reverse the flow ID.")
self.current_flows.pop(flow.reverse_flow_id())
def _timeout_process(self, flow, packet):
"""
The encapsulation of the process of timeout packets.
Parameters
----------
flow: Flow
Currently processing flow.
packet: BasicPacket
The timeout packet.
"""
self.logger.debug(f'Flow Timeout: {flow.flow_id}')
if flow.packet_count > 1:
self.finished_flows.append(flow)
if flow.flow_id in self.current_flows:
self.current_flows.pop(flow.flow_id)
self.current_flows[flow.flow_id] = Flow(packet, self.flow_timeout)
def dumpflows_to_csv(self, output_file):
"""
Dump the statistics of all flows in `finished_flow` and generate target CSV file.
If there are remaining flows in `current_flows`, move them to `finished_flows`.
Parameters
----------
output_file: str
The file name of output file.
"""
header = [
"Flow ID",
"Src IP",
"Src Port",
"Dst IP",
"Dst Port",
"Protocol",
"Start Time",
"End Time",
"Start Timestamp",
"End Timestamp",
"Flow Duration",
"Total Packet Length",
"Min Packet Length",
"Max Packet Length",
"Mean Packet Length",
"Std Packet Length",
"Total Fwd Packet Length",
"Min Fwd Packet Length",
"Max Fwd Packet Length",
"Mean Fwd Packet Length",
"Std Fwd Packet Length",
"Total Bwd Packet Length",
"Min Bwd Packet Length",
"Max Bwd Packet Length",
"Mean Bwd Packet Length",
"Std Bwd Packet Length",
"Total Fwd Header Length",
"Min Fwd Header Length",
"Max Fwd Header Length",
"Mean Fwd Header Length",
"Std Fwd Header Length",
"Total Bwd Header Length",
"Min Bwd Header Length",
"Max Bwd Header Length",
"Mean Bwd Header Length",
"Std Bwd Header Length",
"Total Fwd Payload Length",
"Min Fwd Payload Length",
"Max Fwd Payload Length",
"Mean Fwd Payload Length",
"Std Fwd Payload Length",
"Total Bwd Payload Length",
"Min Bwd Payload Length",
"Max Bwd Payload Length",
"Mean Bwd Payload Length",
"Std Bwd Payload Length",
"Packet Count",
"Fwd Packet Count",
"Bwd Packet Count",
"Packet Rate",
"Fwd Packet Rate",
"Bwd Packet Rate",
"Bytes Rate",
"Fwd Bytes Rate",
"Bwd Bytes Rate",
"SYN Count",
"ACK Count",
"RST Count",
"URG Count",
"PSH Count",
"FIN Count",
"Init Window Size",
"Min Packet Interval",
"Max Packet Interval",
"Mean Packet Interval",
"Std Packet Interval",
"Min Fwd Packet Interval",
"Max Fwd Packet Interval",
"Mean Fwd Packet Interval",
"Std Fwd Packet Interval",
"Min Bwd Packet Interval",
"Max Bwd Packet Interval",
"Mean Bwd Packet Interval",
"Std Bwd Packet Interval",
]
lines = [','.join(header) + '\n']
while self.current_flows:
flow_id = list(self.current_flows.keys())[0]
self._move_flow_from_current_to_finished(self.current_flows[flow_id])
for flow in tqdm(self.finished_flows, total=len(self.finished_flows)):
data_line = [
flow.flow_id,
flow.src_ip,
flow.src_port,
flow.dst_ip,
flow.dst_port,
flow.protocol,
flow.start_time,
flow.end_time,
flow.start_timestamp,
flow.end_timestamp,
flow.flow_duration,
flow.total_packet_length,
flow.min_packet_length,
flow.max_packet_length,
flow.mean_packet_length,
flow.std_packet_length,
flow.total_forward_packet_length,
flow.min_forward_packet_length,
flow.max_forward_packet_length,
flow.mean_forward_packet_length,
flow.std_forward_packet_length,
flow.total_backward_packet_length,
flow.min_backward_packet_length,
flow.max_backward_packet_length,
flow.mean_backward_packet_length,
flow.std_backward_packet_length,
flow.total_forward_packet_header_length,
flow.min_forward_packet_header_length,
flow.max_forward_packet_header_length,
flow.mean_forward_packet_header_length,
flow.std_forward_packet_header_length,
flow.total_backward_packet_header_length,
flow.min_backward_packet_header_length,
flow.max_backward_packet_header_length,
flow.mean_backward_packet_header_length,
flow.std_backward_packet_header_length,
flow.total_forward_packet_header_length,
flow.min_forward_packet_payload_length,
flow.max_forward_packet_payload_length,
flow.mean_forward_packet_payload_length,
flow.std_forward_packet_payload_length,
flow.total_backward_packet_payload_length,
flow.min_backward_packet_payload_length,
flow.max_backward_packet_payload_length,
flow.mean_backward_packet_payload_length,
flow.std_backward_packet_payload_length,
flow.packet_count,
flow.forward_packet_count,
flow.backward_packet_count,
flow.packet_rate,
flow.forward_packet_rate,
flow.backward_packet_rate,
flow.bytes_rate,
flow.forward_bytes_rate,
flow.backward_bytes_rate,
flow.tcp_flags[0],
flow.tcp_flags[1],
flow.tcp_flags[2],
flow.tcp_flags[3],
flow.tcp_flags[4],
flow.tcp_flags[5],
flow.init_window_size,
flow.min_packet_interval,
flow.max_packet_interval,
flow.mean_packet_interval,
flow.std_packet_interval,
flow.min_forward_packet_interval,
flow.max_forward_packet_interval,
flow.mean_forward_packet_interval,
flow.std_forward_packet_interval,
flow.min_backward_packet_interval,
flow.max_backward_packet_interval,
flow.mean_backward_packet_interval,
flow.std_backward_packet_interval,
]
data_line = list(map(str, data_line))
data_str = ','.join(data_line) + '\n'
lines.append(data_str)
try:
of = open(output_file, 'w+')
of.writelines(lines)
self.logger.info(f'Create output file {output_file} successfully.')
of.close()
except FileNotFoundError:
self.logger.error(f'Create output file {output_file} failed!', exc_info=True)
class BaseApi(object):
def __init__(self, driver):
self.driver = driver
self.logger = MyLogger(self.__class__.__name__).getlogger()
def supperElement(self, loc, errorString, time=5):
if not self.checkElement(loc, time):
raise EleNotFound(errorString)
def checkElement(self, loc, timeout=5):
"""
检查元素是否存在
:param loc: loc type is tuple
:return: check result
"""
if not isinstance(timeout, int):
raise EleNotFound("超时时间必须是的整数")
flage = True
while flage and timeout > 0:
try:
self.driver.find_element(*loc)
return True
except:
pass
time.sleep(0.5)
if flage and timeout == 0:
return False
timeout -= 0.5
def checkElements(self, loc, time=10):
"""
检查元素是否存在
:param loc: loc type is tuple
:return: check result
"""
try:
WebDriverWait(self.driver,
time).until(lambda x: x.find_elements(*loc))
return True
except:
return False
def findElement(self, loc, timeout=5):
"""
查找单个元素
:param loc: id,kw
:param timeout: 超时时间
:return: Element
"""
if self.checkElement(loc, timeout):
return self.driver.find_element(*loc)
else:
self.logger.error("元素：{0} 查找失败".format(loc))
raise EleNotFound("{0} 元素列表未找到".format(loc))
def findElements(self, loc, timeout=5):
"""
查找多个元素
:param loc: id,kw
:param timeout: 超时时间
:return: Element LIst
"""
if self.checkElement(loc, timeout):
return self.driver.find_elements(*loc)
else:
self.logger.error("元素：{0} 查找失败".format(loc))
raise EleNotFound("{0} 元素列表未找到".format(loc))
def findElementUiautomator(self, loc):
"""
使用uiuatomator定位元素 Android专用
:param loc: new UiSelector().text("id")
:return: Element
"""
self.logger.info(
str(('正在使用 new UiSelector().{0}(\"{1}\")'.format(loc[0], loc[1]))))
try:
EleRes = self.driver.find_element_by_android_uiautomator \
('new UiSelector().{0}(\"{1}\")'.format(loc[0], loc[1]))
return EleRes
except:
self.logger.error("元素：{0} 查找失败".format(loc))
raise EleNotFound("findElementUiautomator Error")
def checkAccessiblity(self, loc, timeout=10):
"""
Accessiblity 检查元素是否存在
:param loc: accessibility Id
:param timeout: 超时时间
:return: Element
"""
try:
WebDriverWait(self.driver, timeout).until(
lambda x: x.find_element_by_accessibility_id(loc))
return True
except:
return False
def findAccessiblity(self, loc, timeout=10):
"""
accessibility Android 使用的是 content-desc
属性 IOS使用的是 accessibility identifier属性
:param loc:
:param timeout:
:return:
"""
if self.checkAccessiblity(loc, timeout):
ele = self.driver.find_element_by_accessibility_id(loc)
return ele
else:
self.logger.error("元素：{0} 查找失败".format(loc))
raise EleNotFound("findAccessiblity find element error ")
def iosPredicates(self, ele, timeout=5):
"""
查找多个元素 iOS 专用
:param ele:
:param timeout:
:return:
"""
if self.iosCheckElements(ele, timeout):
ele = self.driver.find_elements_by_ios_predicate(ele)
return ele
else:
raise EleNotFound("iOS ios_predicates：{} 元素未找到".format(ele))
def iosPredicate(self, ele, timeout=5):
"""
查找单个元素 iOS专用
:param ele:
:param timeout:
:return:
"""
if self.iosCheckElement(ele, timeout):
ele = self.driver.find_element_by_ios_predicate(ele)
return ele
else:
raise EleNotFound("iOS ios_predicate：{} 元素未找到".format(ele))
def iosCheckElement(self, ele, timeout=5):
"""
检查 iOS 单个元素是否存在
:param ele:
:param timeout:
:return:
"""
findFlage = True
while findFlage and timeout > 0:
try:
self.driver.find_element_by_ios_predicate(ele)
return True
except:
pass
timeout -= 0.5
if timeout == 0:
return False
def iosCheckElements(self, ele, timeout=10):
"""
检查 ios 多个元素是否存在
:param ele:
:param timeout:
:return:
"""
findFlage = True
while findFlage and timeout > 0:
try:
self.driver.find_element_by_ios_predicate(ele)
return True
except:
pass
timeout -= 0.5
if timeout == 0:
return False
def Select(self, loc):
"""
Select 下拉框
:param loc:
:return:
"""
s = Select(self.findElement(loc))
return s
def click(self, *loc):
"""
点击操作
:param loc:
"""
self.findElement(*loc).click()
def getText(self, loc):
"""
获取元素text文本
:param loc: 元素
:return:
"""
textData = self.findElement(loc, 10).text
return textData
def iosClick(self, loc, timeout=5):
"""
ios 点击元素
:param loc:
:return:
"""
self.iosPredicate(loc, timeout).click()
def setValue(self, loc, params, lenth=10):
"""
ios 输入值
:param ele: 元素
:param params: 输入值
:param lenth: 清除次数
:return:
"""
destEle = self.iosPredicate(loc)
destEle.click()
self.iosClearText(lenth)
destEle.set_value(params)
def iosClearText(self, lenth):
"""
ios 使用键盘删除 输入框内的数据
:param lenth: 删除的次数
"""
if lenth:
try:
ele_button = 'type=="XCUIElementTypeKey" and label=="删除"'
ele = self.iosPredicate(ele_button, 5)
except:
ele_button = 'type=="XCUIElementTypeKey" and label=="delete"'
ele = self.iosPredicate(ele_button, 5)
for l in range(lenth):
ele.click()
def sendKeys(self, loc, text):
"""
输入
:param loc: 元素
:param text: 文本
:return:
"""
ele = self.findElement(loc)
ele.clear()
ele.send_keys(text)
def back(self):
"""
键盘返回
:return:
"""
self.driver.keyevent(4)
def getSize(self):
"""
获取窗口大小
:return: x轴、y轴坐标
"""
are = self.driver.get_window_size()
x = are['width']
y = are['height']
return x, y
def getPage(self):
"""
获取当前的 source Page
:return: page
"""
page = self.driver.page_source
return page
def opterioneycode(self, keycode, metastate=None):
"""
键盘输入
具体请看文档： https://blog.csdn.net/weixin_40180628/article/details/79169846
"""
self.driver.press_keycode(keycode, metastate)
def swipe(self, start_x, start_y, end_x, end_y, duration):
"""
从一个坐标移动到另一个坐标
:param start_x: 起始x坐标
:param start_y: 起始y坐标
:param end_x: 结束X 坐标
:param end_y: 结束Y坐标
:param duration:
:return:
"""
return self.driver.swipe(start_x, start_y, end_x, end_y, duration)
def androidap(self, udid, x, y):
"""
Android adb 命令点击坐标
:param udid: 手机udid
:param x: 横坐标
:param y: 纵坐标
:return:
"""
os.popen('adb -s {0} shell input tap {1} {2}'.format(udid, x, y))
def iosap(self, x, y):
"""
ios tap 方法
:return:
"""
are = self.driver.get_window_size()
x = int(float(x / are["width"]) * are["width"])
y = int(float(y / are["height"]) * are["height"])
self.driver.execute_script("mobile: tap", {
"x": x,
"y": y,
"duration": 500
})
def touch(self, x, y, x1, y1):
"""
触摸移动
:param x:
:param y:
:param x1:
:param y1:
:return:
"""
TouchAction(self.driver).press(x=x,
y=y).move_to(x=x1,
y=y1).release().perform()
def swipeUp(self, x_num=0.5, y1_num=0.25, y2_num=0.75, time=1000):
"""
屏幕上滑
:param x_num:
:param y1_num:
:param y2_num:
:param time:
:return:
"""
are = self.getSize()
x = int(are[0] * x_num)
y1 = int(are[1] * y1_num)
y2 = int(are[1] * y2_num)
self.swipe(x, y1, x, y2, time)
def swipeDown(self, x_num=0.5, y1_num=0.75, y2_num=0.25, time=1000):
"""
屏幕下滑
:param x_num:
:param y1_num:
:param y2_num:
:param time:
:return:
"""
are = self.getSize()
x = int(are[0] * x_num)
y1 = int(are[1] * y1_num)
y2 = int(are[1] * y2_num)
self.swipe(x, y1, x, y2, time)
def swipeRight(self, time=500):
"""
屏幕右滑
:param time:
:return:
"""
are = self.getSize()
x1 = int(are[0] * 0.1)
x2 = int(are[0] * 0.9)
y = int(are[1] * 0.5)
self.swipe(x1, y, x2, y, time)
def swipeLeft(self, x_num=0.1, y1_num=0.5, x2_num=0.9, time=500):
"""
屏幕左滑
:param x_num:
:param y1_num:
:param x2_num:
:param time:
:return:
"""
are = self.getSize()
x1 = int(are[0] * x_num)
x2 = int(are[0] * x2_num)
y = int(are[1] * y1_num)
self.swipe(x2, y, x1, y, time)
def moveTo(self, loc):
"""
鼠标移动到元素ele
:param loc:
:return:
"""
ele = self.findElement(loc)
ActionChains(self.driver).move_to_element(ele).perform()
def scroll(self, ele1, ele2):
"""
从ele1 滑动到ele2
:param ele1:
:param ele2:
:return:
"""
self.driver.scroll(ele1, ele2)
def getAllContext(self):
"""
获取所有的Context
:return:
"""
contexts = self.driver.Contexts
self.logger.info("当前所有的contexts: {0}".format(contexts))
return contexts
def getCurrentContext(self):
"""
获取当前窗口 context
:return:
:return:
"""
currentContext = self.driver.current_context
self.logger.info("当前Context: {0}".format(currentContext))
return self.driver.current_context
def switchContext(self, context):
"""
切换到指定的context 上
:param context:
:return:
"""
self.driver._switch_to.context(context)
def getAllHandles(self):
"""
获取所有的窗口的handles
:return:
"""
allHandles = self.driver.window_handles
return allHandles
def switchHandles(self, handles):
"""
切换到指定的headles
:param handles: 需啊哟切换的hadles
:return:
"""
self.driver._switch_to.window(handles)
def switchToFrame(self, loc):
"""frame 窗口切换"""
self.driver.switch_to.frame(loc)
from utils.logger import MyLogger
from entities.BasicPacket import BasicPacket
from utils.NotifierWin32 import NotifierWin32
try:
from utils.NotifierLinux import NotifierLinux
except ImportError:
pass
FLOW_TIMEOUT = 120000000 # 120 seconds
APP_NAME = 'NetworkPacketAnalyzer'
if __name__ == '__main__':
logger = MyLogger('main')
if len(sys.argv) != 3:
logger.error('Argument insufficient. Need Inputfile and Outputfile')
exit(1)
input_file = sys.argv[1]
output_file = sys.argv[2]
# Set notifier
if sys.platform == 'win32':
notifier = NotifierWin32(APP_NAME)
elif sys.platform == 'linux':
notifier = NotifierLinux(APP_NAME)
else:
notifier = None
flow_generator = FlowGenerator(flow_timeout=FLOW_TIMEOUT)
n_valid = 0
n_discard = 0
