class ScaPyCAPimporter(object): def __init__(self, pcapfilename, importLayer=5): # l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values() # type: List[AbstractMessage] self.importLayer = importLayer self.packets = rdpcap(pcapfilename) self._messages = SortedTypedList(AbstractMessage) self._rawmessages = SortedTypedList(AbstractMessage) for pkt in self.packets: # type: Packet self.packetHandler(pkt) @property def messages(self): return self._messages.values() @property def rawMessages(self): return self._rawmessages.values() def packetHandler(self, packet: Packet): epoch = packet.time l1Payload = bytes(packet) if len(l1Payload) == 0: return # Build the RawMessage rawMessage = RawMessage(l1Payload, epoch, source=None, destination=None) if isinstance(packet, RadioTap): # lift layer to Dot11 if there is a RadioTap dummy frame packet = packet.payload if self.importLayer == 2: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload) = self.__decodeLayer2(packet) if len(l2Payload) == 0: return # Build the L2NetworkMessage l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr) self._messages.add(l2Message) self._rawmessages.add(rawMessage) else: # Use Netzob's PCAPImporter if layer 2 is not WLAN raise NetzobImportException("PCAP", "Unsupported import layer. Currently only handles layer 2.", PCAPImporter.INVALID_LAYER2) def __decodeLayer2(self, packet: Packet): """Internal method that parses the specified header and extracts layer2 related proprieties.""" l2Proto = packet.name if isinstance(packet, Raw): print("Ignoring undecoded packet with values:", bytes(packet).hex()) return l2Proto, None, None, "" if isinstance(packet, Dot11): l2DstAddr = packet.fields['addr1'] # receiver address, alt: packet.fields['addr3'] destination address l2SrcAddr = packet.fields['addr2'] # transmitter address, alt: packet.fields['addr4'] source address else: raise NetzobImportException("NEMERE_PCAP", "Unsupported layer 2 protocol " + l2Proto, PCAPImporter.INVALID_LAYER2) l2Payload = bytes(packet.payload) return l2Proto, l2SrcAddr, l2DstAddr, l2Payload
class PCAPImporter(object): """PCAP importer to read pcaps and extract messages out of them. We recommend to use static methods such as - PCAPImporter.readFiles(...) - PCAPimporter.readFile(...) refer to their documentation to have an overview of the required parameters. >>> from netzob.all import * >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap").values() >>> print(len(messages)) 14 >>> for m in messages: ... print(repr(m.data)) b'CMDidentify#\\x07\\x00\\x00\\x00Roberto' b'RESidentify#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' b'CMDinfo#\\x00\\x00\\x00\\x00' b'RESinfo#\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00info' b'CMDstats#\\x00\\x00\\x00\\x00' b'RESstats#\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00stats' b'CMDauthentify#\\n\\x00\\x00\\x00aStrongPwd' b'RESauthentify#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' b'CMDencrypt#\\x06\\x00\\x00\\x00abcdef' b"RESencrypt#\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00$ !&'$" b"CMDdecrypt#\\x06\\x00\\x00\\x00$ !&'$" b'RESdecrypt#\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00abcdef' b'CMDbye#\\x00\\x00\\x00\\x00' b'RESbye#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=2).values() >>> print(repr(messages[0].data)) b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x08\\x00E\\x00\\x003\\xdc\\x11@\\x00@\\x11`\\xa6\\x7f\\x00\\x00\\x01\\x7f\\x00\\x00\\x01\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=3).values() >>> print(repr(messages[0].data)) b'E\\x00\\x003\\xdc\\x11@\\x00@\\x11`\\xa6\\x7f\\x00\\x00\\x01\\x7f\\x00\\x00\\x01\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=4).values() >>> print(repr(messages[0].data)) b'\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=5).values() >>> print(repr(messages[0].data)) b'CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http.pcap", importLayer=5, bpfFilter="tcp").values() >>> print(repr(messages[0].data)) b'GET / HTTP/1.1\\r\\nHost: www.free.fr\\r\\nUser-Agent: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\\r\\nAccept-Language: en-US,en;q=0.5\\r\\nAccept-Encoding: gzip, deflate\\r\\nConnection: keep-alive\\r\\n\\r\\n' Parameter `mergePacketsInFlow` can be use to merge consecutive messages that share the same source and destination (mimic a TCP flow). In practice, this parameter was introduced for L5 network messages to support TCP flows but it can be use for any level of network messages. >>> from netzob.all import * >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http_flow.pcap", mergePacketsInFlow=False).values() >>> print(len(messages)) 4 >>> print(len(messages[1].data)) 1228 >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http_flow.pcap", mergePacketsInFlow=True).values() >>> print(len(messages)) 2 >>> print(len(messages[1].data)) 3224 """ INVALID_BPF_FILTER = 0 INVALID_LAYER2 = 1 INVALID_LAYER3 = 2 INVALID_LAYER4 = 3 PROTOCOL201 = 201 # Supported datalinks (by pcapy) SUPPORTED_DATALINKS = { pcapy.DLT_ARCNET: "DLT_ARCNET", pcapy.DLT_FDDI: "DLT_FDDI", pcapy.DLT_LOOP: "DLT_LOOP", pcapy.DLT_PPP_ETHER: "DLT_PPP_ETHER", pcapy.DLT_ATM_RFC1483: "DLT_ATM_RFC1483", pcapy.DLT_IEEE802: "DLT_IEEE802", pcapy.DLT_LTALK: "DLT_LTALK", pcapy.DLT_PPP_SERIAL: "DLT_PPP_SERIAL", pcapy.DLT_C_HDLC: "DLT_C_HDLC", pcapy.DLT_IEEE802_11: "IEEE802_11", pcapy.DLT_NULL: "DLT_NULL", pcapy.DLT_RAW: "DLT_RAW", pcapy.DLT_EN10MB: "DLT_EN10MB", pcapy.DLT_LINUX_SLL: "LINUX_SLL", pcapy.DLT_PPP: "DLT_PPP", pcapy.DLT_SLIP: "DLT_SLIP", } def __init__(self): pass @typeCheck(str, str, int) def __readMessagesFromFile(self, filePath, bpfFilter, nbPackets): """Internal methods to read all messages from a given PCAP file.""" if (filePath is None): raise TypeError("filePath cannot be None") if (nbPackets < 0): raise ValueError( "A positive (or null) value is required for the number of packets to read." ) # Check file can be opened (and read) try: fp = open(filePath, 'r') fp.close() except IOError as e: if e.errno == errno.EACCES: raise IOError( "Error while trying to open the file {0}, more permissions are required to read it." ).format(filePath) else: raise e # Check (and configure) the bpf filter packetReader = pcapy.open_offline(filePath) try: packetReader.setfilter(bpfFilter) except: raise ValueError( "The provided BPF filter is not valid (it should follow the BPF format)" ) # Check the datalink self.datalink = packetReader.datalink() if self.datalink not in list(PCAPImporter.SUPPORTED_DATALINKS.keys()): self._logger.debug("Unkown datalinks") if self.importLayer > 1 and self.datalink != pcapy.DLT_EN10MB and self.datalink != pcapy.DLT_LINUX_SLL and self.datalink != PCAPImporter.PROTOCOL201: errorMessage = _("This pcap cannot be imported since the " + "layer 2 is not supported ({0})").format( str(self.datalink)) raise NetzobImportException("PCAP", errorMessage, self.INVALID_LAYER2) else: packetReader.loop(nbPackets, self.__packetHandler) def __packetHandler(self, header, payload): """Internal callback executed on each packet when parsing the pcap""" (secs, usecs) = header.getts() epoch = secs + (usecs / 1000000.0) if self.importLayer == 1 or self.importLayer == 2: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2 of a packet: {0}". format(e)) return if len(l2Payload) == 0: return # Build the L2NetworkMessage l2Message = L2NetworkMessage(payload, epoch, l2Proto, l2SrcAddr, l2DstAddr) self.messages.add(l2Message) elif self.importLayer == 3: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) except NetzobImportException as e: self._logger.warn("An error occured while decoding layer2 and layer3 of a packet: {0}".format(e)) (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2 and layer3 of a packet: {0}". format(e)) return if len(l3Payload) == 0: return # Build the L3NetworkMessage l3Message = L3NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr) self.messages.add(l3Message) elif self.importLayer == 4: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) (l4Proto, l4SrcPort, l4DstPort, l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2, layer3 or layer4 of a packet: {0}". format(e)) return if len(l4Payload) == 0: return # Build the L4NetworkMessage l4Message = L4NetworkMessage( l3Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr, l4Proto, l4SrcPort, l4DstPort) self.messages.add(l4Message) else: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) (l4Proto, l4SrcPort, l4DstPort, l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2, layer3, layer4 or layer5 of a packet: {0}". format(e)) return if len(l4Payload) == 0: return l5Message = L4NetworkMessage( l4Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr, l4Proto, l4SrcPort, l4DstPort) self.messages.add(l5Message) def __decodeLayer2(self, header, payload): """Internal method that parses the specified header and extracts layer2 related proprieties.""" def formatMacAddress(arrayMac): return ":".join("{0:0>2}".format(hex(b)[2:]) for b in arrayMac.tolist()) if self.datalink == pcapy.DLT_EN10MB: l2Decoder = Decoders.EthDecoder() l2Proto = "Ethernet" layer2 = l2Decoder.decode(payload) l2SrcAddr = formatMacAddress(layer2.get_ether_shost()) l2DstAddr = formatMacAddress(layer2.get_ether_dhost()) l2Payload = payload[layer2.get_header_size():] etherType = layer2.get_ether_type() elif self.datalink == pcapy.DLT_LINUX_SLL: l2Decoder = Decoders.LinuxSLLDecoder() l2Proto = "Linux SLL" layer2 = l2Decoder.decode(payload) l2SrcAddr = layer2.get_addr() l2DstAddr = None l2Payload = payload[layer2.get_header_size():] etherType = layer2.get_ether_type() elif self.datalink == PCAPImporter.PROTOCOL201: l2Proto = "Protocol 201" hdr = payload.encode('hex')[0:8] if hdr[6:] == "01": l2SrcAddr = "Received" else: l2SrcAddr = "Sent" l2DstAddr = None l2Payload = payload[8:] etherType = payload[4:6] return (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) def __decodeLayer3(self, etherType, l2Payload): """Internal method that parses the specified header and extracts layer3 related proprieties.""" if etherType == Packets.IP.ethertype: l3Proto = "IP" l3Decoder = Decoders.IPDecoder() layer3 = l3Decoder.decode(l2Payload) paddingSize = len(l2Payload) - layer3.get_ip_len() l3SrcAddr = layer3.get_ip_src() l3DstAddr = layer3.get_ip_dst() l3Payload = l2Payload[layer3.get_header_size():] if paddingSize > 0 and len(l3Payload) > paddingSize: l3Payload = l3Payload[:len(l3Payload) - paddingSize] ipProtocolNum = layer3.get_ip_p() return (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) else: warnMessage = _("Cannot import one of the provided packets since " + "its layer 3 is unsupported (Only IP is " + "currently supported, packet ethernet " + "type = {0})").format(etherType) self._logger.warn(warnMessage) raise NetzobImportException("PCAP", warnMessage, self.INVALID_LAYER3) def __decodeLayer4(self, ipProtocolNum, l3Payload): """Internal method that parses the specified header and extracts layer4 related proprieties.""" if ipProtocolNum == Packets.UDP.protocol: l4Proto = "UDP" l4Decoder = Decoders.UDPDecoder() layer4 = l4Decoder.decode(l3Payload) l4SrcPort = layer4.get_uh_sport() l4DstPort = layer4.get_uh_dport() l4Payload = layer4.get_data_as_string() return (l4Proto, l4SrcPort, l4DstPort, l4Payload) elif ipProtocolNum == Packets.TCP.protocol: l4Proto = "TCP" l4Decoder = Decoders.TCPDecoder() layer4 = l4Decoder.decode(l3Payload) l4SrcPort = layer4.get_th_sport() l4DstPort = layer4.get_th_dport() l4Payload = layer4.get_data_as_string() return (l4Proto, l4SrcPort, l4DstPort, l4Payload) else: warnMessage = _("Cannot import one of the provided packets since " + "its layer 4 is unsupported (Only UDP and TCP " + "are currently supported, packet IP protocol " + "number = {0})").format(ipProtocolNum) self._logger.warn(warnMessage) raise NetzobImportException("PCAP", warnMessage, self.INVALID_LAYER4) @typeCheck(list, str, int, int) def readMessages(self, filePathList, bpfFilter="", importLayer=5, nbPackets=0): warnMessage = _("Cannot import one of the provided packets since " + "its layer 4 is unsupported (Only UDP and TCP " + "are currently supported, packet IP protocol " + "number = {0})").format(ipProtocolNum) self._logger.warn(warnMessage) raise NetzobImportException("PCAP", warnMessage, self.INVALID_LAYER4) @typeCheck(list, str, int, int, bool) def readMessages(self, filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False, ): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload). Finally, the number of packets to capture can be specified. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("PCAP", "\n".join(errorMessageList)) # Verify the expected import layer availableLayers = [1, 2, 3, 4, 5] if importLayer not in availableLayers: raise Exception( "Only layers level {0} are available.".format(availableLayers)) self.importLayer = importLayer # Call the method that does the import job for each PCAP file self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, bpfFilter, nbPackets) #Create a session and attribute it to messages: session = Session(list(self.messages.values()),name=filePath) for message in self.messages.values(): message.session = session # if requested, we merge consecutive messages that share same source and destination if mergePacketsInFlow: mergedMessages = SortedTypedList(AbstractMessage) previousMessage = None for message in self.messages.values(): if previousMessage is not None and message.source == previousMessage.source and message.destination == previousMessage.destination: previousMessage.data += message.data else: mergedMessages.add(message) previousMessage = message self.messages = mergedMessages return self.messages @staticmethod @typeCheck(list, str, int, int, bool) def readFiles(filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload) The number of packets to capture can be specified. :param filePathList: a list of pcap files to read :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = PCAPImporter() return importer.readMessages(filePathList,bpfFilter, importLayer, nbPackets, mergePacketsInFlow) @staticmethod @typeCheck(str, str, int, int, bool) def readFile(filePath, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False): """Read all messages from the specified PCAP file. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload) and merge consecutive messages with same source and destination. Finally, the number of packets to capture can be specified. :param filePath: the pcap path :type filePath: :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = PCAPImporter() return importer.readFiles([filePath], bpfFilter, importLayer, nbPackets, mergePacketsInFlow) @staticmethod @typeCheck(L2NetworkMessage) def getMessageDetails(message): """Decode a raw network message and print the content of each encapsulated layer. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` """ decoder = Decoders.EthDecoder() return decoder.decode( TypeConverter.convert(message.data, HexaString, Raw))
class Session(object): """A session includes messages exchanged in the same session. Messages are automaticaly sorted. Applicative data can be attached to sessions. >>> import time >>> from netzob.all import * >>> # we create 3 messages >>> msg1 = RawMessage("ACK", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:05", "%d %b %y %H:%M:%S"))) >>> msg2 = RawMessage("SYN", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:01", "%d %b %y %H:%M:%S"))) >>> msg3 = RawMessage("SYN/ACK", source="B", destination="A", date=time.mktime(time.strptime("9 Aug 13 10:45:03", "%d %b %y %H:%M:%S"))) >>> session = Session([msg1, msg2, msg3]) >>> print(session.messages.values()[0].data) SYN >>> print(session.messages.values()[1].data) SYN/ACK >>> print(session.messages.values()[2].data) ACK """ def __init__(self, messages=None, _id=None, applicativeData=None, name="Session"): """ :parameter messages: the messages exchanged in the current session :type data: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` :parameter _id: the unique identifier of the session :type _id: :class:`uuid.UUID` :keyword applicativeData: a list of :class:`netzob.Model.Vocabulary.ApplicaticeData.ApplicativeData` """ self.__messages = SortedTypedList(AbstractMessage) self.__applicativeData = TypedList(ApplicativeData) if messages is None: messages = [] self.messages = messages if _id is None: _id = uuid.uuid4() self.id = _id if applicativeData is None: applicativeData = [] self.applicativeData = applicativeData self.name = name @property def id(self): """The unique identifier of the session. :type: :class:`uuid.UUID` """ return self.__id @id.setter @typeCheck(uuid.UUID) def id(self, _id): if _id is None: raise TypeError("Id cannot be None") self.__id = _id @property def messages(self): """The messages exchanged in the current session. Messages are sorted. :type: a :class:`netzob.Common.Utils.TypedList.TypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` """ return self.__messages def clearMessages(self): """Delete all the messages attached to the current session""" for msg in list(self.__messages.values()): msg.session = None self.__messages.clear() @messages.setter def messages(self, messages): if messages is None: messages = [] # First it checks the specified messages are all AbstractMessages for msg in messages: if not isinstance(msg, AbstractMessage): raise TypeError( "Cannot add messages of type {0} in the session, only AbstractMessages are allowed." .format(type(msg))) self.clearMessages() for message in messages: self.__messages.add(message) message.session = self @property def applicativeData(self): """Applicative data attached to the current session. >>> from netzob.all import * >>> appData = ApplicativeData("test", Integer(20)) >>> session = Session(applicativeData=[appData]) >>> print(len(session.applicativeData)) 1 >>> appData2 = ApplicativeData("test2", ASCII("helloworld")) >>> session.applicativeData.append(appData2) >>> print(len(session.applicativeData)) 2 >>> print(session.applicativeData[0]) Applicative Data: test=Integer=20 ((8, 8))) >>> print(session.applicativeData[1]) Applicative Data: test2=ASCII=helloworld ((0, 80))) :type: a list of :class:`netzob.Model.Vocabulary.ApplicativeData.ApplicativeData`. """ return self.__applicativeData def clearApplicativeData(self): while (len(self.__applicativeData) > 0): self.__applicativeData.pop() @applicativeData.setter def applicativeData(self, applicativeData): for app in applicativeData: if not isinstance(app, ApplicativeData): raise TypeError( "Cannot add an applicative data with type {0}, only ApplicativeData accepted." .format(type(app))) self.clearApplicativeData() for app in applicativeData: self.applicativeData.append(app) @property def name(self): return self.__name @name.setter @typeCheck(str) def name(self, _name): if _name is None: raise TypeError("Name cannot be None") self.__name = _name def getEndpointsList(self): """Retrieve all the endpoints couples that are present in the session. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getEndpointsList())) 2 >>> print(session.getEndpointsList()) [('A', 'B'), ('A', 'C')] :return: a list containing couple of endpoints (src, dst). :rtype: a :class:`list` """ endpointsList = [] for message in list(self.messages.values()): src = message.source dst = message.destination endpoints1 = (src, dst) endpoints2 = (dst, src) if (not endpoints1 in endpointsList) and (not endpoints2 in endpointsList): endpointsList.append(endpoints1) return endpointsList def getTrueSessions(self): """Retrieve the true sessions embedded in the current session. A session is here characterized by a uniq endpoints couple. TODO: a more precise solution would be to use flow reconstruction (as in TCP). >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getTrueSessions())) 2 >>> for trueSession in session.getTrueSessions(): ... print(trueSession.name) Session: 'A' - 'B' Session: 'A' - 'C' :return: a list containing true sessions embedded in the current session. :rtype: a :class:`list` """ trueSessions = [] for endpoints in self.getEndpointsList(): trueSessionMessages = [] src = None dst = None for message in list(self.messages.values()): if message.source in endpoints and message.destination in endpoints: trueSessionMessages.append(message) if src is None: src = message.source if dst is None: dst = message.destination trueSession = Session(messages=trueSessionMessages, applicativeData=self.applicativeData, name="Session: '" + str(src) + "' - '" + str(dst) + "'") trueSessions.append(trueSession) return trueSessions def isTrueSession(self): """Tell if the current session is true. A session is said to be true if the communication flow pertain to a uniq applicative session between a couple of endpoints. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> print(session.isTrueSession()) True :return: a boolean telling if the current session is a true one (i.e. it corresponds to a uniq applicative session between two endpoints). :rtype: a :class:`bool` """ if len(self.getTrueSessions()) == 1: return True else: return False @typeCheck(list) def abstract(self, symbolList): """This method abstract each message of the current session into symbols according to a list of symbols given as parameter. >>> from netzob.all import * >>> symbolSYN = Symbol([Field(ASCII("SYN"))], name="Symbol_SYN") >>> symbolSYNACK = Symbol([Field(ASCII("SYN/ACK"))], name="Symbol_SYNACK") >>> symbolACK = Symbol([Field(ASCII("ACK"))], name="Symbol_ACK") >>> symbolList = [symbolSYN, symbolSYNACK, symbolACK] >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> if session.isTrueSession(): ... for src, dst, sym in session.abstract(symbolList): ... print(str(src) + " - " + str(dst) + " : " + str(sym.name)) A - B : Symbol_SYN B - A : Symbol_SYNACK A - B : Symbol_ACK :return: a list of tuples containing the following elements : (source, destination, symbol). :rtype: a :class:`list` """ abstractSession = [] if not self.isTrueSession(): self._logger.warn( "The current session cannot be abstracted as it not a true session (i.e. it may contain inner true sessions)." ) return abstractSession for message in list(self.messages.values()): symbol = AbstractField.abstract(message.data, symbolList) abstractSession.append( (message.source, message.destination, symbol)) return abstractSession
class Session(object): """A session includes messages exchanged in the same session. Messages are automaticaly sorted. Applicative data can be attached to sessions. >>> import time >>> from netzob.all import * >>> # we create 3 messages >>> msg1 = RawMessage("ACK", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:05", "%d %b %y %H:%M:%S"))) >>> msg2 = RawMessage("SYN", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:01", "%d %b %y %H:%M:%S"))) >>> msg3 = RawMessage("SYN/ACK", source="B", destination="A", date=time.mktime(time.strptime("9 Aug 13 10:45:03", "%d %b %y %H:%M:%S"))) >>> session = Session([msg1, msg2, msg3]) >>> print(session.messages.values()[0].data) SYN >>> print(session.messages.values()[1].data) SYN/ACK >>> print(session.messages.values()[2].data) ACK """ def __init__(self, messages=None, _id=None, applicativeData=None, name="Session"): """ :parameter messages: the messages exchanged in the current session :type data: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` :parameter _id: the unique identifier of the session :type _id: :class:`uuid.UUID` :keyword applicativeData: a list of :class:`netzob.Model.Vocabulary.ApplicaticeData.ApplicativeData` """ self.__messages = SortedTypedList(AbstractMessage) self.__applicativeData = TypedList(ApplicativeData) if messages is None: messages = [] self.messages = messages if _id is None: _id = uuid.uuid4() self.id = _id if applicativeData is None: applicativeData = [] self.applicativeData = applicativeData self.name = name @property def id(self): """The unique identifier of the session. :type: :class:`uuid.UUID` """ return self.__id @id.setter @typeCheck(uuid.UUID) def id(self, _id): if _id is None: raise TypeError("Id cannot be None") self.__id = _id @property def messages(self): """The messages exchanged in the current session. Messages are sorted. :type: a :class:`netzob.Common.Utils.TypedList.TypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` """ return self.__messages def clearMessages(self): """Delete all the messages attached to the current session""" for msg in list(self.__messages.values()): msg.session = None self.__messages.clear() @messages.setter def messages(self, messages): if messages is None: messages = [] # First it checks the specified messages are all AbstractMessages for msg in messages: if not isinstance(msg, AbstractMessage): raise TypeError( "Cannot add messages of type {0} in the session, only AbstractMessages are allowed.". format(type(msg))) self.clearMessages() for message in messages: self.__messages.add(message) message.session = self @property def applicativeData(self): """Applicative data attached to the current session. >>> from netzob.all import * >>> appData = ApplicativeData("test", Integer(20)) >>> session = Session(applicativeData=[appData]) >>> print(len(session.applicativeData)) 1 >>> appData2 = ApplicativeData("test2", ASCII("helloworld")) >>> session.applicativeData.append(appData2) >>> print(len(session.applicativeData)) 2 >>> print(session.applicativeData[0]) Applicative Data: test=Integer=20 ((8, 8))) >>> print(session.applicativeData[1]) Applicative Data: test2=ASCII=helloworld ((0, 80))) :type: a list of :class:`netzob.Model.Vocabulary.ApplicativeData.ApplicativeData`. """ return self.__applicativeData def clearApplicativeData(self): while (len(self.__applicativeData) > 0): self.__applicativeData.pop() @applicativeData.setter def applicativeData(self, applicativeData): for app in applicativeData: if not isinstance(app, ApplicativeData): raise TypeError( "Cannot add an applicative data with type {0}, only ApplicativeData accepted.". format(type(app))) self.clearApplicativeData() for app in applicativeData: self.applicativeData.append(app) @property def name(self): return self.__name @name.setter @typeCheck(str) def name(self, _name): if _name is None: raise TypeError("Name cannot be None") self.__name = _name def getEndpointsList(self): """Retrieve all the endpoints couples that are present in the session. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getEndpointsList())) 2 >>> print(session.getEndpointsList()) [('A', 'B'), ('A', 'C')] :return: a list containing couple of endpoints (src, dst). :rtype: a :class:`list` """ endpointsList = [] for message in list(self.messages.values()): src = message.source dst = message.destination endpoints1 = (src, dst) endpoints2 = (dst, src) if (not endpoints1 in endpointsList) and ( not endpoints2 in endpointsList): endpointsList.append(endpoints1) return endpointsList def getTrueSessions(self): """Retrieve the true sessions embedded in the current session. A session is here characterized by a uniq endpoints couple. TODO: a more precise solution would be to use flow reconstruction (as in TCP). >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getTrueSessions())) 2 >>> for trueSession in session.getTrueSessions(): ... print(trueSession.name) Session: 'A' - 'B' Session: 'A' - 'C' :return: a list containing true sessions embedded in the current session. :rtype: a :class:`list` """ trueSessions = [] for endpoints in self.getEndpointsList(): trueSessionMessages = [] src = None dst = None for message in list(self.messages.values()): if message.source in endpoints and message.destination in endpoints: trueSessionMessages.append(message) if src is None: src = message.source if dst is None: dst = message.destination trueSession = Session( messages=trueSessionMessages, applicativeData=self.applicativeData, name="Session: '" + str(src) + "' - '" + str(dst) + "'") trueSessions.append(trueSession) return trueSessions def isTrueSession(self): """Tell if the current session is true. A session is said to be true if the communication flow pertain to a uniq applicative session between a couple of endpoints. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> print(session.isTrueSession()) True :return: a boolean telling if the current session is a true one (i.e. it corresponds to a uniq applicative session between two endpoints). :rtype: a :class:`bool` """ if len(self.getTrueSessions()) == 1: return True else: return False @typeCheck(list) def abstract(self, symbolList): """This method abstract each message of the current session into symbols according to a list of symbols given as parameter. >>> from netzob.all import * >>> symbolSYN = Symbol([Field(ASCII("SYN"))], name="Symbol_SYN") >>> symbolSYNACK = Symbol([Field(ASCII("SYN/ACK"))], name="Symbol_SYNACK") >>> symbolACK = Symbol([Field(ASCII("ACK"))], name="Symbol_ACK") >>> symbolList = [symbolSYN, symbolSYNACK, symbolACK] >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> if session.isTrueSession(): ... for src, dst, sym in session.abstract(symbolList): ... print(str(src) + " - " + str(dst) + " : " + str(sym.name)) A - B : Symbol_SYN B - A : Symbol_SYNACK A - B : Symbol_ACK :return: a list of tuples containing the following elements : (source, destination, symbol). :rtype: a :class:`list` """ abstractSession = [] if not self.isTrueSession(): self._logger.warn( "The current session cannot be abstracted as it not a true session (i.e. it may contain inner true sessions)." ) return abstractSession for message in list(self.messages.values()): (symbol, structured_message) = AbstractField.abstract(message.data, symbolList) abstractSession.append((message.source, message.destination, symbol)) return abstractSession
class PCAPImporter(object): """PCAP importer to read pcaps and extract messages out of them. We recommend to use static methods such as - PCAPImporter.readFiles(...) - PCAPimporter.readFile(...) refer to their documentation to have an overview of the required parameters. >>> from netzob.all import * >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap").values() >>> print(len(messages)) 14 >>> for m in messages: ... print(repr(m.data)) b'CMDidentify#\\x07\\x00\\x00\\x00Roberto' b'RESidentify#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' b'CMDinfo#\\x00\\x00\\x00\\x00' b'RESinfo#\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00info' b'CMDstats#\\x00\\x00\\x00\\x00' b'RESstats#\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00stats' b'CMDauthentify#\\n\\x00\\x00\\x00aStrongPwd' b'RESauthentify#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' b'CMDencrypt#\\x06\\x00\\x00\\x00abcdef' b"RESencrypt#\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00$ !&'$" b"CMDdecrypt#\\x06\\x00\\x00\\x00$ !&'$" b'RESdecrypt#\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00abcdef' b'CMDbye#\\x00\\x00\\x00\\x00' b'RESbye#\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=2).values() >>> print(repr(messages[0].data)) b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x08\\x00E\\x00\\x003\\xdc\\x11@\\x00@\\x11`\\xa6\\x7f\\x00\\x00\\x01\\x7f\\x00\\x00\\x01\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=3).values() >>> print(repr(messages[0].data)) b'E\\x00\\x003\\xdc\\x11@\\x00@\\x11`\\xa6\\x7f\\x00\\x00\\x01\\x7f\\x00\\x00\\x01\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=4).values() >>> print(repr(messages[0].data)) b'\\xe1\\xe7\\x10\\x92\\x00\\x1f\\xfe2CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_udp.pcap", importLayer=5).values() >>> print(repr(messages[0].data)) b'CMDidentify#\\x07\\x00\\x00\\x00Roberto' >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http.pcap", importLayer=5, bpfFilter="tcp").values() >>> print(repr(messages[0].data)) b'GET / HTTP/1.1\\r\\nHost: www.free.fr\\r\\nUser-Agent: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\\r\\nAccept-Language: en-US,en;q=0.5\\r\\nAccept-Encoding: gzip, deflate\\r\\nConnection: keep-alive\\r\\n\\r\\n' Parameter `mergePacketsInFlow` can be use to merge consecutive messages that share the same source and destination (mimic a TCP flow). In practice, this parameter was introduced for L5 network messages to support TCP flows but it can be use for any level of network messages. >>> from netzob.all import * >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http_flow.pcap", mergePacketsInFlow=False).values() >>> print(len(messages)) 4 >>> print(len(messages[1].data)) 1228 >>> messages = PCAPImporter.readFile("./test/resources/pcaps/test_import_http_flow.pcap", mergePacketsInFlow=True).values() >>> print(len(messages)) 2 >>> print(len(messages[1].data)) 3224 """ INVALID_BPF_FILTER = 0 INVALID_LAYER2 = 1 INVALID_LAYER3 = 2 INVALID_LAYER4 = 3 PROTOCOL201 = 201 # Supported datalinks (by pcapy) SUPPORTED_DATALINKS = { pcapy.DLT_ARCNET: "DLT_ARCNET", pcapy.DLT_FDDI: "DLT_FDDI", pcapy.DLT_LOOP: "DLT_LOOP", pcapy.DLT_PPP_ETHER: "DLT_PPP_ETHER", pcapy.DLT_ATM_RFC1483: "DLT_ATM_RFC1483", pcapy.DLT_IEEE802: "DLT_IEEE802", pcapy.DLT_LTALK: "DLT_LTALK", pcapy.DLT_PPP_SERIAL: "DLT_PPP_SERIAL", pcapy.DLT_C_HDLC: "DLT_C_HDLC", pcapy.DLT_IEEE802_11: "IEEE802_11", pcapy.DLT_NULL: "DLT_NULL", pcapy.DLT_RAW: "DLT_RAW", pcapy.DLT_EN10MB: "DLT_EN10MB", pcapy.DLT_LINUX_SLL: "LINUX_SLL", pcapy.DLT_PPP: "DLT_PPP", pcapy.DLT_SLIP: "DLT_SLIP", } def __init__(self): pass @typeCheck(str, str, int) def __readMessagesFromFile(self, filePath, bpfFilter, nbPackets): """Internal methods to read all messages from a given PCAP file.""" if (filePath is None): raise TypeError("filePath cannot be None") if (nbPackets < 0): raise ValueError( "A positive (or null) value is required for the number of packets to read." ) # Check file can be opened (and read) try: fp = open(filePath, 'r') fp.close() except IOError as e: if e.errno == errno.EACCES: raise IOError( "Error while trying to open the file {0}, more permissions are required to read it." ).format(filePath) else: raise e # Check (and configure) the bpf filter packetReader = pcapy.open_offline(filePath) try: packetReader.setfilter(bpfFilter) except: raise ValueError( "The provided BPF filter is not valid (it should follow the BPF format)" ) # Check the datalink self.datalink = packetReader.datalink() if self.datalink not in list(PCAPImporter.SUPPORTED_DATALINKS.keys()): self._logger.debug("Unkown datalinks") if self.importLayer > 1 and self.datalink != pcapy.DLT_EN10MB and self.datalink != pcapy.DLT_LINUX_SLL and self.datalink != PCAPImporter.PROTOCOL201: errorMessage = _("This pcap cannot be imported since the " + "layer 2 is not supported ({0})").format( str(self.datalink)) raise NetzobImportException("PCAP", errorMessage, self.INVALID_LAYER2) else: packetReader.loop(nbPackets, self.__packetHandler) def __packetHandler(self, header, payload): """Internal callback executed on each packet when parsing the pcap""" (secs, usecs) = header.getts() epoch = secs + (usecs / 1000000.0) if self.importLayer == 1 or self.importLayer == 2: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2 of a packet: {0}". format(e)) return if len(l2Payload) == 0: return # Build the L2NetworkMessage l2Message = L2NetworkMessage(payload, epoch, l2Proto, l2SrcAddr, l2DstAddr) self.messages.add(l2Message) elif self.importLayer == 3: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2 and layer3 of a packet: {0}". format(e)) return if len(l3Payload) == 0: return # Build the L3NetworkMessage l3Message = L3NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr) self.messages.add(l3Message) elif self.importLayer == 4: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) (l4Proto, l4SrcPort, l4DstPort, l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2, layer3 or layer4 of a packet: {0}". format(e)) return if len(l4Payload) == 0: return # Build the L4NetworkMessage l4Message = L4NetworkMessage( l3Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr, l4Proto, l4SrcPort, l4DstPort) self.messages.add(l4Message) else: try: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) = self.__decodeLayer2(header, payload) (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload) (l4Proto, l4SrcPort, l4DstPort, l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload) except NetzobImportException as e: self._logger.warn( "An error occured while decoding layer2, layer3, layer4 or layer5 of a packet: {0}". format(e)) return if len(l4Payload) == 0: return l5Message = L4NetworkMessage( l4Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr, l3Proto, l3SrcAddr, l3DstAddr, l4Proto, l4SrcPort, l4DstPort) self.messages.add(l5Message) def __decodeLayer2(self, header, payload): """Internal method that parses the specified header and extracts layer2 related proprieties.""" def formatMacAddress(arrayMac): return ":".join("{0:0>2}".format(hex(b)[2:]) for b in arrayMac.tolist()) if self.datalink == pcapy.DLT_EN10MB: l2Decoder = Decoders.EthDecoder() l2Proto = "Ethernet" layer2 = l2Decoder.decode(payload) l2SrcAddr = formatMacAddress(layer2.get_ether_shost()) l2DstAddr = formatMacAddress(layer2.get_ether_dhost()) l2Payload = payload[layer2.get_header_size():] etherType = layer2.get_ether_type() elif self.datalink == pcapy.DLT_LINUX_SLL: l2Decoder = Decoders.LinuxSLLDecoder() l2Proto = "Linux SLL" layer2 = l2Decoder.decode(payload) l2SrcAddr = layer2.get_addr() l2DstAddr = None l2Payload = payload[layer2.get_header_size():] etherType = layer2.get_ether_type() elif self.datalink == PCAPImporter.PROTOCOL201: l2Proto = "Protocol 201" hdr = payload.encode('hex')[0:8] if hdr[6:] == "01": l2SrcAddr = "Received" else: l2SrcAddr = "Sent" l2DstAddr = None l2Payload = payload[8:] etherType = payload[4:6] return (l2Proto, l2SrcAddr, l2DstAddr, l2Payload, etherType) def __decodeLayer3(self, etherType, l2Payload): """Internal method that parses the specified header and extracts layer3 related proprieties.""" if etherType == Packets.IP.ethertype: l3Proto = "IP" l3Decoder = Decoders.IPDecoder() layer3 = l3Decoder.decode(l2Payload) paddingSize = len(l2Payload) - layer3.get_ip_len() l3SrcAddr = layer3.get_ip_src() l3DstAddr = layer3.get_ip_dst() l3Payload = l2Payload[layer3.get_header_size():] if paddingSize > 0 and len(l3Payload) > paddingSize: l3Payload = l3Payload[:len(l3Payload) - paddingSize] ipProtocolNum = layer3.get_ip_p() return (l3Proto, l3SrcAddr, l3DstAddr, l3Payload, ipProtocolNum) else: warnMessage = _("Cannot import one of the provided packets since " + "its layer 3 is unsupported (Only IP is " + "currently supported, packet ethernet " + "type = {0})").format(etherType) self._logger.warn(warnMessage) raise NetzobImportException("PCAP", warnMessage, self.INVALID_LAYER3) def __decodeLayer4(self, ipProtocolNum, l3Payload): """Internal method that parses the specified header and extracts layer4 related proprieties.""" if ipProtocolNum == Packets.UDP.protocol: l4Proto = "UDP" l4Decoder = Decoders.UDPDecoder() layer4 = l4Decoder.decode(l3Payload) l4SrcPort = layer4.get_uh_sport() l4DstPort = layer4.get_uh_dport() l4Payload = layer4.get_data_as_string() return (l4Proto, l4SrcPort, l4DstPort, l4Payload) elif ipProtocolNum == Packets.TCP.protocol: l4Proto = "TCP" l4Decoder = Decoders.TCPDecoder() layer4 = l4Decoder.decode(l3Payload) l4SrcPort = layer4.get_th_sport() l4DstPort = layer4.get_th_dport() l4Payload = layer4.get_data_as_string() return (l4Proto, l4SrcPort, l4DstPort, l4Payload) else: warnMessage = _("Cannot import one of the provided packets since " + "its layer 4 is unsupported (Only UDP and TCP " + "are currently supported, packet IP protocol " + "number = {0})").format(ipProtocolNum) self._logger.warn(warnMessage) raise NetzobImportException("PCAP", warnMessage, self.INVALID_LAYER4) @typeCheck(list, str, int, int, bool) def readMessages(self, filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False, ): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload). Finally, the number of packets to capture can be specified. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("PCAP", "\n".join(errorMessageList)) # Verify the expected import layer availableLayers = [1, 2, 3, 4, 5] if not importLayer in availableLayers: raise Exception( "Only layers level {0} are available.".format(availableLayers)) self.importLayer = importLayer # Call the method that does the import job for each PCAP file self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, bpfFilter, nbPackets) # if requested, we merge consecutive messages that share same source and destination if mergePacketsInFlow: mergedMessages = SortedTypedList(AbstractMessage) previousMessage = None for message in self.messages.values(): if previousMessage is not None and message.source == previousMessage.source and message.destination == previousMessage.destination: previousMessage.data += message.data else: mergedMessages.add(message) previousMessage = message self.messages = mergedMessages return self.messages @staticmethod @typeCheck(list, str, int, int, bool) def readFiles(filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload) The number of packets to capture can be specified. :param filePathList: a list of pcap files to read :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = PCAPImporter() return importer.readMessages(filePathList,bpfFilter, importLayer, nbPackets, mergePacketsInFlow) @staticmethod @typeCheck(str, str, int, int, bool) def readFile(filePath, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False): """Read all messages from the specified PCAP file. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload) and merge consecutive messages with same source and destination. Finally, the number of packets to capture can be specified. :param filePath: the pcap path :type filePath: :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = PCAPImporter() return importer.readFiles([filePath], bpfFilter, importLayer, nbPackets, mergePacketsInFlow) @staticmethod @typeCheck(L2NetworkMessage) def getMessageDetails(message): """Decode a raw network message and print the content of each encapsulated layer. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` """ decoder = Decoders.EthDecoder() return decoder.decode( TypeConverter.convert(message.data, HexaString, Raw))
class FileImporter(object): """An Importer than can extracts messages out of files. We recommend to use static methods such as - FileImporter.readFiles(...) - Fileimporter.readFile(...) refer to their documentation to have an overview of the required parameters. >>> from netzob.all import * >>> messages = FileImporter.readFile("./test/resources/files/test_import_text_message.txt").values() >>> print(len(messages)) 13 >>> for m in messages: ... print(repr(m.data)) b'The life that I have' b'Is all that I have' b'And the life that I have' b'Is yours.' b'The love that I have' b'Of the life that I have' b'Is yours and yours and yours.' b'A sleep I shall have' b'A rest I shall have' b'Yet death will be but a pause.' b'For the peace of my years' b'In the long green grass' b'Will be yours and yours and yours.' >>> from netzob.all import * >>> file1 = "./test/resources/files/test_import_raw_message1.dat" >>> file2 = "./test/resources/files/test_import_raw_message2.dat" >>> messages = FileImporter.readFiles([file1, file2], delimitor=b"\\x00\\x00").values() >>> print(len(messages)) 802 >>> print(messages[10].data) b'\\xbdq75\\x18' >>> print(messages[797].data) b'\\xfcJ\\xd1\\xbf\\xff\\xd90\\x98m\\xeb' >>> print(messages[797].file_path) ./test/resources/files/test_import_raw_message2.dat >>> print(messages[707].file_message_number) 353 """ def __init__(self): pass @typeCheck(list, bytes) def readMessages(self, filePathList, delimitor=b"\n"): """Read all the messages found in the specified filePathList and given a delimitor. :param filePathList: paths of the file to parse :type filePathList: a list of :class:`str` :param delimitor: the delimitor used to find messages in the same file :type delimitor: :class:`str` :return: a sorted list of messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("File", "\n".join(errorMessageList)) self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, delimitor) # Create a session and attribute it to messages: session = Session(list(self.messages.values()), name=filePath) for message in self.messages.values(): message.session = session return self.messages @typeCheck(str, bytes) def __readMessagesFromFile(self, filePath, delimitor=b'\n'): if filePath is None or len(str(filePath).strip()) == 0: raise TypeError("Filepath cannot be None or empty") #if delimitor is None or len(str(delimitor)) == 0 or len(delimitor) == 0: # raise TypeError("Delimitor cannot be None or empty") file_content = None with open(filePath, 'rb') as fd: file_content = fd.read() if file_content is None: raise Exception("No content found in '{}'".format(filePath)) if delimitor is None or len(delimitor) == 0: self.messages.add(FileMessage(file_content, file_path=filePath,)) else: for i_data, data in enumerate(file_content.split(delimitor)): if len(data) > 0: self.messages.add(FileMessage(data, file_path = filePath, file_message_number = i_data)) @staticmethod @typeCheck(list, bytes) def readFiles(filePathList, delimitor=b'\n'): """Read all messages from a list of files. A delimitor must be specified to delimit messages. :param filePathList: a list of files to read :type filePathList: a list of :class:`str` :param delimitor: the delimitor. :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readMessages(filePathList, delimitor = delimitor) @staticmethod @typeCheck(str, bytes) def readFile(filePath, delimitor=b'\n'): """Read all messages from the specified file. Messages are found based on the specified delimitor. :param filePath: the pcap path :type filePath: :class:`str` :param delimitor: the delimitor used to find messages in the specified file :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readFiles([filePath], delimitor = delimitor)