class ScaPyCAPimporter(object): def __init__(self, pcapfilename, importLayer=5): # l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values() # type: List[AbstractMessage] self.importLayer = importLayer self.packets = rdpcap(pcapfilename) self._messages = SortedTypedList(AbstractMessage) self._rawmessages = SortedTypedList(AbstractMessage) for pkt in self.packets: # type: Packet self.packetHandler(pkt) @property def messages(self): return self._messages.values() @property def rawMessages(self): return self._rawmessages.values() def packetHandler(self, packet: Packet): epoch = packet.time l1Payload = bytes(packet) if len(l1Payload) == 0: return # Build the RawMessage rawMessage = RawMessage(l1Payload, epoch, source=None, destination=None) if isinstance(packet, RadioTap): # lift layer to Dot11 if there is a RadioTap dummy frame packet = packet.payload if self.importLayer == 2: (l2Proto, l2SrcAddr, l2DstAddr, l2Payload) = self.__decodeLayer2(packet) if len(l2Payload) == 0: return # Build the L2NetworkMessage l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr) self._messages.add(l2Message) self._rawmessages.add(rawMessage) else: # Use Netzob's PCAPImporter if layer 2 is not WLAN raise NetzobImportException("PCAP", "Unsupported import layer. Currently only handles layer 2.", PCAPImporter.INVALID_LAYER2) def __decodeLayer2(self, packet: Packet): """Internal method that parses the specified header and extracts layer2 related proprieties.""" l2Proto = packet.name if isinstance(packet, Raw): print("Ignoring undecoded packet with values:", bytes(packet).hex()) return l2Proto, None, None, "" if isinstance(packet, Dot11): l2DstAddr = packet.fields['addr1'] # receiver address, alt: packet.fields['addr3'] destination address l2SrcAddr = packet.fields['addr2'] # transmitter address, alt: packet.fields['addr4'] source address else: raise NetzobImportException("NEMERE_PCAP", "Unsupported layer 2 protocol " + l2Proto, PCAPImporter.INVALID_LAYER2) l2Payload = bytes(packet.payload) return l2Proto, l2SrcAddr, l2DstAddr, l2Payload
class FileImporter(object): """An Importer than can extracts messages out of files. We recommend to use static methods such as - FileImporter.readFiles(...) - Fileimporter.readFile(...) refer to their documentation to have an overview of the required parameters. >>> from netzob.all import * >>> messages = FileImporter.readFile("./test/resources/files/test_import_text_message.txt").values() >>> print(len(messages)) 13 >>> for m in messages: ... print(repr(m.data)) b'The life that I have' b'Is all that I have' b'And the life that I have' b'Is yours.' b'The love that I have' b'Of the life that I have' b'Is yours and yours and yours.' b'A sleep I shall have' b'A rest I shall have' b'Yet death will be but a pause.' b'For the peace of my years' b'In the long green grass' b'Will be yours and yours and yours.' >>> from netzob.all import * >>> file1 = "./test/resources/files/test_import_raw_message1.dat" >>> file2 = "./test/resources/files/test_import_raw_message2.dat" >>> messages = FileImporter.readFiles([file1, file2], delimitor=b"\\x00\\x00").values() >>> print(len(messages)) 802 >>> print(messages[10].data) b'\\xbdq75\\x18' >>> print(messages[797].data) b'\\xfcJ\\xd1\\xbf\\xff\\xd90\\x98m\\xeb' >>> print(messages[797].file_path) ./test/resources/files/test_import_raw_message2.dat >>> print(messages[707].file_message_number) 353 """ def __init__(self): pass @typeCheck(list, str, int, int) def readMessages(self, filePathList, delimitor=b"\n"): """Read all the messages found in the specified filePathList and given a delimitor. :param filePathList: paths of the file to parse :type filePathList: a list of :class:`str` :param delimitor: the delimitor used to find messages in the same file :type delimitor: :class:`str` :return: a sorted list of messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _( "Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("File", "\n".join(errorMessageList)) self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, delimitor) return self.messages @typeCheck(str, bytes) def __readMessagesFromFile(self, filePath, delimitor): if filePath is None or len(str(filePath).strip()) == 0: raise TypeError("Filepath cannot be None or empty") if delimitor is None or len(str(delimitor)) == 0: raise TypeError("Delimitor cannot be None or empty") file_content = None with open(filePath, 'rb') as fd: file_content = fd.read() if file_content is None: raise Exception("No content found in '{}'".format(filePath)) for i_data, data in enumerate(file_content.split(delimitor)): if len(data) > 0: self.messages.add( FileMessage(data, file_path=filePath, file_message_number=i_data)) @staticmethod @typeCheck(list, str) def readFiles(filePathList, delimitor=b'\n'): """Read all messages from a list of files. A delimitor must be specified to delimit messages. :param filePathList: a list of files to read :type filePathList: a list of :class:`str` :param delimitor: the delimitor. :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readMessages(filePathList, delimitor=delimitor) @staticmethod @typeCheck(str, str) def readFile(filePath, delimitor=b'\n'): """Read all messages from the specified file. Messages are found based on the specified delimitor. :param filePath: the pcap path :type filePath: :class:`str` :param delimitor: the delimitor used to find messages in the specified file :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readFiles([filePath], delimitor=delimitor)
def readMessages(self, filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False, ): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload). Finally, the number of packets to capture can be specified. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("PCAP", "\n".join(errorMessageList)) # Verify the expected import layer availableLayers = [1, 2, 3, 4, 5] if importLayer not in availableLayers: raise Exception( "Only layers level {0} are available.".format(availableLayers)) self.importLayer = importLayer # Call the method that does the import job for each PCAP file self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, bpfFilter, nbPackets) #Create a session and attribute it to messages: session = Session(list(self.messages.values()),name=filePath) for message in self.messages.values(): message.session = session # if requested, we merge consecutive messages that share same source and destination if mergePacketsInFlow: mergedMessages = SortedTypedList(AbstractMessage) previousMessage = None for message in self.messages.values(): if previousMessage is not None and message.source == previousMessage.source and message.destination == previousMessage.destination: previousMessage.data += message.data else: mergedMessages.add(message) previousMessage = message self.messages = mergedMessages return self.messages
class Session(object): """A session includes messages exchanged in the same session. Messages are automaticaly sorted. Applicative data can be attached to sessions. >>> import time >>> from netzob.all import * >>> # we create 3 messages >>> msg1 = RawMessage("ACK", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:05", "%d %b %y %H:%M:%S"))) >>> msg2 = RawMessage("SYN", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:01", "%d %b %y %H:%M:%S"))) >>> msg3 = RawMessage("SYN/ACK", source="B", destination="A", date=time.mktime(time.strptime("9 Aug 13 10:45:03", "%d %b %y %H:%M:%S"))) >>> session = Session([msg1, msg2, msg3]) >>> print(session.messages.values()[0].data) SYN >>> print(session.messages.values()[1].data) SYN/ACK >>> print(session.messages.values()[2].data) ACK """ def __init__(self, messages=None, _id=None, applicativeData=None, name="Session"): """ :parameter messages: the messages exchanged in the current session :type data: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` :parameter _id: the unique identifier of the session :type _id: :class:`uuid.UUID` :keyword applicativeData: a list of :class:`netzob.Model.Vocabulary.ApplicaticeData.ApplicativeData` """ self.__messages = SortedTypedList(AbstractMessage) self.__applicativeData = TypedList(ApplicativeData) if messages is None: messages = [] self.messages = messages if _id is None: _id = uuid.uuid4() self.id = _id if applicativeData is None: applicativeData = [] self.applicativeData = applicativeData self.name = name @property def id(self): """The unique identifier of the session. :type: :class:`uuid.UUID` """ return self.__id @id.setter @typeCheck(uuid.UUID) def id(self, _id): if _id is None: raise TypeError("Id cannot be None") self.__id = _id @property def messages(self): """The messages exchanged in the current session. Messages are sorted. :type: a :class:`netzob.Common.Utils.TypedList.TypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` """ return self.__messages def clearMessages(self): """Delete all the messages attached to the current session""" for msg in list(self.__messages.values()): msg.session = None self.__messages.clear() @messages.setter def messages(self, messages): if messages is None: messages = [] # First it checks the specified messages are all AbstractMessages for msg in messages: if not isinstance(msg, AbstractMessage): raise TypeError( "Cannot add messages of type {0} in the session, only AbstractMessages are allowed." .format(type(msg))) self.clearMessages() for message in messages: self.__messages.add(message) message.session = self @property def applicativeData(self): """Applicative data attached to the current session. >>> from netzob.all import * >>> appData = ApplicativeData("test", Integer(20)) >>> session = Session(applicativeData=[appData]) >>> print(len(session.applicativeData)) 1 >>> appData2 = ApplicativeData("test2", ASCII("helloworld")) >>> session.applicativeData.append(appData2) >>> print(len(session.applicativeData)) 2 >>> print(session.applicativeData[0]) Applicative Data: test=Integer=20 ((8, 8))) >>> print(session.applicativeData[1]) Applicative Data: test2=ASCII=helloworld ((0, 80))) :type: a list of :class:`netzob.Model.Vocabulary.ApplicativeData.ApplicativeData`. """ return self.__applicativeData def clearApplicativeData(self): while (len(self.__applicativeData) > 0): self.__applicativeData.pop() @applicativeData.setter def applicativeData(self, applicativeData): for app in applicativeData: if not isinstance(app, ApplicativeData): raise TypeError( "Cannot add an applicative data with type {0}, only ApplicativeData accepted." .format(type(app))) self.clearApplicativeData() for app in applicativeData: self.applicativeData.append(app) @property def name(self): return self.__name @name.setter @typeCheck(str) def name(self, _name): if _name is None: raise TypeError("Name cannot be None") self.__name = _name def getEndpointsList(self): """Retrieve all the endpoints couples that are present in the session. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getEndpointsList())) 2 >>> print(session.getEndpointsList()) [('A', 'B'), ('A', 'C')] :return: a list containing couple of endpoints (src, dst). :rtype: a :class:`list` """ endpointsList = [] for message in list(self.messages.values()): src = message.source dst = message.destination endpoints1 = (src, dst) endpoints2 = (dst, src) if (not endpoints1 in endpointsList) and (not endpoints2 in endpointsList): endpointsList.append(endpoints1) return endpointsList def getTrueSessions(self): """Retrieve the true sessions embedded in the current session. A session is here characterized by a uniq endpoints couple. TODO: a more precise solution would be to use flow reconstruction (as in TCP). >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getTrueSessions())) 2 >>> for trueSession in session.getTrueSessions(): ... print(trueSession.name) Session: 'A' - 'B' Session: 'A' - 'C' :return: a list containing true sessions embedded in the current session. :rtype: a :class:`list` """ trueSessions = [] for endpoints in self.getEndpointsList(): trueSessionMessages = [] src = None dst = None for message in list(self.messages.values()): if message.source in endpoints and message.destination in endpoints: trueSessionMessages.append(message) if src is None: src = message.source if dst is None: dst = message.destination trueSession = Session(messages=trueSessionMessages, applicativeData=self.applicativeData, name="Session: '" + str(src) + "' - '" + str(dst) + "'") trueSessions.append(trueSession) return trueSessions def isTrueSession(self): """Tell if the current session is true. A session is said to be true if the communication flow pertain to a uniq applicative session between a couple of endpoints. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> print(session.isTrueSession()) True :return: a boolean telling if the current session is a true one (i.e. it corresponds to a uniq applicative session between two endpoints). :rtype: a :class:`bool` """ if len(self.getTrueSessions()) == 1: return True else: return False @typeCheck(list) def abstract(self, symbolList): """This method abstract each message of the current session into symbols according to a list of symbols given as parameter. >>> from netzob.all import * >>> symbolSYN = Symbol([Field(ASCII("SYN"))], name="Symbol_SYN") >>> symbolSYNACK = Symbol([Field(ASCII("SYN/ACK"))], name="Symbol_SYNACK") >>> symbolACK = Symbol([Field(ASCII("ACK"))], name="Symbol_ACK") >>> symbolList = [symbolSYN, symbolSYNACK, symbolACK] >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> if session.isTrueSession(): ... for src, dst, sym in session.abstract(symbolList): ... print(str(src) + " - " + str(dst) + " : " + str(sym.name)) A - B : Symbol_SYN B - A : Symbol_SYNACK A - B : Symbol_ACK :return: a list of tuples containing the following elements : (source, destination, symbol). :rtype: a :class:`list` """ abstractSession = [] if not self.isTrueSession(): self._logger.warn( "The current session cannot be abstracted as it not a true session (i.e. it may contain inner true sessions)." ) return abstractSession for message in list(self.messages.values()): symbol = AbstractField.abstract(message.data, symbolList) abstractSession.append( (message.source, message.destination, symbol)) return abstractSession
class Session(object): """A session includes messages exchanged in the same session. Messages are automaticaly sorted. Applicative data can be attached to sessions. >>> import time >>> from netzob.all import * >>> # we create 3 messages >>> msg1 = RawMessage("ACK", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:05", "%d %b %y %H:%M:%S"))) >>> msg2 = RawMessage("SYN", source="A", destination="B", date=time.mktime(time.strptime("9 Aug 13 10:45:01", "%d %b %y %H:%M:%S"))) >>> msg3 = RawMessage("SYN/ACK", source="B", destination="A", date=time.mktime(time.strptime("9 Aug 13 10:45:03", "%d %b %y %H:%M:%S"))) >>> session = Session([msg1, msg2, msg3]) >>> print(session.messages.values()[0].data) SYN >>> print(session.messages.values()[1].data) SYN/ACK >>> print(session.messages.values()[2].data) ACK """ def __init__(self, messages=None, _id=None, applicativeData=None, name="Session"): """ :parameter messages: the messages exchanged in the current session :type data: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` :parameter _id: the unique identifier of the session :type _id: :class:`uuid.UUID` :keyword applicativeData: a list of :class:`netzob.Model.Vocabulary.ApplicaticeData.ApplicativeData` """ self.__messages = SortedTypedList(AbstractMessage) self.__applicativeData = TypedList(ApplicativeData) if messages is None: messages = [] self.messages = messages if _id is None: _id = uuid.uuid4() self.id = _id if applicativeData is None: applicativeData = [] self.applicativeData = applicativeData self.name = name @property def id(self): """The unique identifier of the session. :type: :class:`uuid.UUID` """ return self.__id @id.setter @typeCheck(uuid.UUID) def id(self, _id): if _id is None: raise TypeError("Id cannot be None") self.__id = _id @property def messages(self): """The messages exchanged in the current session. Messages are sorted. :type: a :class:`netzob.Common.Utils.TypedList.TypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage` """ return self.__messages def clearMessages(self): """Delete all the messages attached to the current session""" for msg in list(self.__messages.values()): msg.session = None self.__messages.clear() @messages.setter def messages(self, messages): if messages is None: messages = [] # First it checks the specified messages are all AbstractMessages for msg in messages: if not isinstance(msg, AbstractMessage): raise TypeError( "Cannot add messages of type {0} in the session, only AbstractMessages are allowed.". format(type(msg))) self.clearMessages() for message in messages: self.__messages.add(message) message.session = self @property def applicativeData(self): """Applicative data attached to the current session. >>> from netzob.all import * >>> appData = ApplicativeData("test", Integer(20)) >>> session = Session(applicativeData=[appData]) >>> print(len(session.applicativeData)) 1 >>> appData2 = ApplicativeData("test2", ASCII("helloworld")) >>> session.applicativeData.append(appData2) >>> print(len(session.applicativeData)) 2 >>> print(session.applicativeData[0]) Applicative Data: test=Integer=20 ((8, 8))) >>> print(session.applicativeData[1]) Applicative Data: test2=ASCII=helloworld ((0, 80))) :type: a list of :class:`netzob.Model.Vocabulary.ApplicativeData.ApplicativeData`. """ return self.__applicativeData def clearApplicativeData(self): while (len(self.__applicativeData) > 0): self.__applicativeData.pop() @applicativeData.setter def applicativeData(self, applicativeData): for app in applicativeData: if not isinstance(app, ApplicativeData): raise TypeError( "Cannot add an applicative data with type {0}, only ApplicativeData accepted.". format(type(app))) self.clearApplicativeData() for app in applicativeData: self.applicativeData.append(app) @property def name(self): return self.__name @name.setter @typeCheck(str) def name(self, _name): if _name is None: raise TypeError("Name cannot be None") self.__name = _name def getEndpointsList(self): """Retrieve all the endpoints couples that are present in the session. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getEndpointsList())) 2 >>> print(session.getEndpointsList()) [('A', 'B'), ('A', 'C')] :return: a list containing couple of endpoints (src, dst). :rtype: a :class:`list` """ endpointsList = [] for message in list(self.messages.values()): src = message.source dst = message.destination endpoints1 = (src, dst) endpoints2 = (dst, src) if (not endpoints1 in endpointsList) and ( not endpoints2 in endpointsList): endpointsList.append(endpoints1) return endpointsList def getTrueSessions(self): """Retrieve the true sessions embedded in the current session. A session is here characterized by a uniq endpoints couple. TODO: a more precise solution would be to use flow reconstruction (as in TCP). >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="C") >>> session = Session([msg1, msg2, msg3]) >>> print(len(session.getTrueSessions())) 2 >>> for trueSession in session.getTrueSessions(): ... print(trueSession.name) Session: 'A' - 'B' Session: 'A' - 'C' :return: a list containing true sessions embedded in the current session. :rtype: a :class:`list` """ trueSessions = [] for endpoints in self.getEndpointsList(): trueSessionMessages = [] src = None dst = None for message in list(self.messages.values()): if message.source in endpoints and message.destination in endpoints: trueSessionMessages.append(message) if src is None: src = message.source if dst is None: dst = message.destination trueSession = Session( messages=trueSessionMessages, applicativeData=self.applicativeData, name="Session: '" + str(src) + "' - '" + str(dst) + "'") trueSessions.append(trueSession) return trueSessions def isTrueSession(self): """Tell if the current session is true. A session is said to be true if the communication flow pertain to a uniq applicative session between a couple of endpoints. >>> from netzob.all import * >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> print(session.isTrueSession()) True :return: a boolean telling if the current session is a true one (i.e. it corresponds to a uniq applicative session between two endpoints). :rtype: a :class:`bool` """ if len(self.getTrueSessions()) == 1: return True else: return False @typeCheck(list) def abstract(self, symbolList): """This method abstract each message of the current session into symbols according to a list of symbols given as parameter. >>> from netzob.all import * >>> symbolSYN = Symbol([Field(ASCII("SYN"))], name="Symbol_SYN") >>> symbolSYNACK = Symbol([Field(ASCII("SYN/ACK"))], name="Symbol_SYNACK") >>> symbolACK = Symbol([Field(ASCII("ACK"))], name="Symbol_ACK") >>> symbolList = [symbolSYN, symbolSYNACK, symbolACK] >>> msg1 = RawMessage("SYN", source="A", destination="B") >>> msg2 = RawMessage("SYN/ACK", source="B", destination="A") >>> msg3 = RawMessage("ACK", source="A", destination="B") >>> session = Session([msg1, msg2, msg3]) >>> if session.isTrueSession(): ... for src, dst, sym in session.abstract(symbolList): ... print(str(src) + " - " + str(dst) + " : " + str(sym.name)) A - B : Symbol_SYN B - A : Symbol_SYNACK A - B : Symbol_ACK :return: a list of tuples containing the following elements : (source, destination, symbol). :rtype: a :class:`list` """ abstractSession = [] if not self.isTrueSession(): self._logger.warn( "The current session cannot be abstracted as it not a true session (i.e. it may contain inner true sessions)." ) return abstractSession for message in list(self.messages.values()): (symbol, structured_message) = AbstractField.abstract(message.data, symbolList) abstractSession.append((message.source, message.destination, symbol)) return abstractSession
def readMessages(self, filePathList, bpfFilter="", importLayer=5, nbPackets=0, mergePacketsInFlow=False, ): """Read all messages from a list of PCAP files. A BPF filter can be set to limit the captured packets. The layer of import can also be specified: - When layer={1, 2}, it means we want to capture a raw layer (such as Ethernet). - If layer=3, we capture at the network level (such as IP). - If layer=4, we capture at the transport layer (such as TCP or UDP). - If layer=5, we capture at the applicative layer (such as the TCP or UDP payload). Finally, the number of packets to capture can be specified. :param filePathList: the messages to cluster. :type filePathList: a list of :class:`str` :param bpfFilter: a string representing a BPF filter. :type bpfFilter: :class:`str` :param importLayer: an integer representing the protocol layer to start importing. :type importLayer: :class:`int` :param nbPackets: the number of packets to import :type nbPackets: :class:`int` :param mergePacketsInFlow: if True, consecutive packets with same source and destination ar merged (i.e. to mimic a flow) :type mergePacketsInFlow: :class:`bool` :return: a list of captured messages :rtype: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("PCAP", "\n".join(errorMessageList)) # Verify the expected import layer availableLayers = [1, 2, 3, 4, 5] if not importLayer in availableLayers: raise Exception( "Only layers level {0} are available.".format(availableLayers)) self.importLayer = importLayer # Call the method that does the import job for each PCAP file self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, bpfFilter, nbPackets) # if requested, we merge consecutive messages that share same source and destination if mergePacketsInFlow: mergedMessages = SortedTypedList(AbstractMessage) previousMessage = None for message in self.messages.values(): if previousMessage is not None and message.source == previousMessage.source and message.destination == previousMessage.destination: previousMessage.data += message.data else: mergedMessages.add(message) previousMessage = message self.messages = mergedMessages return self.messages
class FileImporter(object): """An Importer than can extracts messages out of files. We recommend to use static methods such as - FileImporter.readFiles(...) - Fileimporter.readFile(...) refer to their documentation to have an overview of the required parameters. >>> from netzob.all import * >>> messages = FileImporter.readFile("./test/resources/files/test_import_text_message.txt").values() >>> print(len(messages)) 13 >>> for m in messages: ... print(repr(m.data)) b'The life that I have' b'Is all that I have' b'And the life that I have' b'Is yours.' b'The love that I have' b'Of the life that I have' b'Is yours and yours and yours.' b'A sleep I shall have' b'A rest I shall have' b'Yet death will be but a pause.' b'For the peace of my years' b'In the long green grass' b'Will be yours and yours and yours.' >>> from netzob.all import * >>> file1 = "./test/resources/files/test_import_raw_message1.dat" >>> file2 = "./test/resources/files/test_import_raw_message2.dat" >>> messages = FileImporter.readFiles([file1, file2], delimitor=b"\\x00\\x00").values() >>> print(len(messages)) 802 >>> print(messages[10].data) b'\\xbdq75\\x18' >>> print(messages[797].data) b'\\xfcJ\\xd1\\xbf\\xff\\xd90\\x98m\\xeb' >>> print(messages[797].file_path) ./test/resources/files/test_import_raw_message2.dat >>> print(messages[707].file_message_number) 353 """ def __init__(self): pass @typeCheck(list, bytes) def readMessages(self, filePathList, delimitor=b"\n"): """Read all the messages found in the specified filePathList and given a delimitor. :param filePathList: paths of the file to parse :type filePathList: a list of :class:`str` :param delimitor: the delimitor used to find messages in the same file :type delimitor: :class:`str` :return: a sorted list of messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ # Verify the existence of input files errorMessageList = [] for filePath in filePathList: try: fp = open(filePath) fp.close() except IOError as e: errorMessage = _("Error while trying to open the " + "file {0}.").format(filePath) if e.errno == errno.EACCES: errorMessage = _("Error while trying to open the file " + "{0}, more permissions are required for " + "reading it.").format(filePath) errorMessageList.append(errorMessage) self._logger.warn(errorMessage) if errorMessageList != []: raise NetzobImportException("File", "\n".join(errorMessageList)) self.messages = SortedTypedList(AbstractMessage) for filePath in filePathList: self.__readMessagesFromFile(filePath, delimitor) return self.messages @typeCheck(str, bytes) def __readMessagesFromFile(self, filePath, delimitor=b'\n'): if filePath is None or len(str(filePath).strip()) == 0: raise TypeError("Filepath cannot be None or empty") if delimitor is None or len(str(delimitor)) == 0: raise TypeError("Delimitor cannot be None or empty") file_content = None with open(filePath, 'rb') as fd: file_content = fd.read() if file_content is None: raise Exception("No content found in '{}'".format(filePath)) for i_data, data in enumerate(file_content.split(delimitor)): if len(data) > 0: self.messages.add(FileMessage(data, file_path = filePath, file_message_number = i_data)) @staticmethod @typeCheck(list, bytes) def readFiles(filePathList, delimitor=b'\n'): """Read all messages from a list of files. A delimitor must be specified to delimit messages. :param filePathList: a list of files to read :type filePathList: a list of :class:`str` :param delimitor: the delimitor. :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readMessages(filePathList, delimitor = delimitor) @staticmethod @typeCheck(str, bytes) def readFile(filePath, delimitor=b'\n'): """Read all messages from the specified file. Messages are found based on the specified delimitor. :param filePath: the pcap path :type filePath: :class:`str` :param delimitor: the delimitor used to find messages in the specified file :type delimitor: :class:`str` :return: a list of captured messages :rtype: a :class:`netzob.Common.Utils.SortedTypedList.SortedTypedList` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage` """ importer = FileImporter() return importer.readFiles([filePath], delimitor = delimitor)