def __search(self, searchCases):
"""Execute the different search cases and build
search results that will be returned.
:parameter searchCases: an iterable of tupple (bitarray, :class:`netzob.Inference.Vocabulary.Search.SearchTask.SearchTask`)
:type searchCases: a list
:return: the obtained results
:rtype: a list of :class:`netzob.Inference.Vocabulary.Search.SearchResult.SearchResult`
"""
if searchCases is None:
raise TypeError("There should be at least one search case.")
results = SearchResults()
for (target, searchTask) in searchCases:
if target is None or not isinstance(target, bitarray) or searchTask is None or not isinstance(searchTask, SearchTask):
raise TypeError("Each search case must a tupple made of a bitarray and a SearchTask instance")
ranges = []
for startIndex in target.search(searchTask.data):
self._logger.debug("Search found {}: {}>{}".format(searchTask.data, startIndex, len(searchTask.data)))
ranges.append((startIndex, startIndex + len(searchTask.data)))
if len(ranges) > 0:
results.append(SearchResult(target, searchTask, ranges))
return results
def __search(self, searchCases):
"""Execute the different search cases and build
search results that will be returned.
:parameter searchCases: an iterable of tupple (bitarray, :class:`netzob.Inference.Vocabulary.Search.SearchTask.SearchTask`)
:type searchCases: a list
:return: the obtained results
:rtype: a list of :class:`netzob.Inference.Vocabulary.Search.SearchResult.SearchResult`
"""
if searchCases is None:
raise TypeError("There should be at least one search case.")
results = SearchResults()
for (target, searchTask) in searchCases:
if target is None or not isinstance(
target, bitarray) or searchTask is None or not isinstance(
searchTask, SearchTask):
raise TypeError(
"Each search case must a tupple made of a bitarray and a SearchTask instance"
)
ranges = []
for startIndex in target.search(searchTask.data):
self._logger.debug("Search found {}: {}>{}".format(
searchTask.data, startIndex, len(searchTask.data)))
ranges.append((startIndex, startIndex + len(searchTask.data)))
if len(ranges) > 0:
results.append(SearchResult(target, searchTask, ranges))
return results
def searchDataInMessages(self, datas, messages, addTags=True, inParallel=True, dataLabels=None):
"""Search all the data specified in the given messages. Per default, this operation is executed in parallel.
Example of a search operation executed in sequential
>>> from netzob.all import *
>>> stuff = ["protocols", "communication", "games", "tools", "crypto", "people :)"]
>>> tools = ["Netzob", "zoby", "toto", "your hand", "a knive"]
>>> places = ["my office", "school", "your bedroom", "your car", "hell"]
>>> msgs = [ RawMessage("Reversing {0} with {1} in {2} !".format(s, w, p)) for s in stuff for w in tools for p in places]
>>> sData = [ ASCII("protocol"), ASCII("Reversed"), Integer(10)]
>>> se = SearchEngine()
>>> results = se.searchDataInMessages(sData, msgs, inParallel=False)
>>> print results
25 occurence(s) found.
Example of a search operation executed in parallel
>>> from netzob.all import *
>>> stuff = ["protocols", "communication", "games", "tools", "crypto", "people :)"]
>>> tools = ["Netzob", "zoby", "toto", "your hand", "a knive"]
>>> places = ["my office", "school", "your bedroom", "your car", "hell"]
>>> msgs = [ RawMessage("Reversing {0} with {1} in {2}!".format(s, w, p)) for s in stuff for w in tools for p in places]
>>> print len(msgs)
150
>>> sData = [ASCII("protocol"), ASCII("Reversed"), Integer(10)]
>>> se = SearchEngine()
>>> results = se.searchDataInMessages(sData, msgs, inParallel=True)
>>> print results
25 occurence(s) found.
:parameter data: a list of data to search after. Each data must be provided with its netzob type.
:type data: a list of :class:`netzob.Common.Models.Types.AbstractType.AbstractType`.
:parameter messages: the messages in which the search will take place
:type message: a list of :class:`netzob.Common.Models.Vocabulary.Messages.AbstractMessage`
:keyword addTags: if set to True, visualization functions are added to the message to highlights found results.
:type addTags: :class:`bool`
:keyword inParallel: if set to True, the search will be executed in parallel.
:type addTags: :class:`bool`
:keyword dataLabels: an optionnal dict to attach to each data a label to simplify search results identification
:type dataLabels: dict
:return: a list of search results detailling where and how occurrences where found. Occurences are also
identified in the message through dedicated visualization functions automaticaly added to the message.
:rtype: a list of :class:`netzob.Inference.Vocabulary.SearchEngine.SearchResults.SearchResults`
"""
if datas is None or len(datas) == 0:
raise TypeError("There should be at least one data to search after.")
for data in datas:
if not isinstance(data, AbstractType):
raise TypeError("At least one specified data is not an AbstractType.")
for message in messages:
if not isinstance(message, AbstractMessage):
raise TypeError("At least one specified message is not An AbstractMessage.")
# Remove any duplicate data
noDuplicateDatas = list(set(datas))
results = SearchResults()
if not inParallel:
# Measure start time
# start = time.time()
for message in messages:
results.extend(self.searchDataInMessage(noDuplicateDatas, message, addTags, dataLabels))
# Measure end time
# end = time.time()
else:
# Async result hosting search results
self.asyncResult = []
# Measure start time
# start = time.time()
nbThread = multiprocessing.cpu_count()
# Create a pool of 'nbThead' threads (process)
pool = multiprocessing.Pool(nbThread)
# Execute search operations
pool.map_async(_executeSearch, zip([noDuplicateDatas] * len(messages), messages, [addTags] * len(messages), [dataLabels] * len(messages)), callback=self.__collectResults_cb)
# Waits all alignment tasks finish
pool.close()
pool.join()
# Measure end time
# end = time.time()
results.extend(self.asyncResult)
return results
def searchDataInMessages(self,
datas,
messages,
addTags=True,
inParallel=True,
dataLabels=None):
"""Search all the data specified in the given messages. Per default, this operation is executed in parallel.
Example of a search operation executed in sequential
>>> from netzob.all import *
>>> stuff = ["protocols", "communication", "games", "tools", "crypto", "people :)"]
>>> tools = ["Netzob", "zoby", "toto", "your hand", "a knive"]
>>> places = ["my office", "school", "your bedroom", "your car", "hell"]
>>> msgs = [ RawMessage("Reversing {0} with {1} in {2} !".format(s, w, p).encode('utf-8')) for s in stuff for w in tools for p in places]
>>> sData = [ ASCII("protocol"), ASCII("Reversed"), Integer(10)]
>>> se = SearchEngine()
>>> results = se.searchDataInMessages(sData, msgs, inParallel=False)
>>> print(results)
25 occurence(s) found.
Example of a search operation executed in parallel
>>> from netzob.all import *
>>> stuff = [b"protocols", b"communication", b"games", b"tools", b"crypto", b"people :)"]
>>> tools = [b"Netzob", b"zoby", b"toto", b"your hand", b"a knive"]
>>> places = [b"my office", b"school", b"your bedroom", b"your car", b"hell"]
>>> msgs = [ RawMessage("Reversing {0} with {1} in {2}!".format(s, w, p)) for s in stuff for w in tools for p in places]
>>> print(len(msgs))
150
>>> sData = [ASCII("protocol"), ASCII("Reversed"), Integer(10)]
>>> se = SearchEngine()
>>> results = se.searchDataInMessages(sData, msgs, inParallel=True)
>>> print(results)
25 occurence(s) found.
:parameter data: a list of data to search after. Each data must be provided with its netzob type.
:type data: a list of :class:`netzob.Model.Types.AbstractType.AbstractType`.
:parameter messages: the messages in which the search will take place
:type message: a list of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage`
:keyword addTags: if set to True, visualization functions are added to the message to highlights found results.
:type addTags: :class:`bool`
:keyword inParallel: if set to True, the search will be executed in parallel.
:type addTags: :class:`bool`
:keyword dataLabels: an optionnal dict to attach to each data a label to simplify search results identification
:type dataLabels: dict
:return: a list of search results detailling where and how occurrences where found. Occurences are also
identified in the message through dedicated visualization functions automaticaly added to the message.
:rtype: a list of :class:`netzob.Inference.Vocabulary.SearchEngine.SearchResults.SearchResults`
"""
if datas is None or len(datas) == 0:
raise TypeError(
"There should be at least one data to search after.")
for data in datas:
if not isinstance(data, AbstractType):
raise TypeError(
"At least one specified data is not an AbstractType.")
for message in messages:
if not isinstance(message, AbstractMessage):
raise TypeError(
"At least one specified message is not An AbstractMessage.")
# Remove any duplicate data
noDuplicateDatas = list(set(datas))
results = SearchResults()
if not inParallel:
# Measure start time
# start = time.time()
for message in messages:
results.extend(
self.searchDataInMessage(noDuplicateDatas, message,
addTags, dataLabels))
# Measure end time
# end = time.time()
else:
# Async result hosting search results
self.asyncResult = []
# Measure start time
# start = time.time()
nbThread = multiprocessing.cpu_count()
# Create a pool of 'nbThead' threads (process)
pool = multiprocessing.Pool(nbThread)
# Execute search operations
pool.map_async(
_executeSearch,
list(
zip([noDuplicateDatas] * len(messages), messages, [
addTags
] * len(messages), [dataLabels] * len(messages))),
callback=self.__collectResults_cb)
# Waits all alignment tasks finish
pool.close()
pool.join()
# Measure end time
# end = time.time()
results.extend(self.asyncResult)
return results