def displayMessage(self, message):
# Clean the hexdump view
self.textview.get_buffer().delete(self.textview.get_buffer().get_start_iter(), self.textview.get_buffer().get_end_iter())
# Fecth the content of the message to display
hexContent = TypeConvertor.hexdump(TypeConvertor.netzobRawToPythonRaw(message.getData()))
# Update the hexdump
self.textview.get_buffer().insert_with_tags_by_name(self.textview.get_buffer().get_start_iter(), hexContent, "normalTag")
def test_alignmentOfEquivalentMessages(self):
alignmentSolution = NeedlemanAndWunsch(8)
nbTest = 1000
for i_test in range(0, nbTest):
common_pattern = self.generateRandomString(30, 40)
# Generate the content of two messages
data1 = TypeConvertor.stringToNetzobRaw(common_pattern)
data2 = TypeConvertor.stringToNetzobRaw(common_pattern)
# Create the messages
message1 = RawMessage(str(uuid.uuid4()), str(time.time()), data1)
message2 = RawMessage(str(uuid.uuid4()), str(time.time()), data2)
(scores, alignment) = alignmentSolution.alignTwoMessages(
False, message1, message2)
(score1, score2, score3) = scores
self.assertEqual(score1, 100.0)
self.assertEqual(score2, 100.0)
self.assertEqual(score3, 100.0)
(scores, alignment) = alignmentSolution.alignTwoMessages(
True, message1, message2)
(score1, score2, score3) = scores
self.assertEqual(score1, 100.0)
self.assertEqual(score2, 100.0)
self.assertEqual(score3, 100.0)
def displayMessage(self, message):
# Clean the hexdump view
self.textview.get_buffer().delete(self.textview.get_buffer().get_start_iter(), self.textview.get_buffer().get_end_iter())
# Fecth the content of the message to display
hexContent = TypeConvertor.hexdump(TypeConvertor.netzobRawToPythonRaw(message.getData()))
# Update the hexdump
self.textview.get_buffer().insert_with_tags_by_name(self.textview.get_buffer().get_start_iter(), hexContent, "normalTag")
def learn(self, value, indice, negative, vocabulary, memory):
if self.format == Format.ASCII:
currentContent = TypeConvertor.bin2string(value[indice:])
IPRegex = re.compile("(((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))")
hasMatched = False
for t in range(min(len(currentContent), 15), 7, -1):
currentPossibleIP = currentContent[:t]
result = IPRegex.match(currentPossibleIP)
if result != None:
hasMatched = True
elif hasMatched:
break
if hasMatched:
result = currentContent[:t + 2]
self.log.debug("Learn from received message : " + str(result))
strCurrentValue = str(result)
binCurrentValue = TypeConvertor.string2bin(result, 'big')
memory.memorize(self, (binCurrentValue, strCurrentValue))
return len(TypeConvertor.string2bin(result, 'big'))
else:
self.log.debug("Compare on format was not successfull")
return -1
else:
raise NotImplementedError("Error, the current variable (IPv4Variable) doesn't support function compareFormat in this case")
def test_serializeValues(self):
# Generate randoms values and retrieve their
# serializations
nb_test = 100
for i_test in range(0, nb_test):
values = []
nb_values = random.randint(5, 200)
for i_value in range(0, nb_values):
# Generate the content of a random value
value = TypeConvertor.stringToNetzobRaw(
self.generateRandomString(5, 100))
values.append(value)
# start the serialization process
(serializedValues,
format) = TypeConvertor.serializeValues(values, 8)
# start the deserialisation process
deserializedValues = TypeConvertor.deserializeValues(
serializedValues, format)
for i_value in range(0, len(values)):
value = values[i_value]
self.assertEqual(value, deserializedValues[i_value])
def save(message, xmlMessages, namespace_project, namespace_common):
root = etree.SubElement(xmlMessages, "{" + namespace_common + "}message")
root.set("id", str(message.getID()))
root.set("timestamp", str(message.getTimestamp()))
root.set("{http://www.w3.org/2001/XMLSchema-instance}type", "netzob-common:FileMessage")
# data
subData = etree.SubElement(root, "{" + namespace_common + "}data")
subData.text = str(message.getData())
# line number
subLineNumber = etree.SubElement(root, "{" + namespace_common + "}lineNumber")
subLineNumber.text = str(message.getLineNumber())
# filename
subFilename = etree.SubElement(root, "{" + namespace_common + "}filename")
subFilename.text = message.getFilename()
# creationDate
subCreationDate = etree.SubElement(root, "{" + namespace_common + "}creationDate")
subCreationDate.text = TypeConvertor.pythonDatetime2XSDDatetime(message.getCreationDate())
# creationDate
subModificationDate = etree.SubElement(root, "{" + namespace_common + "}modificationDate")
subModificationDate.text = TypeConvertor.pythonDatetime2XSDDatetime(message.getModificationDate())
# owner
subOwner = etree.SubElement(root, "{" + namespace_common + "}owner")
subOwner.text = message.getOwner()
# size
subSize = etree.SubElement(root, "{" + namespace_common + "}size")
subSize.text = str(message.getSize())
def getSearchedDataForString(self, value):
# Creation of a SearchTask
task = SearchTask(value, value, Format.STRING)
task.registerVariation(TypeConvertor.stringToNetzobRaw(value), "String representation of '%s'" % value)
task.registerVariation(TypeConvertor.stringToNetzobRaw(value[::-1]), "Inverted string representation of '%s'" % value[::-1])
task.registerVariation(TypeConvertor.stringToNetzobRaw(value.decode('utf-8')), "String representation of '%s' encoded in UTF-8" % value)
return [task]
def compareFormat(self, value, indice, negative, vocabulary, memory):
"""compareFormat:
Compute if the provided data is "format-compliant" and return the size of the biggest compliant data.
@type value: bitarray.bitarray
@param value: a bit array a subarray of which we compare to the current variable binray value.
@type indice: integer
@param indice: the starting point of comparison in value.
@type negative: boolean
@param negative: tells if we use the variable or a logical not of it.
@type vocabulary: netzob.Common.Vocabulary.Vocabulary
@param vocabulary: the vocabulary of the current project.
@type memory: netzob.Common.MMSTD.Memory.Memory
@param memory: a memory which can contain a former value of the variable.
@rtype: integer
@return: the size of the biggest compliant data, -1 if it does not comply.
"""
tmp = value[indice:]
size = len(tmp)
if size <= 16:
self.log.debug("Too small, not even 16 bits available (2 letters)")
return -1
for i in range(size, 16, -1):
subValue = value[indice:indice + i - 1]
if (i - 1) % 8 == 0:
strVal = TypeConvertor.bin2string(TypeConvertor.strBitarray2Bitarray(subValue))
typeIdentifier = TypeIdentifier()
if typeIdentifier.isAscii(strVal):
self.log.debug("Its an ascii : (" + str(strVal) + ")")
if (not ' ' in strVal and not '\n' in strVal and not '\r' in strVal):
self.log.debug("Its an ascii without space : (" + str(strVal) + ")")
self.log.debug("Binary value of the ascii : %s" % str(TypeConvertor.strBitarray2Bitarray(subValue)))
return indice + i - 1
return -1
def test_semanticAlignment_bug1(self):
"""test_semanticAlignment_bug1:
A bug on the semantic alignment has been identified which prevent
the computation of a valid regex. This test verifies the bug is not comming back.
@date 18/04/2013
"""
firstname1 = "antoine"
email1 = "*****@*****.**"
firstname2 = "luc"
email2 = "*****@*****.**"
msg1 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("6" + firstname1 + "GAHFSHQS" + email1))
msg2 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("3" + firstname2 + "CVSDHISD" + email2))
project = Project(uuid.uuid4(), "Experiment", datetime.now(), "")
nwEngine = NeedlemanAndWunsch(8, project, False, None)
symbol = Symbol(uuid.uuid4(), "Test", project)
symbol.addMessages([msg1, msg2])
msg1.addSemanticTag("firstname", 2, 2 + len(firstname1) * 2)
msg1.addSemanticTag("email", 2 + len(firstname1) * 2 + 16, 2 + len(firstname1) * 2 + 16 + len(email1) * 2)
msg2.addSemanticTag("firstname", 2, 2 + len(firstname2) * 2)
msg2.addSemanticTag("email", 2 + len(firstname2) * 2 + 16, 2 + len(firstname2) * 2 + 16 + len(email2) * 2)
nwEngine.alignField(symbol.getField())
symbol.getField().setFormat(Format.STRING)
print("Computed Regex : {0}".format(symbol.getRegex()))
print(symbol.getCells(True))
computedFields = symbol.getExtendedFields()
self.assertTrue(len(computedFields) > 1, "Only one field has been computed which tells us something went wrong.")
def updateDisplayFollowingSplitPosition(self):
messages = self.field.getCells()
# Colorize text according to position
self.view.buffer.get_buffer().set_text("")
for m in messages:
# Crate padding in case of right alignment
if self.split_align == "right":
padding = " " * (self.split_max_len - len(m))
self.view.buffer.get_buffer().insert_with_tags_by_name(
self.view.buffer.get_buffer().get_end_iter(), padding,
"greenTag")
split_index = -self.split_position
else:
split_index = self.split_position
leftContent = m[:split_index]
rightContent = m[split_index:]
self.view.buffer.get_buffer().insert_with_tags_by_name(
self.view.buffer.get_buffer().get_end_iter(),
TypeConvertor.encodeNetzobRawToGivenType(
leftContent, self.field.getFormat()) + " ", "redTag")
self.view.buffer.get_buffer().insert_with_tags_by_name(
self.view.buffer.get_buffer().get_end_iter(),
TypeConvertor.encodeNetzobRawToGivenType(
rightContent, self.field.getFormat()) + "\n", "greenTag")
value = self.split_position * (self.view.getMaxSizeOfHBuffer() /
self.split_max_len)
self.view.adjustHPositionOfBuffer(value)
def processUPGMA(self):
self.log.debug("Computing the associated matrix")
# Serialize the symbols
(serialSymbols, formatSymbols) = TypeConvertor.serializeSymbols(self.symbols, self.unitSize, self.scores)
self.log.debug("Clustering input format " + formatSymbols)
# Execute the Clustering part in C
debug = False
logging.debug("Execute the clustering part in C ...")
(i_max, j_max, maxScore, scores) = _libNeedleman.getHighestEquivalentGroup(self.doInternalSlick, len(self.symbols), formatSymbols, serialSymbols, self.cb_executionStatus, debug)
listScores = TypeConvertor.deserializeScores(self.symbols, scores)
# Retrieve the scores for each association of symbols
self.scores = {}
for (iuid, juid, score) in listScores:
if iuid not in self.scores.keys():
self.scores[iuid] = {}
if juid not in self.scores.keys():
self.scores[juid] = {}
self.scores[iuid][juid] = score
if iuid not in self.scores[juid].keys():
self.scores[juid][iuid] = score
# Reduce the UPGMA matrix (merge symbols by similarity)
self.computePhylogenicTree()
return (i_max, j_max, maxScore)
def loadFromXML(rootElement, namespace, version, id, timestamp, data):
# Retrieves the lineNumber (default -1)
msg_lineNumber = int(rootElement.find("{" + namespace + "}lineNumber").text)
# Retrieves the filename
msg_filename = rootElement.find("{" + namespace + "}filename").text.encode("utf-8")
# Retrieves the creation date
msg_creationDate = TypeConvertor.xsdDatetime2PythonDatetime(rootElement.find("{" + namespace + "}creationDate").text)
# Retrieves the modification date
if rootElement.find("{" + namespace + "}modificationDate").text is not None:
msg_modificationDate = TypeConvertor.xsdDatetime2PythonDatetime(rootElement.find("{" + namespace + "}modificationDate").text)
else:
msg_modificationDate = msg_creationDate
# Retrieves the owner
msg_owner = rootElement.find("{" + namespace + "}owner").text
# Retrieves the size
msg_size = int(rootElement.find("{" + namespace + "}size").text)
# TODO : verify this ! Circular imports in python !
# WARNING : verify this ! Circular imports in python !
from netzob.Common.Models.FileMessage import FileMessage
result = FileMessage(id, timestamp, data, msg_filename, msg_creationDate, msg_modificationDate, msg_owner, msg_size, msg_lineNumber)
return result
def receiveSymbolWithTimeout(self, timeout):
# First we read from the input the message
receivedData = self.communicationChannel.read(timeout)
nbMaxAttempts = 5
if receivedData is None:
self.log.warn("The communication channel seems to be closed !")
# return (EmptySymbol(), None)
return (None, None)
if len(receivedData) > 0:
now = datetime.datetime.now()
receptionTime = now.strftime("%H:%M:%S")
self.log.info("Received following message : " + TypeConvertor.bin2strhex(receivedData))
# Now we abstract the message
symbol = self.abstract(receivedData)
# We store the received messages its time and its abstract representation
self.inputMessages.append([receptionTime, TypeConvertor.bin2strhex(receivedData), symbol.getName()])
self.registerInputSymbol(receptionTime, TypeConvertor.bin2strhex(receivedData), symbol)
return (symbol, receivedData)
else:
if len(self.manipulatedSymbols) > nbMaxAttempts:
if self.manipulatedSymbols[len(self.manipulatedSymbols) - 1].getType() == EmptySymbol.TYPE or self.manipulatedSymbols[len(self.manipulatedSymbols) - 1].getType() == UnknownSymbol.TYPE:
self.log.warn("Consider client has disconnected since no valid symbol received after " + str(nbMaxAttempts) + " attempts")
return (None, None)
now = datetime.datetime.now()
receptionTime = now.strftime("%H:%M:%S")
symbol = EmptySymbol()
self.registerInputSymbol(receptionTime, "", symbol)
return (symbol, None)
def read(self, timeout):
self.log.debug("Reading from the socket some data (timeout = " + str(timeout))
result = bitarray(endian='big')
chars = []
try:
if timeout > 0:
ready = select.select([self.socket], [], [], timeout)
if ready[0]:
chars = self.socket.recv(4096)
else:
ready = select.select([self.socket], [], [])
self.log.debug("ready = " + str(ready[0]))
if ready[0]:
chars = self.socket.recv(4096)
except:
self.log.debug("Impossible to read from the network socket")
return None
if (len(chars) == 0):
return result
result = TypeConvertor.stringB2bin(chars)
self.log.debug("Received : {0}".format(TypeConvertor.bin2strhex(result)))
return result
def generateXMLConfigFile(self):
# Register the namespace
etree.register_namespace("netzob", PROJECT_NAMESPACE)
etree.register_namespace("netzob-common", COMMON_NAMESPACE)
# Dump the file
root = etree.Element("{" + PROJECT_NAMESPACE + "}project")
root.set("id", str(self.getID()))
root.set("path", str(self.getPath()))
# Warning, changed because of project = Project.createProject(self.netzob.getCurrentWorkspace(), projectName)
if isinstance(self.getCreationDate(), types.TupleType):
root.set("creation_date", TypeConvertor.pythonDatetime2XSDDatetime(self.getCreationDate()[0]))
else:
root.set("creation_date", TypeConvertor.pythonDatetime2XSDDatetime(self.getCreationDate()))
root.set("name", str(self.getName()))
if self.description:
root.set("description", str(self.description))
# Save the configuration in it
self.getConfiguration().save(root, PROJECT_NAMESPACE)
# Save the vocabulary in it
self.getVocabulary().save(root, PROJECT_NAMESPACE, COMMON_NAMESPACE)
# Save the grammar in it
if self.getGrammar() is not None:
self.getGrammar().save(root, PROJECT_NAMESPACE)
# Save the simulator in it
self.getSimulator().save(root, PROJECT_NAMESPACE)
return root
def test_alignmentOfAlmostEquivalentMessages(self):
alignmentSolution = NeedlemanAndWunsch(8)
nbTest = 1000
for i_test in range(0, nbTest):
common_pattern_before = self.generateRandomString(30, 40)
common_pattern_after = self.generateRandomString(30, 40)
# Generate the content of two messages
data1 = TypeConvertor.stringToNetzobRaw(common_pattern_before +
"hercule" +
common_pattern_after)
data2 = TypeConvertor.stringToNetzobRaw(common_pattern_before +
"thomas" +
common_pattern_after)
# Create the messages
message1 = RawMessage(str(uuid.uuid4()), str(time.time()), data1)
message2 = RawMessage(str(uuid.uuid4()), str(time.time()), data2)
(scores, alignment) = alignmentSolution.alignTwoMessages(
False, message1, message2)
(score1, score2, score3) = scores
(scoresBis, alignment2) = alignmentSolution.alignTwoMessages(
True, message1, message2)
(scoreBis1, scoreBis2, scoreBis3) = scoresBis
self.assertGreater(scoreBis1, score1)
self.assertGreater(scoreBis1, 95)
def compareFormat(self, value, indice, negative, vocabulary, memory):
"""compareFormat:
Compute if the provided data is "format-compliant" and return the size of the biggest compliant data.
@type value: bitarray.bitarray
@param value: a bit array a subarray of which we compare to the current variable binray value.
@type indice: integer
@param indice: the starting point of comparison in value.
@type negative: boolean
@param negative: tells if we use the variable or a logical not of it.
@type vocabulary: netzob.Common.Vocabulary.Vocabulary
@param vocabulary: the vocabulary of the current project.
@type memory: netzob.Common.MMSTD.Memory.Memory
@param memory: a memory which can contain a former value of the variable.
@rtype: integer
@return: the size of the biggest compliant data, -1 if it does not comply.
"""
tmp = value[indice:]
size = len(tmp)
if size <= 8:
self.log.debug("Too small, not even 8 bits available (1 number)")
return -1
for i in range(size, 8, -1):
subValue = value[indice:indice + i - 1]
strVal = TypeConvertor.bin2string(
TypeConvertor.strBitarray2Bitarray(subValue))
typeIdentifier = TypeIdentifier()
if typeIdentifier.isAscii(strVal):
if (strVal.isdigit()):
self.log.debug("Its a numeric : (" + str(strVal) + ")")
return i + indice - 1
self.log.debug("the value " + str(
TypeConvertor.bin2string(TypeConvertor.strBitarray2Bitarray(tmp)))
+ " cannot be parsed as a decimalWord")
return -1
def save(message, xmlMessage, namespace_project, namespace_common):
xmlMessage.set("{http://www.w3.org/2001/XMLSchema-instance}type",
FileMessageFactory.XML_SCHEMA_TYPE)
# line number
subLineNumber = etree.SubElement(
xmlMessage, "{" + namespace_common + "}lineNumber")
subLineNumber.text = str(message.getLineNumber())
# filename
subFilename = etree.SubElement(xmlMessage,
"{" + namespace_common + "}filename")
subFilename.text = message.getFilename().decode('utf-8')
# creationDate
subCreationDate = etree.SubElement(
xmlMessage, "{" + namespace_common + "}creationDate")
subCreationDate.text = TypeConvertor.pythonDatetime2XSDDatetime(
message.getCreationDate())
# creationDate
subModificationDate = etree.SubElement(
xmlMessage, "{" + namespace_common + "}modificationDate")
subModificationDate.text = TypeConvertor.pythonDatetime2XSDDatetime(
message.getModificationDate())
# owner
subOwner = etree.SubElement(xmlMessage,
"{" + namespace_common + "}owner")
subOwner.text = message.getOwner()
# size
subSize = etree.SubElement(xmlMessage,
"{" + namespace_common + "}size")
subSize.text = str(message.getSize())
def generateValue(self, negative, dictionary):
self.log.debug("Generate value of hex")
if self.min != -1 and self.max != -1:
# generate a value in int
r = random.randint(self.min, self.max)
self.log.debug("Generating hex of value : " + str(r))
self.binValue = TypeConvertor.int2bin(r, self.size)
self.strValue = TypeConvertor.int2string(r)
def getSearchedDataForDecimal(self, value):
if not value.isdigit():
return []
# Creation of a SearchTask
task = SearchTask(value, value, Format.DECIMAL)
task.registerVariation(TypeConvertor.decimalToNetzobRaw(value), "Decimal representation of '{0}'".format(TypeConvertor.decimalToNetzobRaw(value)))
task.registerVariation(TypeConvertor.decimalToNetzobRaw(value[::-1]), "Inverted decimal representation of '{0}'".format(TypeConvertor.decimalToNetzobRaw(value[::-1])))
return [task]
def generateValue(self, negative, dictionary):
self.log.debug("Generate value of hex")
if self.min != -1 and self.max != -1:
# generate a value in int
r = random.randint(self.min, self.max)
self.log.debug("Generating hex of value : " + str(r))
self.binValue = TypeConvertor.int2bin(r, self.size)
self.strValue = TypeConvertor.int2string(r)
def getMessageDetails(self, messageID):
if not messageID in self._payloadDict:
errorMessage = _("Message ID: {0} not found in importer " +
"message list").format(messageID)
logging.error(errorMessage)
raise NetzobImportException("IPC", errorMessage, ERROR)
payload = self._payloadDict[messageID]
return TypeConvertor.hexdump(TypeConvertor.netzobRawToPythonRaw(payload))
def doGetMessageDetails(self, messageID):
message = self.model.getMessageByID(str(messageID))
properties = [(props.getName(), props.getCurrentValue()) for props in message.getProperties()
if props.getName() != 'Data']
messageDetails = "\n".join(["{0}: {1}".format(*prop)
for prop in properties])
messageDetails += "\n\n" + TypeConvertor.hexdump(TypeConvertor.netzobRawToPythonRaw(message.getStringData()))
return messageDetails
def getMessageDetails(self, messageID):
if not messageID in self._payloadDict:
errorMessage = _("Message ID: {0} not found in importer " +
"message list").format(messageID)
logging.error(errorMessage)
raise NetzobImportException("IPC", errorMessage, ERROR)
payload = self._payloadDict[messageID]
return TypeConvertor.hexdump(
TypeConvertor.netzobRawToPythonRaw(payload))
def learn(self, value, indice, negative, vocabulary, memory):
# First we retrieve the size of the value to memorize
size = self.compare(value, indice, negative, vocabulary, memory)
if size > 0:
# memorize
self.log.debug("Memorize : " + str(value[indice:size]))
memory.memorize(self, (value[indice:size], TypeConvertor.bin2string(TypeConvertor.strBitarray2Bitarray(value[indice:size]))))
return size
else:
self.log.debug("Incompatible for learning")
return -1
def generateValue(self, negative, dictionary):
variable = dictionary.getVariableByID(self.idVar)
(binValue, strValue) = variable.getValue(negative, dictionary)
self.log.debug("GENERATE VALUE of size : " + str(binValue))
nb_letter = TypeConvertor.bin2int(binValue)
self.strVal = ''.join(random.choice(string.ascii_letters) for x in range(nb_letter))
self.binVal = TypeConvertor.string2bin(self.strVal, 'big')
self.log.debug("Generated value = " + self.strVal)
self.log.debug("Generated value = " + str(self.binVal))
def write(self, message):
self.log.debug("Writing to the socket")
self.outputMessages.append(message)
# This work only for values between 0x00 and 0x7f
# self.socket.send(message.tostring())
if self.protocol == "UDP":
self.socket.sendto(TypeConvertor.bin2string(message), (self.getTargetIP(), self.getTargetPort()))
else: # TCP
self.socket.send(TypeConvertor.bin2string(message))
self.log.debug("Write down !")
def doGetMessageDetails(self, messageID):
message = self.model.getMessageByID(str(messageID))
properties = [(props.getName(), props.getCurrentValue())
for props in message.getProperties()
if props.getName() != 'Data']
messageDetails = "\n".join(
["{0}: {1}".format(*prop) for prop in properties])
messageDetails += "\n\n" + TypeConvertor.hexdump(
TypeConvertor.netzobRawToPythonRaw(message.getStringData()))
return messageDetails
def generateValue(self, negative, dictionary):
variable = dictionary.getVariableByID(self.idVar)
(binValue, strValue) = variable.getValue(negative, dictionary)
self.log.debug("GENERATE VALUE of size : " + str(binValue))
nb_letter = TypeConvertor.bin2int(binValue)
self.strVal = ''.join(random.choice(string.ascii_letters) for x in range(nb_letter))
self.binVal = TypeConvertor.string2bin(self.strVal, 'big')
self.log.debug("Generated value = " + self.strVal)
self.log.debug("Generated value = " + str(self.binVal))
def reverse(self, message):
"""reverse:
Encode in B64 the provided message"""
result = message
try:
rawContent = TypeConvertor.netzobRawToPythonRaw(message)
b64Content = base64.b64encode(rawContent)
result = TypeConvertor.pythonRawToNetzobRaw(b64Content)
except TypeError as error:
logging.warning("Impossible to compute the base64 value of message (error={0})".format(str(error)))
result = ""
return result
def apply(self, message):
result = message
rawData = TypeConvertor.netzobRawToPythonRaw(message)
compressedstream = StringIO.StringIO(rawData)
try:
gzipper = gzip.GzipFile(fileobj=compressedstream)
data = gzipper.read()
result = TypeConvertor.pythonRawToNetzobRaw(data)
except Exception as e:
logging.info("Impossible to apply GZip function on provided message (error= {0})".format(str(e)))
result = ""
return result
def getSearchedDataForString(self, value):
# Creation of a SearchTask
task = SearchTask(value, value, Format.STRING)
task.registerVariation(TypeConvertor.stringToNetzobRaw(value),
"String representation of '%s'" % value)
task.registerVariation(
TypeConvertor.stringToNetzobRaw(value[::-1]),
"Inverted string representation of '%s'" % value[::-1])
task.registerVariation(
TypeConvertor.stringToNetzobRaw(value.decode('utf-8')),
"String representation of '%s' encoded in UTF-8" % value)
return [task]
def reverse(self, message):
"""reverse:
This compress the provided message in bz2 format"""
result = message
rawData = TypeConvertor.netzobRawToPythonRaw(message)
try:
rawResult = bz2.compress(rawData)
result = TypeConvertor.pythonRawToNetzobRaw(rawResult)
except Exception as e:
logging.info("Impossible to reverse BZ2 function (compress) on provided message (error= {0})".format(str(e)))
result = ""
return result
def reverse(self, message):
result = message
rawData = TypeConvertor.netzobRawToPythonRaw(message)
compressedstream = StringIO.StringIO()
try:
gzipper = gzip.GzipFile(fileobj=compressedstream, mode='w')
gzipper.write(rawData)
gzipper.close()
result = TypeConvertor.pythonRawToNetzobRaw(compressedstream.getvalue())
except Exception as e:
logging.info("Impossible to apply GZip function on provided message (error= {0})".format(str(e)))
result = ""
return result
def getSearchedDataForDecimal(self, value):
if not value.isdigit():
return []
# Creation of a SearchTask
task = SearchTask(value, value, Format.DECIMAL)
task.registerVariation(
TypeConvertor.decimalToNetzobRaw(value),
"Decimal representation of '{0}'".format(
TypeConvertor.decimalToNetzobRaw(value)))
task.registerVariation(
TypeConvertor.decimalToNetzobRaw(value[::-1]),
"Inverted decimal representation of '{0}'".format(
TypeConvertor.decimalToNetzobRaw(value[::-1])))
return [task]
def abstract(self, message):
"""abstract:
Searches in the vocabulary the symbol which abstract the received message.
@type message: netzob.Common.Models.AbstractMessage
@param message: the message that is being read/compare/learn.
@rtype: netzob.Common.Symbol
@return: the symbol which content matches the message.
"""
self.log.debug("We abstract the received message : " +
TypeConvertor.bin2strhex(message))
# we search in the vocabulary an entry which match the message
for symbol in self.vocabulary.getSymbols():
self.log.debug("Try to abstract message through : {0}.".format(
symbol.getName()))
readingToken = VariableReadingToken(
False, self.vocabulary, self.memory,
TypeConvertor.strBitarray2Bitarray(message), 0)
symbol.getRoot().read(readingToken)
logging.debug(
"ReadingToken: isOk: {0}, index: {1}, len(value): {2}".format(
str(readingToken.isOk()), str(readingToken.getIndex()),
str(len(readingToken.getValue()))))
# The message matches if the read is ok and the whole entry was read.
if readingToken.isOk() and readingToken.getIndex() == len(
readingToken.getValue()):
self.log.debug("The message matches symbol {0}.".format(
symbol.getName()))
# It matches so we learn from it if it's possible
return symbol
else:
self.log.debug("The message doesn't match symbol {0}.".format(
symbol.getName()))
# This is now managed in the variables modules.
#===================================================================
# self.memory.createMemory()
# self.log.debug("We memorize the symbol " + str(symbol.getRoot()))
# readingToken = VariableReadingToken(False, self.vocabulary, self.memory, TypeConvertor.strBitarray2Bitarray(message), 0)
# symbol.getRoot().learn(readingToken)
# self.memory.persistMemory()
# return symbol
# else:
# self.log.debug("Entry " + str(symbol.getID()) + " doesn't match")
# # we first restore a possibly learned value
# self.log.debug("Restore possibly learned value")
# processingToken = AbstractVariableProcessingToken(False, self.vocabulary, self.memory)
# symbol.getRoot().restore(processingToken)
#===================================================================
return UnknownSymbol()
def reverse(self, message):
"""reverse:
Encode in B64 the provided message"""
result = message
try:
rawContent = TypeConvertor.netzobRawToPythonRaw(message)
b64Content = base64.b64encode(rawContent)
result = TypeConvertor.pythonRawToNetzobRaw(b64Content)
except TypeError as error:
logging.warning(
"Impossible to compute the base64 value of message (error={0})"
.format(str(error)))
result = ""
return result
def generateValue(self, negative, dictionary):
# Retrieve the value of the data to hash
var = dictionary.getVariableByID(self.id_var)
(binToHash, strToHash) = var.getValue(negative, dictionary)
toHash = TypeConvertor.bin2string(binToHash)
self.log.debug("Will hash the followings : " + toHash)
md5core = hashlib.md5(self.init)
md5core.update(toHash)
md5Hex = md5core.digest()
self.binVal = TypeConvertor.hex2bin(md5Hex)
self.strVal = TypeConvertor.bin2strhex(self.binVal)
self.log.debug("Generated MD5 = " + self.strVal)
def learn(self, val, indice, isForced, dictionary):
self.log.debug("LEARN")
variable = dictionary.getVariableByID(self.idVar)
(binValue, strValue) = variable.getValue(False, dictionary)
nb_letter = TypeConvertor.bin2int(binValue) * 8
self.log.debug("nb_letter = " + str(nb_letter))
tmp = val[indice:]
self.log.debug("tmp size : " + str(len(tmp)))
if (len(tmp) >= nb_letter):
self.binVal = tmp[:nb_letter]
self.strVal = TypeConvertor.bin2string(self.binVal)
self.log.debug("Value learnt : " + self.strVal)
return indice + nb_letter
return -1
def generateValue(self, negative, dictionary):
# Retrieve the value of the data to hash
var = dictionary.getVariableByID(self.id_var)
(binToHash, strToHash) = var.getValue(negative, dictionary)
toHash = TypeConvertor.bin2string(binToHash)
self.log.debug("Will hash the followings : " + toHash)
md5core = hashlib.md5(self.init)
md5core.update(toHash)
md5Hex = md5core.digest()
self.binVal = TypeConvertor.hex2bin(md5Hex)
self.strVal = TypeConvertor.bin2strhex(self.binVal)
self.log.debug("Generated MD5 = " + self.strVal)
def learn(self, val, indice, isForced, dictionary):
self.log.debug("LEARN")
variable = dictionary.getVariableByID(self.idVar)
(binValue, strValue) = variable.getValue(False, dictionary)
nb_letter = TypeConvertor.bin2int(binValue) * 8
self.log.debug("nb_letter = " + str(nb_letter))
tmp = val[indice:]
self.log.debug("tmp size : " + str(len(tmp)))
if (len(tmp) >= nb_letter):
self.binVal = tmp[:nb_letter]
self.strVal = TypeConvertor.bin2string(self.binVal)
self.log.debug("Value learnt : " + self.strVal)
return indice + nb_letter
return -1
def applySession(self, session):
# retrieve the automata
automata = self.project.getGrammar().getAutomata()
self.log.debug("automata: %s" % automata.getDotCode())
if automata is None:
self.log.warn("Cannot apply a session on the current automata because it doesn't exist")
return None
difference = None
# Configure the role-play environment
# with :
# - a memory
memory = Memory()
# memory = Memory(None)
# - an abstraction layer
abstractionLayer = AbstractionLayer(None, self.project.getVocabulary(), memory, None, None)
currentState = automata.getInitialState()
# We execute the opening transition
if len(currentState.getTransitions()) == 1 and currentState.getTransitions()[0].getType() == OpenChannelTransition.TYPE:
currentState = currentState.getTransitions()[0].getOutputState()
isInput = True
for message in session.getMessages():
self.log.debug("Inject message: %s" % (message.getData()))
# we abstract the message
symbol = abstractionLayer.abstract(TypeConvertor.netzobRawToBitArray(str(message.getData())))
if isInput:
# We simulate the reception of the message
# - verify its a valid input symbol
# - find out the associated transition
currentTransition = None
for transition in currentState.getTransitions():
if transition.getInputSymbol() == symbol:
currentTransition = transition
break
if currentTransition is None:
self.log.warn("Input symbol %s doesn't match any existing transition in current state %s" % (symbol.getName(), currentState.getName()))
self.log.warn("We forget this message.")
else:
self.log.debug("Input symbol %s matchs the transition %s from state %s" % (symbol.getName(), currentTransition.getName(), currentState.getName()))
isInput = False
else:
# We simulate emiting the message
# - we just verify the symbol matches available output message in current transition
found = False
for (outputSymbol, probability, time) in currentTransition.getOutputSymbols():
if symbol.getID() == outputSymbol.getID():
found = True
isInput = True
currentState = currentTransition.getOutputState()
break
if not found:
self.log.info("A difference has been found, symbol %s is not an output symbol of transition %s" % (symbol.getName(), currentTransition.getName()))
return (currentTransition, symbol)
return difference
def getTargetPort(self):
"""Returns in int the value of the bind Port retrieved from memory"""
binPort = self.memory.recall(self.varTargetPort)
if binPort is None:
self.log.warn("Impossible to find the memorized value of the Target Port")
return None
return int(TypeConvertor.binB2string(binPort))
def saveConfigFile(self, overrideTraces=[]):
"""This functions allows to save the current (and only)
instance of the Workspace. You can supply a list of traces
that should be written on-disk through the `overrideTraces`
variable. This allows to override specific traces that where
modified.
:param overrideTraces: a list of trace identifiers that should
be written on-disk, even if they already exists.
"""
workspaceFile = os.path.join(self.path, Workspace.CONFIGURATION_FILENAME)
logging.info("Save the config file of the workspace {0} in {1}".format(self.getName(), workspaceFile))
# Register the namespace
etree.register_namespace('netzob', WORKSPACE_NAMESPACE)
etree.register_namespace('netzob-common', COMMON_NAMESPACE)
# Dump the file
root = etree.Element("{" + WORKSPACE_NAMESPACE + "}workspace")
root.set("creation_date", TypeConvertor.pythonDatetime2XSDDatetime(self.getCreationDate()))
root.set("name", str(self.getName()))
xmlWorkspaceConfig = etree.SubElement(root, "{" + WORKSPACE_NAMESPACE + "}configuration")
relTracePath = os.path.relpath(self.getPathOfTraces(), self.path)
xmlTraces = etree.SubElement(xmlWorkspaceConfig, "{" + WORKSPACE_NAMESPACE + "}traces")
xmlTraces.text = str(self.getPathOfTraces())
xmlLogging = etree.SubElement(xmlWorkspaceConfig, "{" + WORKSPACE_NAMESPACE + "}logging")
xmlLogging.text = str(self.getPathOfLogging())
xmlPrototypes = etree.SubElement(xmlWorkspaceConfig, "{" + WORKSPACE_NAMESPACE + "}prototypes")
xmlPrototypes.text = str(self.getPathOfPrototypes())
xmlPrototypes = etree.SubElement(xmlWorkspaceConfig, "{" + WORKSPACE_NAMESPACE + "}enable_bug_reporting")
xmlPrototypes.text = str(self.enableBugReporting).lower()
xmlWorkspaceProjects = etree.SubElement(root, "{" + WORKSPACE_NAMESPACE + "}projects")
for projectPath in self.getProjectsPath():
xmlProject = etree.SubElement(xmlWorkspaceProjects, "{" + WORKSPACE_NAMESPACE + "}project")
xmlProject.set("path", projectPath)
xmlWorkspaceImported = etree.SubElement(root, "{" + WORKSPACE_NAMESPACE + "}traces")
for importedTrace in self.getImportedTraces():
# overrideTraces variable contains the list of
# ImportedTraces that should be overriden. This is useful
# in case of message removal for example.
forceOverride = (importedTrace.id in overrideTraces)
importedTrace.save(xmlWorkspaceImported, WORKSPACE_NAMESPACE, COMMON_NAMESPACE,
os.path.join(self.path, self.getPathOfTraces()), forceOverride)
xmlWorkspaceFunctions = etree.SubElement(root, "{" + WORKSPACE_NAMESPACE + "}functions")
for function in self.getCustomFunctions():
function.save(xmlWorkspaceFunctions, WORKSPACE_NAMESPACE)
tree = ElementTree(root)
tree.write(workspaceFile, pretty_print=True)
def getTargetIP(self):
"""Returns in string the value of the target IP retrieved from memory"""
binIP = self.memory.recall(self.varTargetIP)
if binIP is None:
self.log.warn("Impossible to find the memorized value of the Target IP")
return None
return TypeConvertor.binB2string(binIP)
def toXML(self, root, namespace):
"""toXML:
Create the xml tree associated to this variable.
"""
xmlVariable = etree.SubElement(root, "{" + namespace + "}variable")
# Header specific to the definition of a variable
xmlVariable.set("id", str(self.getID()))
xmlVariable.set("name", str(self.getName()))
xmlVariable.set("{http://www.w3.org/2001/XMLSchema-instance}type",
"netzob:BinaryVariable")
# Original Value
if self.getOriginalValue() is not None:
xmlBinaryVariableOriginalValue = etree.SubElement(
xmlVariable, "{" + namespace + "}originalValue")
xmlBinaryVariableOriginalValue.text = TypeConvertor.bitarray2StrBitarray(
self.getOriginalValue())
# Minimum bits
xmlBinaryVariableStartValue = etree.SubElement(
xmlVariable, "{" + namespace + "}minBits")
xmlBinaryVariableStartValue.text = str(self.getMinBits())
# Maximum bits
xmlBinaryVariableEndValue = etree.SubElement(
xmlVariable, "{" + namespace + "}maxBits")
xmlBinaryVariableEndValue.text = str(self.getMaxBits())
def loadProject_0_1(projectFile):
# Parse the XML Document as 0.1 version
tree = ElementTree()
tree.parse(projectFile)
xmlProject = tree.getroot()
# Register the namespace
etree.register_namespace("netzob", PROJECT_NAMESPACE)
etree.register_namespace("netzob-common", COMMON_NAMESPACE)
projectID = str(xmlProject.get("id"))
projectName = xmlProject.get("name", "none")
projectCreationDate = TypeConvertor.xsdDatetime2PythonDatetime(xmlProject.get("creation_date"))
projectPath = xmlProject.get("path")
project = Project(projectID, projectName, projectCreationDate, projectPath)
description = xmlProject.get("description")
project.setDescription(description)
# Parse the configuration
if xmlProject.find("{" + PROJECT_NAMESPACE + "}configuration") is not None:
projectConfiguration = ProjectConfiguration.loadProjectConfiguration(
xmlProject.find("{" + PROJECT_NAMESPACE + "}configuration"), PROJECT_NAMESPACE, "0.1"
)
project.setConfiguration(projectConfiguration)
# Parse the vocabulary
if xmlProject.find("{" + PROJECT_NAMESPACE + "}vocabulary") is not None:
projectVocabulary = Vocabulary.loadVocabulary(
xmlProject.find("{" + PROJECT_NAMESPACE + "}vocabulary"),
PROJECT_NAMESPACE,
COMMON_NAMESPACE,
"0.1",
project,
)
project.setVocabulary(projectVocabulary)
# Parse the grammar
if xmlProject.find("{" + PROJECT_NAMESPACE + "}grammar") is not None:
projectGrammar = Grammar.loadGrammar(
xmlProject.find("{" + PROJECT_NAMESPACE + "}grammar"), projectVocabulary, PROJECT_NAMESPACE, "0.1"
)
if projectGrammar is not None:
project.setGrammar(projectGrammar)
# Parse the simulator
if xmlProject.find("{" + PROJECT_NAMESPACE + "}simulator") is not None:
projectSimulator = Simulator.loadSimulator(
xmlProject.find("{" + PROJECT_NAMESPACE + "}simulator"),
PROJECT_NAMESPACE,
"0.1",
project.getGrammar().getAutomata(),
project.getVocabulary(),
)
if projectSimulator is not None:
project.setSimulator(projectSimulator)
return project
def computeCurrentValue(self, strValue):
if strValue != None:
strCurrentValue = strValue
binCurrentValue = TypeConvertor.netzobRawToBitArray(strValue)
self.currentValue = (binCurrentValue, strCurrentValue)
else:
self.currentValue = None
def __init__(self, text):
AbstractValue.__init__(self, "TextValue")
# create logger with the given configuration
self.log = logging.getLogger('netzob.Common.MMSTD.Dictionary.Values.TextValue.py')
self.strtext = text
self.bintext = TypeConvertor.string2bin(self.strtext, 'big')
def loadFromXML(xmlRoot, namespace, version):
"""loadFromXML:
Load a binary variable from an XML definition.
"""
if version == "0.1":
varId = xmlRoot.get("id")
varName = xmlRoot.get("name")
xmlBinaryVariableOriginalValue = xmlRoot.find("{" + namespace +
"}originalValue")
if xmlBinaryVariableOriginalValue is not None:
originalValue = TypeConvertor.strBitarray2Bitarray(
xmlBinaryVariableOriginalValue.text)
else:
originalValue = None
xmlBinaryVariableStartValue = xmlRoot.find("{" + namespace +
"}minBits")
minBits = int(xmlBinaryVariableStartValue.text)
xmlBinaryVariableEndValue = xmlRoot.find("{" + namespace +
"}maxBits")
maxBits = int(xmlBinaryVariableEndValue.text)
return BinaryVariable(varId, varName, originalValue, minBits,
maxBits)
return None
def save(self, root, namespace):
"""Save in the XML tree the actor definition"""
xmlActor = etree.SubElement(root, "{" + namespace + "}actor")
xmlActor.set('id', str(self.getID()))
xmlActor.set('name', str(self.getName()))
xmlActor.set('initiator', TypeConvertor.bool2str(self.isInitiator()))
self.abstractionLayer.save(xmlActor, namespace)
def addPropertiesToElement(xmlMessage, message, namespace):
# direction
subDirection = etree.SubElement(xmlMessage,
"{" + namespace + "}direction")
subDirection.text = message.getDirection()
# major
subMajor = etree.SubElement(xmlMessage, "{" + namespace + "}major")
subMajor.text = message.getMajor()
# minor
subMinor = etree.SubElement(xmlMessage, "{" + namespace + "}minor")
subMinor.text = str(message.getMinor())
# requestMode
subRequestMode = etree.SubElement(xmlMessage,
"{" + namespace + "}requestMode")
subRequestMode.text = message.getRequestMode()
# pid
subPid = etree.SubElement(xmlMessage, "{" + namespace + "}pid")
subPid.text = str(message.getPID())
# status
subStatus = etree.SubElement(xmlMessage, "{" + namespace + "}status")
subStatus.text = str(message.getStatus())
# information
subInformation = etree.SubElement(xmlMessage,
"{" + namespace + "}information")
subInformation.text = str(message.getInformation())
# cancel
subCancel = etree.SubElement(xmlMessage, "{" + namespace + "}cancel")
subCancel.text = TypeConvertor.bool2str(message.getCancel())
# sizeIn
subSizeIn = etree.SubElement(xmlMessage, "{" + namespace + "}sizeIn")
subSizeIn.text = str(message.getSizeIn())
# sizeOut
subSizeOut = etree.SubElement(xmlMessage, "{" + namespace + "}sizeOut")
subSizeOut.text = str(message.getSizeOut())
def receiveSymbolWithTimeout(self, timeout):
# First we read from the input the message
receivedData = self.communicationChannel.read(timeout)
nbMaxAttempts = 5
if receivedData is None:
self.log.warn("The communication channel seems to be closed !")
# return (EmptySymbol(), None)
return (None, None)
if len(receivedData) > 0:
now = datetime.datetime.now()
receptionTime = now.strftime("%H:%M:%S")
self.log.info("Received following message : " +
TypeConvertor.bin2strhex(receivedData))
# Now we abstract the message
symbol = self.abstract(receivedData)
# We store the received messages its time and its abstract representation
self.inputMessages.append([
receptionTime,
TypeConvertor.bin2strhex(receivedData),
symbol.getName()
])
self.registerInputSymbol(receptionTime,
TypeConvertor.bin2strhex(receivedData),
symbol)
return (symbol, receivedData)
else:
if len(self.manipulatedSymbols) > nbMaxAttempts:
if self.manipulatedSymbols[
len(self.manipulatedSymbols) - 1].getType(
) == EmptySymbol.TYPE or self.manipulatedSymbols[
len(self.manipulatedSymbols) -
1].getType() == UnknownSymbol.TYPE:
self.log.warn(
"Consider client has disconnected since no valid symbol received after "
+ str(nbMaxAttempts) + " attempts")
return (None, None)
now = datetime.datetime.now()
receptionTime = now.strftime("%H:%M:%S")
symbol = EmptySymbol()
self.registerInputSymbol(receptionTime, "", symbol)
return (symbol, None)
def __init__(self, text):
AbstractValue.__init__(self, "TextValue")
# create logger with the given configuration
self.log = logging.getLogger(
'netzob.Common.MMSTD.Dictionary.Values.TextValue.py')
self.strtext = text
self.bintext = TypeConvertor.string2bin(self.strtext, 'big')
def getTargetPort(self):
"""Returns in int the value of the bind Port retrieved from memory"""
binPort = self.memory.recall(self.varTargetPort)
if binPort is None:
self.log.warn(
"Impossible to find the memorized value of the Target Port")
return None
return int(TypeConvertor.binB2string(binPort))
