def test_little_endian_16_bit(self):
mb = MessageTypeBuilder("little_endian_16_length_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 16)
num_messages_by_data_length = {256 * 8: 5, 16: 4, 512: 2}
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"},
little_endian=True)
random.seed(0)
for data_length, num_messages in num_messages_by_data_length.items():
for i in range(num_messages):
pg.generate_message(data="".join(
[random.choice(["0", "1"]) for _ in range(data_length)]))
#self.save_protocol("little_endian_16_length_test", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
length_engine = LengthEngine(ff.bitvectors)
highscored_ranges = length_engine.find(n_gram_length=8)
self.assertEqual(len(highscored_ranges), 3)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertGreater(len(ff.message_types[0]), 0)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.LENGTH)
self.assertEqual(label.start, 24)
self.assertEqual(label.length, 16)
def test_sequence_number_little_endian_16_bit(self):
mb = MessageTypeBuilder("16bit_seq_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
num_messages = 8
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"},
little_endian=True,
sequence_number_increment=64)
for i in range(num_messages):
pg.generate_message(data="0xcafe")
#self.save_protocol("16bit_litte_endian_seq", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertEqual(
ff.message_types[0].num_labels_with_type(
FieldType.Function.SEQUENCE_NUMBER), 1)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.SEQUENCE_NUMBER)
self.assertEqual(label.start, 24)
self.assertEqual(label.length, 16)
def test_one_participant(self):
"""
Test a simple protocol with
preamble, sync and length field (8 bit) and some random data
:return:
"""
mb = MessageTypeBuilder("simple_address_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
num_messages_by_data_length = {8: 5, 16: 10, 32: 15}
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"},
participants=[self.alice])
for data_length, num_messages in num_messages_by_data_length.items():
for i in range(num_messages):
pg.generate_message(data=pg.decimal_to_bits(
22 * i, data_length),
source=self.alice)
#self.save_protocol("address_one_participant", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
address_engine = AddressEngine(ff.hexvectors, ff.participant_indices)
address_dict = address_engine.find_addresses()
self.assertEqual(len(address_dict), 0)
def test_format_finding_rwe(self):
ff, messages = self.get_format_finder_from_protocol_file("rwe.proto.xml", return_messages=True)
ff.run()
sync1, sync2 = "0x9a7d9a7d", "0x67686768"
preprocessor = Preprocessor([np.array(msg.plain_bits, dtype=np.uint8) for msg in messages])
possible_syncs = preprocessor.find_possible_syncs()
self.assertIn(ProtocolGenerator.to_bits(sync1), possible_syncs)
self.assertIn(ProtocolGenerator.to_bits(sync2), possible_syncs)
ack_messages = (3, 5, 7, 9, 11, 13, 15, 17, 20)
ack_message_type = next(mt for mt, messages in ff.existing_message_types.items() if ack_messages[0] in messages)
self.assertTrue(all(ack_msg in ff.existing_message_types[ack_message_type] for ack_msg in ack_messages))
for mt in ff.message_types:
preamble = mt.get_first_label_with_type(FieldType.Function.PREAMBLE)
self.assertEqual(preamble.start, 0)
self.assertEqual(preamble.length, 32)
sync = mt.get_first_label_with_type(FieldType.Function.SYNC)
self.assertEqual(sync.start, 32)
self.assertEqual(sync.length, 32)
length = mt.get_first_label_with_type(FieldType.Function.LENGTH)
self.assertEqual(length.start, 64)
self.assertEqual(length.length, 8)
dst = mt.get_first_label_with_type(FieldType.Function.DST_ADDRESS)
self.assertEqual(dst.length, 24)
if mt == ack_message_type or 1 in ff.existing_message_types[mt]:
self.assertEqual(dst.start, 72)
else:
self.assertEqual(dst.start, 88)
if mt != ack_message_type and 1 not in ff.existing_message_types[mt]:
src = mt.get_first_label_with_type(FieldType.Function.SRC_ADDRESS)
self.assertEqual(src.start, 112)
self.assertEqual(src.length, 24)
elif 1 in ff.existing_message_types[mt]:
# long ack
src = mt.get_first_label_with_type(FieldType.Function.SRC_ADDRESS)
self.assertEqual(src.start, 96)
self.assertEqual(src.length, 24)
crc = mt.get_first_label_with_type(FieldType.Function.CHECKSUM)
self.assertIsNotNone(crc)
def _prepare_protocol_5() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="1337")
bob = Participant("Bob", address_hex="beef")
carl = Participant("Carl", address_hex="cafe")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 16)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.LENGTH, 8)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 16)
pg = ProtocolGenerator([mb.message_type, mb_ack.message_type],
syncs_by_mt={
mb.message_type: "0x9a7d",
mb_ack.message_type: "0x9a7d"
},
preambles_by_mt={
mb.message_type: "10" * 8,
mb_ack.message_type: "10" * 8
},
participants=[alice, bob, carl])
return pg
def _prepare_protocol_3() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="1337")
bob = Participant("Bob", address_hex="beef")
checksum = GenericCRC.from_standard_checksum("CRC8 CCITT")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
mb.add_label(FieldType.Function.DATA, 10 * 8)
mb.add_checksum_label(8, checksum)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 16)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.LENGTH, 8)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 16)
mb_ack.add_checksum_label(8, checksum)
pg = ProtocolGenerator([mb.message_type, mb_ack.message_type],
syncs_by_mt={
mb.message_type: "0x9a7d",
mb_ack.message_type: "0x9a7d"
},
preambles_by_mt={
mb.message_type: "10" * 8,
mb_ack.message_type: "10" * 8
},
participants=[alice, bob])
return pg
def _prepare_protocol_8() -> ProtocolGenerator:
alice = Participant("Alice")
mb = MessageTypeBuilder("data1")
mb.add_label(FieldType.Function.PREAMBLE, 4)
mb.add_label(FieldType.Function.SYNC, 4)
mb.add_label(FieldType.Function.LENGTH, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
mb.add_label(FieldType.Function.DATA, 8 * 542)
mb2 = MessageTypeBuilder("data2")
mb2.add_label(FieldType.Function.PREAMBLE, 4)
mb2.add_label(FieldType.Function.SYNC, 4)
mb2.add_label(FieldType.Function.LENGTH, 16)
mb2.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
mb2.add_label(FieldType.Function.DATA, 8 * 260)
pg = ProtocolGenerator([mb.message_type, mb2.message_type],
syncs_by_mt={
mb.message_type: "0x9",
mb2.message_type: "0x9"
},
preambles_by_mt={
mb.message_type: "10" * 2,
mb2.message_type: "10" * 2
},
sequence_number_increment=32,
participants=[alice],
little_endian=True)
return pg
def test_without_preamble(self):
alice = Participant("Alice", address_hex="24")
broadcast = Participant("Broadcast", address_hex="ff")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 8)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x8e88"},
preambles_by_mt={mb.message_type: "10" * 8},
participants=[alice, broadcast])
for i in range(20):
data_bits = 16 if i % 2 == 0 else 32
source = pg.participants[i % 2]
destination = pg.participants[(i + 1) % 2]
pg.generate_message(data="1010" * (data_bits // 4),
source=source,
destination=destination)
#self.save_protocol("without_preamble", pg)
self.clear_message_types(pg.messages)
ff = FormatFinder(pg.messages)
ff.known_participant_addresses.clear()
ff.run()
self.assertEqual(len(ff.message_types), 1)
mt = ff.message_types[0]
sync = mt.get_first_label_with_type(FieldType.Function.SYNC)
self.assertEqual(sync.start, 0)
self.assertEqual(sync.length, 16)
length = mt.get_first_label_with_type(FieldType.Function.LENGTH)
self.assertEqual(length.start, 16)
self.assertEqual(length.length, 8)
dst = mt.get_first_label_with_type(FieldType.Function.SRC_ADDRESS)
self.assertEqual(dst.start, 24)
self.assertEqual(dst.length, 8)
seq = mt.get_first_label_with_type(FieldType.Function.SEQUENCE_NUMBER)
self.assertEqual(seq.start, 32)
self.assertEqual(seq.length, 8)
def test_easy_protocol(self):
"""
preamble, sync, sequence number, length field (8 bit) and some random data
:return:
"""
mb = MessageTypeBuilder("easy_length_test")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
num_messages_by_data_length = {32: 10, 64: 15, 16: 5, 24: 7}
pg = ProtocolGenerator([mb.message_type],
preambles_by_mt={mb.message_type: "10" * 8},
syncs_by_mt={mb.message_type: "0xcafe"})
for data_length, num_messages in num_messages_by_data_length.items():
for i in range(num_messages):
if i % 4 == 0:
data = "1" * data_length
elif i % 4 == 1:
data = "0" * data_length
elif i % 4 == 2:
data = "10" * (data_length // 2)
else:
data = "01" * (data_length // 2)
pg.generate_message(data=data)
#self.save_protocol("easy_length", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
length_engine = LengthEngine(ff.bitvectors)
highscored_ranges = length_engine.find(n_gram_length=8)
self.assertEqual(len(highscored_ranges), 4)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertGreater(len(ff.message_types[0]), 0)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.LENGTH)
self.assertIsInstance(label, ProtocolLabel)
self.assertEqual(label.start, 32)
self.assertEqual(label.length, 8)
def test_sync_word_finding_with_two_sync_words(self):
preamble = "0xaaaa"
sync1, sync2 = "0x1234", "0xcafe"
pg = self.build_protocol_generator(preamble_syncs=[(preamble, sync1),
(preamble, sync2)],
num_messages=(15, 10),
data=(lambda i: 12 * i,
lambda i: 16 * i))
preprocessor = Preprocessor([
np.array(msg.plain_bits, dtype=np.uint8)
for msg in pg.protocol.messages
])
possible_syncs = preprocessor.find_possible_syncs()
#self.save_protocol("two_syncs", pg)
self.assertGreaterEqual(len(possible_syncs), 2)
self.assertIn(ProtocolGenerator.to_bits(sync1), possible_syncs)
self.assertIn(ProtocolGenerator.to_bits(sync2), possible_syncs)
def test_no_sequence_number(self):
"""
Ensure no sequence number is labeled, when it cannot be found
:return:
"""
alice = Participant("Alice", address_hex="dead")
bob = Participant("Bob", address_hex="beef")
mb = MessageTypeBuilder("protocol_with_one_message_type")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
num_messages = 3
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x1337"},
participants=[alice, bob])
for i in range(num_messages):
if i % 2 == 0:
source, destination = alice, bob
else:
source, destination = bob, alice
pg.generate_message(data="",
source=source,
destination=destination)
#self.save_protocol("protocol_1", pg)
# Delete message type information -> no prior knowledge
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.known_participant_addresses.clear()
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertEqual(
ff.message_types[0].num_labels_with_type(
FieldType.Function.SEQUENCE_NUMBER), 0)
def __prepare_simple_example_protocol(self):
random.seed(0)
alice = Participant("Alice", "A", address_hex="1234")
bob = Participant("Bob", "B", address_hex="cafe")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.TYPE, 8)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x6768"},
participants=[alice, bob])
for i in range(10):
pg.generate_message(data="".join(
[random.choice(["0", "1"]) for _ in range(16)]),
source=alice,
destination=bob)
pg.generate_message(data="".join(
[random.choice(["0", "1"]) for _ in range(8)]),
source=bob,
destination=alice)
return pg.protocol
def test_find_generated_crc16(self):
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.DATA, 32)
mb.add_checksum_label(16, GenericCRC.from_standard_checksum("CRC16 CCITT"))
mb2 = MessageTypeBuilder("data2")
mb2.add_label(FieldType.Function.PREAMBLE, 8)
mb2.add_label(FieldType.Function.SYNC, 16)
mb2.add_label(FieldType.Function.LENGTH, 8)
mb2.add_label(FieldType.Function.DATA, 16)
mb2.add_checksum_label(16, GenericCRC.from_standard_checksum("CRC16 CCITT"))
pg = ProtocolGenerator([mb.message_type, mb2.message_type], syncs_by_mt={mb.message_type: "0x1234", mb2.message_type: "0x1234"})
num_messages = 5
for i in range(num_messages):
pg.generate_message(data="{0:032b}".format(i), message_type=mb.message_type)
pg.generate_message(data="{0:016b}".format(i), message_type=mb2.message_type)
#self.save_protocol("crc16_test", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.run()
self.assertEqual(len(ff.message_types), 2)
for mt in ff.message_types:
checksum_label = mt.get_first_label_with_type(FieldType.Function.CHECKSUM)
self.assertEqual(checksum_label.length, 16)
self.assertEqual(checksum_label.checksum.caption, "CRC16 CCITT")
def test_simple_protocol(self):
"""
Test a simple protocol with
preamble, sync and increasing sequence number (8 bit) and some constant data
:return:
"""
mb = MessageTypeBuilder("simple_seq_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
num_messages = 20
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"})
for i in range(num_messages):
pg.generate_message(data="0xcafe")
#self.save_protocol("simple_sequence_number", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
seq_engine = SequenceNumberEngine(ff.bitvectors, n_gram_length=8)
highscored_ranges = seq_engine.find()
self.assertEqual(len(highscored_ranges), 1)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertGreater(len(ff.message_types[0]), 0)
self.assertEqual(
ff.message_types[0].num_labels_with_type(
FieldType.Function.SEQUENCE_NUMBER), 1)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.SEQUENCE_NUMBER)
self.assertEqual(label.start, 24)
self.assertEqual(label.length, 8)
def test_sync_word_finding_common_prefix(self):
"""
Messages are very similar (odd and even ones are the same)
However, they do not have two different sync words!
The algorithm needs to check for a common prefix of the two found sync words
:return:
"""
sync = "0x1337"
num_messages = 10
alice = Participant("Alice", address_hex="dead01")
bob = Participant("Bob", address_hex="beef24")
mb = MessageTypeBuilder("protocol_with_one_message_type")
mb.add_label(FieldType.Function.PREAMBLE, 72)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 24)
mb.add_label(FieldType.Function.DST_ADDRESS, 24)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x1337"},
preambles_by_mt={mb.message_type: "10" * 36},
participants=[alice, bob])
random.seed(0)
for i in range(num_messages):
if i % 2 == 0:
source, destination = alice, bob
data_length = 8
else:
source, destination = bob, alice
data_length = 16
pg.generate_message(data=pg.decimal_to_bits(
random.randint(0, 2**(data_length - 1)), data_length),
source=source,
destination=destination)
preprocessor = Preprocessor([
np.array(msg.plain_bits, dtype=np.uint8)
for msg in pg.protocol.messages
])
possible_syncs = preprocessor.find_possible_syncs()
#self.save_protocol("sync_by_common_prefix", pg)
self.assertEqual(len(possible_syncs), 1)
# +0000 is okay, because this will get fixed by correction in FormatFinder
self.assertIn(possible_syncs[0], [
ProtocolGenerator.to_bits(sync),
ProtocolGenerator.to_bits(sync) + "0000"
])
def test_16bit_seq_nr_with_zeros_in_first_part(self):
mb = MessageTypeBuilder("16bit_seq_first_byte_zero_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
num_messages = 10
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"},
sequence_number_increment=1)
for i in range(num_messages):
pg.generate_message(data="0xcafe" + "abc" * i)
#self.save_protocol("16bit_seq_first_byte_zero_test", pg)
bitvectors = FormatFinder.get_bitvectors_from_messages(
pg.protocol.messages, sync_ends=[24] * num_messages)
seq_engine = SequenceNumberEngine(bitvectors, n_gram_length=8)
highscored_ranges = seq_engine.find()
self.assertEqual(len(highscored_ranges), 1)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertGreater(len(ff.message_types[0]), 0)
self.assertEqual(
ff.message_types[0].num_labels_with_type(
FieldType.Function.SEQUENCE_NUMBER), 1)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.SEQUENCE_NUMBER)
# Not consider constants as part of SEQ Nr!
self.assertEqual(label.start, 40)
self.assertEqual(label.length, 8)
def test_simple_protocol(self):
"""
Test a simple protocol with
preamble, sync and length field (8 bit) and some random data
:return:
"""
mb = MessageTypeBuilder("simple_length_test")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
num_messages_by_data_length = {8: 5, 16: 10, 32: 15}
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"})
random.seed(0)
for data_length, num_messages in num_messages_by_data_length.items():
for i in range(num_messages):
pg.generate_message(data="".join(
[random.choice(["0", "1"]) for _ in range(data_length)]))
#self.save_protocol("simple_length", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
length_engine = LengthEngine(ff.bitvectors)
highscored_ranges = length_engine.find(n_gram_length=8)
self.assertEqual(len(highscored_ranges), 3)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 1)
self.assertGreater(len(ff.message_types[0]), 0)
label = ff.message_types[0].get_first_label_with_type(
FieldType.Function.LENGTH)
self.assertEqual(label.start, 24)
self.assertEqual(label.length, 8)
def _prepare_protocol_4() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="1337")
bob = Participant("Bob", address_hex="beef")
checksum = GenericCRC.from_standard_checksum("CRC16 CCITT")
mb = MessageTypeBuilder("data1")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.DATA, 8 * 8)
mb.add_checksum_label(16, checksum)
mb2 = MessageTypeBuilder("data2")
mb2.add_label(FieldType.Function.PREAMBLE, 16)
mb2.add_label(FieldType.Function.SYNC, 16)
mb2.add_label(FieldType.Function.LENGTH, 8)
mb2.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb2.add_label(FieldType.Function.DST_ADDRESS, 16)
mb2.add_label(FieldType.Function.DATA, 64 * 8)
mb2.add_checksum_label(16, checksum)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 16)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.LENGTH, 8)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 16)
mb_ack.add_checksum_label(16, checksum)
mt1, mt2, mt3 = mb.message_type, mb2.message_type, mb_ack.message_type
preamble = "10001000" * 2
pg = ProtocolGenerator([mt1, mt2, mt3],
syncs_by_mt={
mt1: "0x9a7d",
mt2: "0x9a7d",
mt3: "0x9a7d"
},
preambles_by_mt={
mt1: preamble,
mt2: preamble,
mt3: preamble
},
participants=[alice, bob])
return pg
def build_protocol_generator(preamble_syncs: list, num_messages: tuple,
data: tuple) -> ProtocolGenerator:
message_types = []
preambles_by_mt = dict()
syncs_by_mt = dict()
assert len(preamble_syncs) == len(num_messages) == len(data)
for i, (preamble, sync_word) in enumerate(preamble_syncs):
assert isinstance(preamble, str)
assert isinstance(sync_word, str)
preamble, sync_word = map(ProtocolGenerator.to_bits,
(preamble, sync_word))
mb = MessageTypeBuilder("message type #{0}".format(i))
mb.add_label(FieldType.Function.PREAMBLE, len(preamble))
mb.add_label(FieldType.Function.SYNC, len(sync_word))
message_types.append(mb.message_type)
preambles_by_mt[mb.message_type] = preamble
syncs_by_mt[mb.message_type] = sync_word
pg = ProtocolGenerator(message_types,
preambles_by_mt=preambles_by_mt,
syncs_by_mt=syncs_by_mt)
for i, msg_type in enumerate(message_types):
for j in range(num_messages[i]):
if callable(data[i]):
msg_data = pg.decimal_to_bits(data[i](j), num_bits=8)
else:
msg_data = data[i]
pg.generate_message(message_type=msg_type, data=msg_data)
return pg
def _prepare_protocol_6() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="24")
broadcast = Participant("Bob", address_hex="ff")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 8)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x8e88"},
preambles_by_mt={mb.message_type: "10" * 8},
participants=[alice, broadcast])
return pg
def _prepare_protocol_1() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="dead")
bob = Participant("Bob", address_hex="beef")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x1337"},
participants=[alice, bob])
return pg
def _prepare_protocol_7() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="313370")
bob = Participant("Bob", address_hex="031337")
charly = Participant("Charly", address_hex="110000")
daniel = Participant("Daniel", address_hex="001100")
# broadcast = Participant("Broadcast", address_hex="ff") #TODO: Sometimes messages to broadcast
checksum = GenericCRC.from_standard_checksum("CRC16 CC1101")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.DST_ADDRESS, 24)
mb.add_label(FieldType.Function.SRC_ADDRESS, 24)
mb.add_label(FieldType.Function.DATA, 8 * 8)
mb.add_checksum_label(16, checksum)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 8)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 24)
mb_ack.add_checksum_label(16, checksum)
mb_kex = MessageTypeBuilder("kex")
mb_kex.add_label(FieldType.Function.PREAMBLE, 24)
mb_kex.add_label(FieldType.Function.SYNC, 16)
mb_kex.add_label(FieldType.Function.DST_ADDRESS, 24)
mb_kex.add_label(FieldType.Function.SRC_ADDRESS, 24)
mb_kex.add_label(FieldType.Function.DATA, 64 * 8)
mb_kex.add_checksum_label(16, checksum)
pg = ProtocolGenerator(
[mb.message_type, mb_ack.message_type, mb_kex.message_type],
syncs_by_mt={
mb.message_type: "0x0420",
mb_ack.message_type: "0x2222",
mb_kex.message_type: "0x6767"
},
preambles_by_mt={
mb.message_type: "10" * 8,
mb_ack.message_type: "10" * 4,
mb_kex.message_type: "10" * 12
},
participants=[alice, bob, charly, daniel])
return pg
def test_medium_protocol(self):
"""
Protocol with two message types. Length field only present in one of them
:return:
"""
mb1 = MessageTypeBuilder("data")
mb1.add_label(FieldType.Function.PREAMBLE, 8)
mb1.add_label(FieldType.Function.SYNC, 8)
mb1.add_label(FieldType.Function.LENGTH, 8)
mb1.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
mb2 = MessageTypeBuilder("ack")
mb2.add_label(FieldType.Function.PREAMBLE, 8)
mb2.add_label(FieldType.Function.SYNC, 8)
pg = ProtocolGenerator([mb1.message_type, mb2.message_type],
syncs_by_mt={
mb1.message_type: "11110011",
mb2.message_type: "11110011"
})
num_messages_by_data_length = {8: 5, 16: 10, 32: 5}
for data_length, num_messages in num_messages_by_data_length.items():
for i in range(num_messages):
pg.generate_message(data=pg.decimal_to_bits(
10 * i, data_length),
message_type=mb1.message_type)
pg.generate_message(message_type=mb2.message_type, data="0xaf")
#self.save_protocol("medium_length", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.perform_iteration()
self.assertEqual(len(ff.message_types), 2)
length_mt = next(mt for mt in ff.message_types
if mt.get_first_label_with_type(
FieldType.Function.LENGTH) is not None)
length_label = length_mt.get_first_label_with_type(
FieldType.Function.LENGTH)
for i, sync_end in enumerate(ff.sync_ends):
self.assertEqual(sync_end, 16, msg=str(i))
self.assertEqual(16, length_label.start)
self.assertEqual(8, length_label.length)
def __prepare_example_protocol(self) -> ProtocolAnalyzer:
alice = Participant("Alice", "A", address_hex="1234")
bob = Participant("Bob", "B", address_hex="cafe")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.TYPE, 8)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 8)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.LENGTH, 8)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 16)
num_messages = 50
pg = ProtocolGenerator([mb.message_type, mb_ack.message_type],
syncs_by_mt={
mb.message_type: "0x6768",
mb_ack.message_type: "0x6768"
},
participants=[alice, bob])
random.seed(0)
for i in range(num_messages):
if i % 2 == 0:
source, destination = alice, bob
data_length = 8
else:
source, destination = bob, alice
data_length = 16
pg.generate_message(data=pg.decimal_to_bits(
random.randint(0, 2**(data_length - 1)), data_length),
source=source,
destination=destination)
pg.generate_message(data="",
message_type=mb_ack.message_type,
destination=source,
source=destination)
#self.save_protocol("labeled_protocol", pg)
return pg.protocol
def _prepare_protocol_2() -> ProtocolGenerator:
alice = Participant("Alice", address_hex="dead01")
bob = Participant("Bob", address_hex="beef24")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 72)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 24)
mb.add_label(FieldType.Function.DST_ADDRESS, 24)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x1337"},
preambles_by_mt={mb.message_type: "10" * 36},
sequence_number_increment=32,
participants=[alice, bob])
return pg
def generate_homematic(cls, num_messages: int, save_protocol=True):
mb_m_frame = MessageTypeBuilder("mframe")
mb_c_frame = MessageTypeBuilder("cframe")
mb_r_frame = MessageTypeBuilder("rframe")
mb_a_frame = MessageTypeBuilder("aframe")
participants = [
Participant("CCU", address_hex="3927cc"),
Participant("Switch", address_hex="3101cc")
]
checksum = GenericCRC.from_standard_checksum("CRC16 CC1101")
for mb_builder in [mb_m_frame, mb_c_frame, mb_r_frame, mb_a_frame]:
mb_builder.add_label(FieldType.Function.PREAMBLE, 32)
mb_builder.add_label(FieldType.Function.SYNC, 32)
mb_builder.add_label(FieldType.Function.LENGTH, 8)
mb_builder.add_label(FieldType.Function.SEQUENCE_NUMBER, 8)
mb_builder.add_label(FieldType.Function.TYPE, 16)
mb_builder.add_label(FieldType.Function.SRC_ADDRESS, 24)
mb_builder.add_label(FieldType.Function.DST_ADDRESS, 24)
if mb_builder.name == "mframe":
mb_builder.add_label(FieldType.Function.DATA,
16,
name="command")
elif mb_builder.name == "cframe":
mb_builder.add_label(FieldType.Function.DATA,
16 * 4,
name="command+challenge+magic")
elif mb_builder.name == "rframe":
mb_builder.add_label(FieldType.Function.DATA,
32 * 4,
name="cipher")
elif mb_builder.name == "aframe":
mb_builder.add_label(FieldType.Function.DATA,
10 * 4,
name="command + auth")
mb_builder.add_checksum_label(16, checksum)
message_types = [
mb_m_frame.message_type, mb_c_frame.message_type,
mb_r_frame.message_type, mb_a_frame.message_type
]
preamble = "0xaaaaaaaa"
sync = "0xe9cae9ca"
initial_sequence_number = 36
pg = ProtocolGenerator(
message_types,
participants,
preambles_by_mt={mt: preamble
for mt in message_types},
syncs_by_mt={mt: sync
for mt in message_types},
sequence_numbers={
mt: initial_sequence_number
for mt in message_types
},
message_type_codes={
mb_m_frame.message_type: 42560,
mb_c_frame.message_type: 40962,
mb_r_frame.message_type: 40963,
mb_a_frame.message_type: 32770
})
for i in range(num_messages):
mt = pg.message_types[i % 4]
data_length = mt.get_first_label_with_type(
FieldType.Function.DATA).length
data = "".join(
random.choice(["0", "1"]) for _ in range(data_length))
pg.generate_message(mt,
data,
source=pg.participants[i % 2],
destination=pg.participants[(i + 1) % 2])
if save_protocol:
cls.save_protocol("homematic", pg)
cls.clear_message_types(pg.messages)
return pg.protocol
def test_two_participants(self):
mb = MessageTypeBuilder("address_two_participants")
mb.add_label(FieldType.Function.PREAMBLE, 8)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
num_messages = 50
pg = ProtocolGenerator([mb.message_type],
syncs_by_mt={mb.message_type: "0x9a9d"},
participants=[self.alice, self.bob])
for i in range(num_messages):
if i % 2 == 0:
source, destination = self.alice, self.bob
data_length = 8
else:
source, destination = self.bob, self.alice
data_length = 16
pg.generate_message(data=pg.decimal_to_bits(4 * i, data_length),
source=source,
destination=destination)
#self.save_protocol("address_two_participants", pg)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
address_engine = AddressEngine(ff.hexvectors, ff.participant_indices)
address_dict = address_engine.find_addresses()
self.assertEqual(len(address_dict), 2)
addresses_1 = list(
map(util.convert_numbers_to_hex_string, address_dict[0]))
addresses_2 = list(
map(util.convert_numbers_to_hex_string, address_dict[1]))
self.assertIn(self.alice.address_hex, addresses_1)
self.assertIn(self.alice.address_hex, addresses_2)
self.assertIn(self.bob.address_hex, addresses_1)
self.assertIn(self.bob.address_hex, addresses_2)
ff.known_participant_addresses.clear()
self.assertEqual(len(ff.known_participant_addresses), 0)
ff.perform_iteration()
self.assertEqual(len(ff.known_participant_addresses), 2)
self.assertIn(bytes([int(h, 16) for h in self.alice.address_hex]),
map(bytes, ff.known_participant_addresses.values()))
self.assertIn(bytes([int(h, 16) for h in self.bob.address_hex]),
map(bytes, ff.known_participant_addresses.values()))
self.assertEqual(len(ff.message_types), 1)
mt = ff.message_types[0]
dst_addr = mt.get_first_label_with_type(FieldType.Function.DST_ADDRESS)
self.assertIsNotNone(dst_addr)
self.assertEqual(dst_addr.start, 32)
self.assertEqual(dst_addr.length, 16)
src_addr = mt.get_first_label_with_type(FieldType.Function.SRC_ADDRESS)
self.assertIsNotNone(src_addr)
self.assertEqual(src_addr.start, 48)
self.assertEqual(src_addr.length, 16)
def test_three_participants_with_ack(self):
alice = Participant("Alice", address_hex="1337")
bob = Participant("Bob", address_hex="4711")
carl = Participant("Carl", address_hex="cafe")
mb = MessageTypeBuilder("data")
mb.add_label(FieldType.Function.PREAMBLE, 16)
mb.add_label(FieldType.Function.SYNC, 16)
mb.add_label(FieldType.Function.LENGTH, 8)
mb.add_label(FieldType.Function.SRC_ADDRESS, 16)
mb.add_label(FieldType.Function.DST_ADDRESS, 16)
mb.add_label(FieldType.Function.SEQUENCE_NUMBER, 16)
mb_ack = MessageTypeBuilder("ack")
mb_ack.add_label(FieldType.Function.PREAMBLE, 16)
mb_ack.add_label(FieldType.Function.SYNC, 16)
mb_ack.add_label(FieldType.Function.LENGTH, 8)
mb_ack.add_label(FieldType.Function.DST_ADDRESS, 16)
pg = ProtocolGenerator([mb.message_type, mb_ack.message_type],
syncs_by_mt={
mb.message_type: "0x9a7d",
mb_ack.message_type: "0x9a7d"
},
preambles_by_mt={
mb.message_type: "10" * 8,
mb_ack.message_type: "10" * 8
},
participants=[alice, bob, carl])
i = -1
while len(pg.protocol.messages) < 20:
i += 1
source = pg.participants[i % len(pg.participants)]
destination = pg.participants[(i + 1) % len(pg.participants)]
if i % 2 == 0:
data_bytes = 8
else:
data_bytes = 16
data = "".join(
random.choice(["0", "1"]) for _ in range(data_bytes * 8))
pg.generate_message(data=data,
source=source,
destination=destination)
if "ack" in (msg_type.name
for msg_type in pg.protocol.message_types):
pg.generate_message(message_type=1,
data="",
source=destination,
destination=source)
self.clear_message_types(pg.protocol.messages)
ff = FormatFinder(pg.protocol.messages)
ff.known_participant_addresses.clear()
self.assertEqual(len(ff.known_participant_addresses), 0)
ff.run()
# Since there are ACKS in this protocol, the engine must be able to assign the correct participant addresses
# IN CORRECT ORDER!
self.assertEqual(
util.convert_numbers_to_hex_string(
ff.known_participant_addresses[0]), "1337")
self.assertEqual(
util.convert_numbers_to_hex_string(
ff.known_participant_addresses[1]), "4711")
self.assertEqual(
util.convert_numbers_to_hex_string(
ff.known_participant_addresses[2]), "cafe")
