def test_metasploit_payload_uuid(self):
""" Test a Metasploit payload UUID url """
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'https://example.com/4PGoVGYmx8l6F3sVI4Rc8g1wms758YNVXPczHlPobpJENARS'
'uSHb57lFKNndzVSpivRDSi5VH2U-w-pEq_CroLcB--cNbYRroyFuaAgCyMCJDpWbws/'
)
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 11)
self.assertEqual(test.total_nodes, 11)
# confirm that unique id parsed
self.assertIn('Unique ID: e0f1a8546626c7c9', test.nodes[7].label)
# confirm that arch parsed
self.assertIn('Architecture: X64', test.nodes[9].label)
# confirm embedded timestamp parsed
self.assertEqual(1502815973, test.nodes[10].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_mailto(self):
""" Test a typical mailto URL """
# test a Bing search url
test = Unfurl()
test.add_to_queue(
data_type="url",
key=None,
value="mailto:[email protected][email protected]&[email protected]&subject=Big%20News",
)
test.parse_queue()
self.assertEqual(test.nodes[2].label, "to: [email protected]")
self.assertEqual(test.nodes[3].label, "cc: [email protected]")
self.assertEqual(test.nodes[3].key, "cc")
self.assertEqual(test.nodes[3].value, "*****@*****.**")
self.assertEqual(test.nodes[4].key, "bcc")
self.assertEqual(test.nodes[4].value, "*****@*****.**")
self.assertEqual(test.nodes[5].key, "subject")
self.assertEqual(test.nodes[5].value, "subject=Big%20News")
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_google_search_with_rlz(self):
""" Test a Google search URL with a RLZ param """
test = Unfurl()
test.add_to_queue(
data_type='url', key=None,
value='https://www.google.com/search?rlz=1C1GCAB_enUS907US907q=dfir+data')
test.parse_queue()
# Check the number of nodes
self.assertEqual(len(test.nodes.keys()), 18)
self.assertEqual(test.total_nodes, 18)
# Confirm that RLZ AP parsed
self.assertEqual('Application: C1', test.nodes[12].label)
# Language parses
self.assertEqual('Language: English (en)', test.nodes[15].label)
# Search cohort parses
self.assertIn('United States the week of 2020-06-22', test.nodes[17].label)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_b64_zip_protobuf(self):
""" Test a protobuf that is zipped, then base64-encoded."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'eJzj4tLP1TcwNajKKi8yYPSSTcvMSVUoriwuSc1VSMsvSs0rzkxWSMxLzKksziwGADbBDzw'
)
test.parse_queue()
# Check the number of nodes
self.assertEqual(len(test.nodes.keys()), 5)
self.assertEqual(test.total_nodes, 5)
# Confirm that it was detected as bytes, not ascii
self.assertEqual('bytes', test.nodes[2].data_type)
# Confirm that bytes decoded correctly
self.assertEqual(
b'\n\n/m/050zjwr0\x01J\x1dfile system forensic analysis',
test.nodes[2].value)
# Confirm that text/bytes proto field decoded correctly
self.assertEqual("b'file system forensic analysis'",
test.nodes[5].value)
# Make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_no_lookups(self):
""" Test a shortlink with remote lookups disabled"""
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='https://t.co/g6VWYYwY12')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 6)
self.assertEqual(test.total_nodes, 6)
def test_uuid_v1(self):
""" Test parsing of a UUIDv1, with a legit timestamp and valid MAC address """
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='a28cad70-0d73-11ea-aaef-0800200c9a66')
test.parse_queue()
# confirm the Node ID is parsed as a MAC address
self.assertEqual('MAC address: 08:00:20:0C:9A:66', test.nodes[5].label)
# confirm the time is parsed correctly
self.assertIn('2019-11-22 22:01:28.775', test.nodes[6].label)
def test_lang_param(self):
""" Test a url with a language query string param"""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value='https://www.example.com/testing/1?2=3&4=5&lang=en')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 14)
self.assertEqual(test.total_nodes, 14)
# confirm the scheme is parsed
self.assertIn('English', test.nodes[14].label)
def test_sonyflake(self):
""" Test of a Sonyflake """
test = Unfurl()
test.add_to_queue(data_type='url', key=None, value='45eec4a4600041b')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 5)
self.assertEqual(test.total_nodes, 5)
# confirm the machine ID is parsed correctly
self.assertIn('4.27', test.nodes[4].label)
# confirm the time is parsed correctly
self.assertIn('2020-08-12 17:35:29.98', test.nodes[5].label)
def test_uuid_v4(self):
""" Test parsing of a UUIDv4 """
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='43c18e2b-6441-49e2-b907-f4c6262b22de')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 3)
self.assertEqual(test.total_nodes, 3)
# confirm the description is correct
self.assertEqual('Version 4 UUID is randomly generated',
test.nodes[3].label)
def test_md5_hash_no_lookups(self):
""" Test detecting a MD5 in a URL with lookups disabled """
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value='http://hashes.com/test?1=5f4dcc3b5aa765d61d8327deb882cf99')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 9)
self.assertEqual(test.total_nodes, 9)
# confirm that detected as MD5
self.assertIn('Potential MD5 hash', test.nodes[9].label)
def test_file_path_url(self):
""" Test a URL that ends with a file path"""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value='https://dfir.blog/content/images/2019/01/logo.png')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 13)
self.assertEqual(test.total_nodes, 13)
# confirm the scheme is parsed
self.assertIn('File Extension: .png', test.nodes[13].label)
def test_ksuid(self):
""" Test of a typical ksuid """
# unit test for a unique ksuid.
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='0o5Fs0EELR0fUjHjbCnEtdUwQe3')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 5)
self.assertEqual(test.total_nodes, 5)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_discord(self):
""" Test a typical and a unique Discord url """
# unit test for a unique Discord url.
test = Unfurl()
test.add_to_queue(
data_type='url', key=None,
value='https://discordapp.com/channels/427876741990711298/551531058039095296')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 21)
self.assertEqual(test.total_nodes, 21)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_uuid_v1_null_ts_random_node_id(self):
""" Test parsing of a UUIDv1, with a null timestamp and randomly-generated Node ID """
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='00000000000010008f06a139ab18f414')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 6)
self.assertEqual(test.total_nodes, 6)
# confirm the Node ID is parsed as random
self.assertEqual('The Node ID in this UUID is random',
test.nodes[5].label)
# confirm the time is parsed correctly
self.assertIn('1582-10-15 00:00:00', test.nodes[6].label)
def test_linkedin_shortlink(self):
""" Test a LinkedIn shortlink; these work a little different than the rest"""
test = Unfurl(remote_lookups=True)
test.add_to_queue(data_type='url',
key=None,
value='https://lnkd.in/fDJnJ64')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 16)
self.assertEqual(test.total_nodes, 16)
self.assertEqual(test.nodes[4].value, '/fDJnJ64')
self.assertEqual(test.nodes[9].value, 'thisweekin4n6.com')
self.assertEqual(test.nodes[16].key, 4)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_metasploit_checksum_url(self):
""" Test a Metasploit checksum url """
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='https://example.com/WsJH')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 7)
self.assertEqual(test.total_nodes, 7)
# confirm that unique id parsed
self.assertIn('Matches Metasploit URL checksum for Windows',
test.nodes[7].label)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_twitter_shortlink(self):
""" Test a Twitter shortlink; these use 301 redirects like most shortlinks"""
test = Unfurl(remote_lookups=True)
test.add_to_queue(data_type='url',
key=None,
value='https://t.co/g6VWYYwY12')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 15)
self.assertEqual(test.total_nodes, 15)
self.assertEqual(test.nodes[4].value, '/g6VWYYwY12')
self.assertEqual(test.nodes[12].value, 'github.com')
self.assertEqual(test.nodes[14].label, '1: obsidianforensics')
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_bing(self):
""" Test a typical and a unique Bing url """
# test a Bing search url
test = Unfurl()
test.add_to_queue(
data_type='url', key=None,
value='https://www.bing.com/search?q=digital+forensics&qs=n&form=QBLH&sp=-1'
'&pq=digital+forensic&sc=8-16&sk=&cvid=77BF13B59CF84B98B13C067AAA3DB701')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 23)
self.assertEqual(test.total_nodes, 23)
# Test query parsing
self.assertEqual('q: digital forensics', test.nodes[9].label)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_url(self):
""" Test a generic url with a query string"""
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='https://www.example.com/testing/1?2=3&4=5')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 12)
self.assertEqual(test.total_nodes, 12)
# confirm the scheme is parsed
self.assertIn('https', test.nodes[2].label)
# confirm the scheme is parsed
self.assertEqual('/testing/1', test.nodes[4].label)
# confirm the query string params parse
self.assertEqual('4: 5', test.nodes[12].label)
def test_incorrect_padded_b64_ascii(self):
""" Test a simple ASCII string that is base64-encoded, with incorrect padding"""
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='dGVzdHl0ZXN0dGVzdA=')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 2)
self.assertEqual(test.total_nodes, 2)
# confirm that it was decoded from b64 to a string
self.assertEqual('string', test.nodes[2].data_type)
# confirm that text decoded correctly
self.assertEqual('testytesttest', test.nodes[2].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_urlsafe_b64_protobuf(self):
""" Test a protobuf that is encoded with urlsafe b64."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'CkQKCEpvaG4gRG9lENIJGhBqZG9lQGV4YW1wbGUuY29tIOr__________wEoks28w_3B2LS5ATF90LNZ9TkSQDoEABI0Vg'
)
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 9)
self.assertEqual(test.total_nodes, 9)
self.assertEqual('proto.dict', test.nodes[2].data_type)
self.assertEqual("b'*****@*****.**'", test.nodes[5].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_twitter(self):
""" Test a typical and a unique Twitter url """
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value='https://twitter.com/_RyanBenson/status/1098230906194546688')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 13)
self.assertEqual(test.total_nodes, 13)
# confirm that snowflake was detected
self.assertIn('Twitter Snowflakes', test.nodes[9].hover)
# embedded timestamp parses correctly
self.assertEqual('2019-02-20 14:40:26.837', test.nodes[13].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_youtube(self):
""" Test a YouTube.com URL, with t in seconds"""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'https://www.youtube.com/watch?v=LnhSTZgzKuY&list=PLlFGZ98XmfGfV6RAY9fQSeRfyIuhVGSdm&index=2&t=42s'
)
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 14)
self.assertEqual(test.total_nodes, 14)
# Test query parsing
self.assertEqual('Video will start playing at 42 seconds',
test.nodes[14].label)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_md5_hash_with_lookups(self):
""" Test detecting a MD5 in a URL with lookups enabled """
test = Unfurl(remote_lookups=True)
test.add_to_queue(
data_type='url',
key=None,
value='http://hashes.com/test?1=5f4dcc3b5aa765d61d8327deb882cf99')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 10)
self.assertEqual(test.total_nodes, 10)
# confirm that detected as MD5
self.assertIn('Potential MD5 hash', test.nodes[9].label)
# confirm that plaintext lookup succeeded
self.assertIn('Plaintext: password', test.nodes[10].label)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_hex_protobuf(self):
""" Test a protobuf that is encoded as hex."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'0a440a084a6f686e20446f6510d2091a106a646f65406578616d706c652e636f6d20ea'
'ffffffffffffffff012892cdbcc3fdc1d8b4b901317dd0b359f53912403a0400123456'
)
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 9)
self.assertEqual(test.total_nodes, 9)
self.assertEqual('proto.dict', test.nodes[2].data_type)
self.assertEqual("b'*****@*****.**'", test.nodes[5].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_youtu_be(self):
""" Test a youtu.be URL, with t as int"""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'https://youtu.be/LnhSTZgzKuY?list=PLlFGZ98XmfGfV6RAY9fQSeRfyIuhVGSdm&t=301'
)
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 11)
self.assertEqual(test.total_nodes, 11)
# Test query parsing
self.assertEqual('Video will start playing at 05:01 (mm:ss)',
test.nodes[11].label)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)