def test_b64_zip_protobuf(self):
""" Test a protobuf that is zipped, then base64-encoded."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'eJzj4tLP1TcwNajKKi8yYPSSTcvMSVUoriwuSc1VSMsvSs0rzkxWSMxLzKksziwGADbBDzw'
)
test.parse_queue()
# Check the number of nodes
self.assertEqual(len(test.nodes.keys()), 5)
self.assertEqual(test.total_nodes, 5)
# Confirm that it was detected as bytes, not ascii
self.assertEqual('bytes', test.nodes[2].data_type)
# Confirm that bytes decoded correctly
self.assertEqual(
b'\n\n/m/050zjwr0\x01J\x1dfile system forensic analysis',
test.nodes[2].value)
# Confirm that text/bytes proto field decoded correctly
self.assertEqual("b'file system forensic analysis'",
test.nodes[5].value)
# Make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def api(api_path):
# Get the referrer from the request, which has the full url + query string.
# Split off the local server and keep just the url we want to parse
unfurl_this = request.referrer.split(':5000/', 1)[1]
unfurl_instance = Unfurl()
unfurl_instance.add_to_queue(
data_type="url", key=None, extra_options={'widthConstraint': {'maximum': 1200}},
value=unfurl_this)
unfurl_instance.parse_queue()
unfurl_json = unfurl_instance.generate_json()
return unfurl_json
def test_bing(self):
""" Test a tyipcal and a unique Bing url """
# unit test for a unique Discord url.
test = Unfurl()
test.add_to_queue(data_type='url', key=None, value='http://bit.ly/36XFLt9')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 15)
self.assertEqual(test.total_nodes, 15)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_bing(self):
""" Test a tyipcal and a unique Bing url """
# unit test for a unique Discord url.
test = Unfurl()
test.add_to_queue(data_type='url', key=None,
value='https://www.bing.com/search?q=digital+forensics&qs=n&form=QBLH&sp=-1&pq=digital+forensic&sc=8-16&sk=&cvid=97BF13B59CF84B98B13C067AAA3DB701')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 19)
self.assertEqual(test.total_nodes, 19)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_ksuid(self):
""" Test of a typical ksuid """
# unit test for a unique ksuid.
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='0o5Fs0EELR0fUjHjbCnEtdUwQe3')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 5)
self.assertEqual(test.total_nodes, 5)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_linkedin_shortlink(self):
""" Test a LinkedIn shortlink; these work a little different than the rest"""
test = Unfurl()
test.add_to_queue(data_type='url', key=None, value='https://lnkd.in/fDJnJ64')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 16)
self.assertEqual(test.total_nodes, 16)
self.assertEqual(test.nodes[4].value, '/fDJnJ64')
self.assertEqual(test.nodes[9].value, 'thisweekin4n6.com')
self.assertEqual(test.nodes[16].key, 4)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_twitter_shortlink(self):
""" Test a Twitter shortlink; these use 301 redirects like most shortlinks"""
test = Unfurl()
test.add_to_queue(data_type='url', key=None, value='https://t.co/g6VWYYwY12')
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 15)
self.assertEqual(test.total_nodes, 15)
self.assertEqual(test.nodes[4].value, '/g6VWYYwY12')
self.assertEqual(test.nodes[12].value, 'github.com')
self.assertEqual(test.nodes[14].label, '1: obsidianforensics')
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_discord(self):
""" Test a tyipcal and a unique Discord url """
# unit test for a unique Discord url.
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'https://discordapp.com/channels/427876741990711298/551531058039095296'
)
test.parse_queue()
# test number of nodes
self.assertEqual(len(test.nodes.keys()), 21)
self.assertEqual(test.total_nodes, 21)
# is processing finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_urlsafe_b64_protobuf(self):
""" Test a protobuf that is encoded with urlsafe b64."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'CkQKCEpvaG4gRG9lENIJGhBqZG9lQGV4YW1wbGUuY29tIOr__________wEoks28w_3B2LS5ATF90LNZ9TkSQDoEABI0Vg'
)
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 9)
self.assertEqual(test.total_nodes, 9)
self.assertEqual('proto.dict', test.nodes[2].data_type)
self.assertEqual("b'*****@*****.**'", test.nodes[5].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_padded_b64_ascii(self):
""" Test a simple ASCII string that is base64-encoded."""
test = Unfurl()
test.add_to_queue(data_type='url',
key=None,
value='dGVzdHl0ZXN0dGVzdA==')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 2)
self.assertEqual(test.total_nodes, 2)
# confirm that it was detected as b64
self.assertEqual('b64', test.nodes[2].data_type)
# confirm that text decoded correctly
self.assertEqual('testytesttest', test.nodes[2].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_twitter(self):
""" Test a typical and a unique Twitter url """
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value='https://twitter.com/_RyanBenson/status/1098230906194546688')
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 13)
self.assertEqual(test.total_nodes, 13)
# confirm that snowflake was detected
self.assertIn('Twitter Snowflakes', test.nodes[9].hover)
# embedded timestamp parses correctly
self.assertEqual('2019-02-20 14:40:26.837', test.nodes[13].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
def test_hex_protobuf(self):
""" Test a protobuf that is encoded as hex."""
test = Unfurl()
test.add_to_queue(
data_type='url',
key=None,
value=
'0a440a084a6f686e20446f6510d2091a106a646f65406578616d706c652e636f6d20ea'
'ffffffffffffffff012892cdbcc3fdc1d8b4b901317dd0b359f53912403a0400123456'
)
test.parse_queue()
# check the number of nodes
self.assertEqual(len(test.nodes.keys()), 9)
self.assertEqual(test.total_nodes, 9)
self.assertEqual('proto.dict', test.nodes[2].data_type)
self.assertEqual("b'*****@*****.**'", test.nodes[5].value)
# make sure the queue finished empty
self.assertTrue(test.queue.empty())
self.assertEqual(len(test.edges), 0)
import configparser
import argparse
import json
from unfurl import Unfurl
config = configparser.ConfigParser()
config.read('unfurl.ini')
parser = argparse.ArgumentParser()
parser.add_argument("url", help="URL to parse")
parser.add_argument("--indent", type=int, default=4, help="indent level")
parser.add_argument("--output", help="file.json output to file.json")
args = parser.parse_args()
unfurl_instance = Unfurl()
unfurl_instance.add_to_queue(
data_type='url',
key=None,
extra_options={'widthConstraint': {
'maximum': 1200
}},
value=args.url)
unfurl_instance.parse_queue()
unfurl_json = unfurl_instance.generate_json()
ret = ""
if (args.indent < 1):
ret = json.dumps(unfurl_json)
else:
ret = json.dumps(unfurl_json, indent=args.indent)