def testUrlDanger4(self):
jobId = com.Console.submitJob(
"thug feed.url=http://js.honeysploit.hsn/filtertests.html")
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 120, 2, True)
self.assertTrue(finished, "Job failed or took too long.")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertEqual(ret[1].isSet("thug_active"), True)
self.assertEquals(ret[1].thug_active, True)
self.assertEqual(ret[1].isSet("thug_behaviors"), True)
behHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].thug_behaviors.key),
method="GET")
bList = ow.fromBehaviorList(behHandle)
self.assertTrue(len(bList) > 0, "Behavior shouldn't be zero.")
self.assertEqual(ret[1].isSet("js_context_list"), True)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].js_context_list.key),
method="GET")
cList = ow.fromJSContextList(contHandle)
self.assertTrue(len(cList) > 0, "Contexts should have been reported")
self.assertEqual(ret[1].isSet("thug_time_start"), True)
self.assertEqual(ret[1].isSet("thug_time_stop"), True)
self.assertTrue(ret[1].thug_time_stop - ret[1].thug_time_start > 0,
"Thug should have run was some time")
def testCompletedWithMatches(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-simple.json yara1.rules_filename=/tmp/tests/resources/yara/rules1.yar"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
details = com.JobDetails(jobId)
self.assertEqual(details["job_status"],
"COMPLETED",
name1="job_status")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertTrue(ret[1].isSet("yara_time_start"))
self.assertTrue(ret[1].isSet("yara_time_stop"))
self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start >= 0)
self.assertTrue(ret[1].isSet("yara_matches_found"))
self.assertTrue(ret[1].isSet("yara_matches_list"))
self.assertTrue(ret[1].yara_matches_found)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].yara_matches_list.key),
method="GET")
matches = ow.fromYaraMatchesList(contHandle)
self.assertTrue(
len(matches) == 2,
"There should be two yara matches for test-file.txt")
self.assertRuleMatches(matches, 'TextExample', 'default')
self.assertRuleMatches(matches, 'AnotherTextExample', 'default')
def testPackersUpx(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-upx.json yara1.rules_filename=/tmp/tests/resources/yara/packers.yar"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
details = com.JobDetails(jobId)
self.assertEqual(details["job_status"],
"COMPLETED",
name1="job_status")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertTrue(ret[1].isSet("yara_time_start"))
self.assertTrue(ret[1].isSet("yara_time_stop"))
self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start > 0,
"Expected yara running for more than 0 milliseconds.")
self.assertTrue(ret[1].isSet("yara_matches_found"))
self.assertTrue(ret[1].isSet("yara_matches_list"))
self.assertTrue(ret[1].yara_matches_found)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].yara_matches_list.key),
method="GET")
matches = ow.fromYaraMatchesList(contHandle)
self.assertTrue(
len(matches) == 1, "There should be one yara match for calc.upx")
self.assertRuleMatches(matches, 'UPX', 'default')
def testSwfCveMalicious(self):
jobId = com.Console.submitJob(
"swf-cve-1 feed.uri=/tmp/tests/resources/json/swf-cve-malicious.json"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertSet(ret[1], "swf_cve_time_begin")
self.assertSet(ret[1], "swf_cve_time_end")
self.assertSet(ret[1], "swf_cve_detected")
self.assertSet(ret[1], "swf_cve_list")
self.assertEqual(
ret[1].swf_cve_detected, 1,
"Expected 1 CVE detected, got %s." % ret[1].swf_cve_detected)
handle = com.getUrlHandle('http://localhost:8080/data/%s/%s' %
(jobId, ret[1].swf_cve_list.getKey()),
method='GET')
html = ''.join(handle.readlines())
self.assertIn(
'CVE_2007_0071', html,
"Expected CVE_2007_0071 detected, but in data-store there was only:\n%s"
% (html))
def testUrlDanger1(self):
jobId = com.Console.submitJob("thug feed.url=http://js.honeysploit.hsn/dangerous.html")
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 16, 2, True)
self.assertTrue(finished, "Job failed or took too long.")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertEqual(ret[1].isSet("thug_active"), True)
self.assertEquals(ret[1].thug_active, True)
self.assertEqual(ret[1].isSet("thug_behaviors"), True)
behHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].thug_behaviors.key) , method="GET")
bList = ow.fromBehaviorList(behHandle)
self.assertTrue(len(bList) > 0, "Behavior shouldn't be zero.")
self.assertEqual(ret[1].isSet("js_context_list"), True)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].js_context_list.key) , method="GET")
cList = ow.fromJSContextList(contHandle)
self.assertTrue(len(cList) > 0, "Contexts should have been reported")
self.assertEqual(ret[1].isSet("thug_time_start"), True)
self.assertEqual(ret[1].isSet("thug_time_stop"), True)
self.assertTrue(ret[1].thug_time_stop - ret[1].thug_time_start > 0, "Thug should have run was some time")
def testSwfCveMalicious(self):
jobId = com.Console.submitJob("swf-cve-1 feed.uri=/tmp/tests/resources/json/swf-cve-malicious.json")
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertSet(ret[1], "swf_cve_time_begin")
self.assertSet(ret[1], "swf_cve_time_end")
self.assertSet(ret[1], "swf_cve_detected")
self.assertSet(ret[1], "swf_cve_list")
self.assertEqual(ret[1].swf_cve_detected, 1, "Expected 1 CVE detected, got %s." % ret[1].swf_cve_detected)
handle = com.getUrlHandle('http://localhost:8080/data/%s/%s' % (jobId, ret[1].swf_cve_list.getKey()), method='GET')
html = ''.join(handle.readlines())
self.assertIn('CVE_2007_0071', html, "Expected CVE_2007_0071 detected, but in data-store there was only:\n%s" % (html))
def testSimpleJob(self):
logging.info("Test if CouchDB responds to HTTP requests...")
ret = com.Console.call("curl 127.0.0.1:5984 --stderr /dev/null")
logging.info(ret[1])
self.assertIn('{"couchdb":"Welcome","version":"1.2', ret[1],
"Couldn't connect to CouchDB.")
logging.info("Test if submitted job is accepted...")
jobId = com.Console.submitJob(
"simple --param feeder1.uri=/tmp/tests/file.txt")
self.assertIsNotNone(jobId, "Job ID not found.")
logging.info('The job ID is %s' % jobId)
logging.info("Test if job finishes successfully...")
completed = com.Console.waitForCompletion(jobId=jobId,
maxTime=30,
period=1,
verbose=True)
ret = com.Console.call("hc j d %s" % jobId)
logging.info(ret[1])
self.assertTrue(completed, "Job failed to finish successfully.")
# TODO: check time elapsed if completed in >30s
logging.info("Test if job finished in less than 30 seconds...")
jd = com.JobDetails(jobId)
timeElapsed = int(jd.get("job_processing_time_sec"))
self.assertGreaterEqual(
50, timeElapsed,
"The job %s took %s seconds, which is more than accepted 30s" %
(jobId, timeElapsed))
logging.info("Test if there are 4 objects found by unicorn...")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertEqual(
len(ret), 4,
"Expected 2 objects from unicorn, received %d" % len(ret))
logging.info("Test if there are 2 objects found in CouchDB...")
couch = couchdb.Server()['hsn']
self.assertEqual(
len(couch), 2,
"Expected 2 objects from CouchDB, received %d" % len(couch))
logging.info(
"Test if JavaScript contexts are consistent between webclient and js-sta..."
)
for objId in couch:
self.assertIn(
objId, [str(jobId) + ":2:js-sta",
str(jobId) + ":2:webclient"],
"%s is not allowed object id in CouchDB" % objId)
logging.info("Test if there is 1 context detected by webclient...")
ret = couch[str(jobId) + ':2:webclient']['details']['value']
i = 0
for c in ret:
if c['name'] == u'js_contexts':
break
i += 1
self.assertEqual(
len(ret[i]['value']), 10,
"Expected 10 JavaScript context in 1:2:webclient, found %s" %
len(ret[i]['value']))
logging.info("Test if there is 1 context analysed by js-sta...")
ret = couch[str(jobId) + ':2:js-sta']['details']['value'][0]['value']
self.assertEqual(
len(ret), 10,
"Expected 10 JavaScript context in 1:2:js-sta, found %s" %
len(ret))
# logging.info("Test if content 0 is obfuscated...")
# ret = jsonpath(couch[str(jobId) + ':2:js-sta'],"$.details.value[?(@.name='Individual contexts')].value[?(@.name='Context no. 0')].value[?(@.name='classification')].value")[0]
# self.assertEqual(ret, u'OBFUSCATED', "Expected context 0 classified as OBFUSCATED, got %s" % ret)
logging.info("Test if there is max. 1 HTTP request...")
ret = com.Configuration.getConf("/var/log/hsn2/site-80.log")
self.assertEqual(len(ret), 1,
"Expected 1 HTTP request, got %d" % len(ret))
logging.info("Test if there is 1 file attached to 1:2:webclient...")
ret = couch[str(jobId) + ':2:webclient']['_attachments'].keys()
self.assertEqual(
len(ret), 1,
"Expected 1 attachment to 1:2:webclinet, got %d" % len(ret))
handle = com.getUrlHandle('http://localhost:80', method='GET')
html = ''.join(handle.readlines())
attachment = ''.join(
couch.get_attachment(str(jobId) + ":2:webclient",
"%s" % ret[0]).readlines())
msg = "Expected HTML attached to 1:2:webclient is exactly the same as the page from http://localhost, but they differ:\nhtml:\n%s\nattachment:\n%s" % (
html, attachment)
self.assertEqual(html, attachment, msg)
def testSimpleJob(self):
logging.info("Test if CouchDB responds to HTTP requests...")
ret = com.Console.call("curl 127.0.0.1:5984 --stderr /dev/null")
logging.info(ret[1])
self.assertIn('{"couchdb":"Welcome","version":"1.2', ret[1], "Couldn't connect to CouchDB.")
logging.info("Test if submitted job is accepted...")
jobId = com.Console.submitJob("simple --param feeder1.uri=/tmp/tests/file.txt")
self.assertIsNotNone(jobId, "Job ID not found.")
logging.info("The job ID is %s" % jobId)
logging.info("Test if job finishes successfully...")
completed = com.Console.waitForCompletion(jobId=jobId, maxTime=30, period=1, verbose=True)
ret = com.Console.call("hc j d %s" % jobId)
logging.info(ret[1])
self.assertTrue(completed, "Job failed to finish successfully.")
# TODO: check time elapsed if completed in >30s
logging.info("Test if job finished in less than 30 seconds...")
jd = com.JobDetails(jobId)
timeElapsed = int(jd.get("job_processing_time_sec"))
self.assertGreaterEqual(
50, timeElapsed, "The job %s took %s seconds, which is more than accepted 30s" % (jobId, timeElapsed)
)
logging.info("Test if there are 4 objects found by unicorn...")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertEqual(len(ret), 4, "Expected 2 objects from unicorn, received %d" % len(ret))
logging.info("Test if there are 2 objects found in CouchDB...")
couch = couchdb.Server()["hsn"]
self.assertEqual(len(couch), 2, "Expected 2 objects from CouchDB, received %d" % len(couch))
logging.info("Test if JavaScript contexts are consistent between webclient and js-sta...")
for objId in couch:
self.assertIn(
objId,
[str(jobId) + ":2:js-sta", str(jobId) + ":2:webclient"],
"%s is not allowed object id in CouchDB" % objId,
)
logging.info("Test if there is 1 context detected by webclient...")
ret = couch[str(jobId) + ":2:webclient"]["details"]["value"]
i = 0
for c in ret:
if c["name"] == u"js_contexts":
break
i += 1
self.assertEqual(
len(ret[i]["value"]), 10, "Expected 10 JavaScript context in 1:2:webclient, found %s" % len(ret[i]["value"])
)
logging.info("Test if there is 1 context analysed by js-sta...")
ret = couch[str(jobId) + ":2:js-sta"]["details"]["value"][0]["value"]
self.assertEqual(len(ret), 10, "Expected 10 JavaScript context in 1:2:js-sta, found %s" % len(ret))
# logging.info("Test if content 0 is obfuscated...")
# ret = jsonpath(couch[str(jobId) + ':2:js-sta'],"$.details.value[?(@.name='Individual contexts')].value[?(@.name='Context no. 0')].value[?(@.name='classification')].value")[0]
# self.assertEqual(ret, u'OBFUSCATED', "Expected context 0 classified as OBFUSCATED, got %s" % ret)
logging.info("Test if there is max. 1 HTTP request...")
ret = com.Configuration.getConf("/var/log/hsn2/site-80.log")
self.assertEqual(len(ret), 1, "Expected 1 HTTP request, got %d" % len(ret))
logging.info("Test if there is 1 file attached to 1:2:webclient...")
ret = couch[str(jobId) + ":2:webclient"]["_attachments"].keys()
self.assertEqual(len(ret), 1, "Expected 1 attachment to 1:2:webclinet, got %d" % len(ret))
handle = com.getUrlHandle("http://localhost:80", method="GET")
html = "".join(handle.readlines())
attachment = "".join(couch.get_attachment(str(jobId) + ":2:webclient", "%s" % ret[0]).readlines())
msg = (
"Expected HTML attached to 1:2:webclient is exactly the same as the page from http://localhost, but they differ:\nhtml:\n%s\nattachment:\n%s"
% (html, attachment)
)
self.assertEqual(html, attachment, msg)
