def testUrlDanger4(self): jobId = com.Console.submitJob( "thug feed.url=http://js.honeysploit.hsn/filtertests.html") self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 120, 2, True) self.assertTrue(finished, "Job failed or took too long.") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertEqual(ret[1].isSet("thug_active"), True) self.assertEquals(ret[1].thug_active, True) self.assertEqual(ret[1].isSet("thug_behaviors"), True) behHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].thug_behaviors.key), method="GET") bList = ow.fromBehaviorList(behHandle) self.assertTrue(len(bList) > 0, "Behavior shouldn't be zero.") self.assertEqual(ret[1].isSet("js_context_list"), True) contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].js_context_list.key), method="GET") cList = ow.fromJSContextList(contHandle) self.assertTrue(len(cList) > 0, "Contexts should have been reported") self.assertEqual(ret[1].isSet("thug_time_start"), True) self.assertEqual(ret[1].isSet("thug_time_stop"), True) self.assertTrue(ret[1].thug_time_stop - ret[1].thug_time_start > 0, "Thug should have run was some time")
def testCompletedWithMatches(self): com.Configuration.setWorkflow( "/tmp/tests/workflows/integration/yara-1.hwl") jobId = com.Console.submitJob( "yara-1 feed.uri=/tmp/tests/resources/json/yara-simple.json yara1.rules_filename=/tmp/tests/resources/yara/rules1.yar" ) self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 5, 1, True) self.assertTrue(finished, "Job failed or took too long.") details = com.JobDetails(jobId) self.assertEqual(details["job_status"], "COMPLETED", name1="job_status") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertTrue(ret[1].isSet("yara_time_start")) self.assertTrue(ret[1].isSet("yara_time_stop")) self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start >= 0) self.assertTrue(ret[1].isSet("yara_matches_found")) self.assertTrue(ret[1].isSet("yara_matches_list")) self.assertTrue(ret[1].yara_matches_found) contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].yara_matches_list.key), method="GET") matches = ow.fromYaraMatchesList(contHandle) self.assertTrue( len(matches) == 2, "There should be two yara matches for test-file.txt") self.assertRuleMatches(matches, 'TextExample', 'default') self.assertRuleMatches(matches, 'AnotherTextExample', 'default')
def testPackersUpx(self): com.Configuration.setWorkflow( "/tmp/tests/workflows/integration/yara-1.hwl") jobId = com.Console.submitJob( "yara-1 feed.uri=/tmp/tests/resources/json/yara-upx.json yara1.rules_filename=/tmp/tests/resources/yara/packers.yar" ) self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 5, 1, True) self.assertTrue(finished, "Job failed or took too long.") details = com.JobDetails(jobId) self.assertEqual(details["job_status"], "COMPLETED", name1="job_status") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertTrue(ret[1].isSet("yara_time_start")) self.assertTrue(ret[1].isSet("yara_time_stop")) self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start > 0, "Expected yara running for more than 0 milliseconds.") self.assertTrue(ret[1].isSet("yara_matches_found")) self.assertTrue(ret[1].isSet("yara_matches_list")) self.assertTrue(ret[1].yara_matches_found) contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].yara_matches_list.key), method="GET") matches = ow.fromYaraMatchesList(contHandle) self.assertTrue( len(matches) == 1, "There should be one yara match for calc.upx") self.assertRuleMatches(matches, 'UPX', 'default')
def testSwfCveMalicious(self): jobId = com.Console.submitJob( "swf-cve-1 feed.uri=/tmp/tests/resources/json/swf-cve-malicious.json" ) self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 5, 1, True) self.assertTrue(finished, "Job failed or took too long.") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertSet(ret[1], "swf_cve_time_begin") self.assertSet(ret[1], "swf_cve_time_end") self.assertSet(ret[1], "swf_cve_detected") self.assertSet(ret[1], "swf_cve_list") self.assertEqual( ret[1].swf_cve_detected, 1, "Expected 1 CVE detected, got %s." % ret[1].swf_cve_detected) handle = com.getUrlHandle('http://localhost:8080/data/%s/%s' % (jobId, ret[1].swf_cve_list.getKey()), method='GET') html = ''.join(handle.readlines()) self.assertIn( 'CVE_2007_0071', html, "Expected CVE_2007_0071 detected, but in data-store there was only:\n%s" % (html))
def testUrlDanger1(self): jobId = com.Console.submitJob("thug feed.url=http://js.honeysploit.hsn/dangerous.html") self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 16, 2, True) self.assertTrue(finished, "Job failed or took too long.") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertEqual(ret[1].isSet("thug_active"), True) self.assertEquals(ret[1].thug_active, True) self.assertEqual(ret[1].isSet("thug_behaviors"), True) behHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].thug_behaviors.key) , method="GET") bList = ow.fromBehaviorList(behHandle) self.assertTrue(len(bList) > 0, "Behavior shouldn't be zero.") self.assertEqual(ret[1].isSet("js_context_list"), True) contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" % (jobId, ret[1].js_context_list.key) , method="GET") cList = ow.fromJSContextList(contHandle) self.assertTrue(len(cList) > 0, "Contexts should have been reported") self.assertEqual(ret[1].isSet("thug_time_start"), True) self.assertEqual(ret[1].isSet("thug_time_stop"), True) self.assertTrue(ret[1].thug_time_stop - ret[1].thug_time_start > 0, "Thug should have run was some time")
def testSwfCveMalicious(self): jobId = com.Console.submitJob("swf-cve-1 feed.uri=/tmp/tests/resources/json/swf-cve-malicious.json") self.assertIsNotNone(jobId, "Returned job id is none.") finished = com.Console.waitForCompletion(jobId, 5, 1, True) self.assertTrue(finished, "Job failed or took too long.") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertSet(ret[1], "swf_cve_time_begin") self.assertSet(ret[1], "swf_cve_time_end") self.assertSet(ret[1], "swf_cve_detected") self.assertSet(ret[1], "swf_cve_list") self.assertEqual(ret[1].swf_cve_detected, 1, "Expected 1 CVE detected, got %s." % ret[1].swf_cve_detected) handle = com.getUrlHandle('http://localhost:8080/data/%s/%s' % (jobId, ret[1].swf_cve_list.getKey()), method='GET') html = ''.join(handle.readlines()) self.assertIn('CVE_2007_0071', html, "Expected CVE_2007_0071 detected, but in data-store there was only:\n%s" % (html))
def testSimpleJob(self): logging.info("Test if CouchDB responds to HTTP requests...") ret = com.Console.call("curl 127.0.0.1:5984 --stderr /dev/null") logging.info(ret[1]) self.assertIn('{"couchdb":"Welcome","version":"1.2', ret[1], "Couldn't connect to CouchDB.") logging.info("Test if submitted job is accepted...") jobId = com.Console.submitJob( "simple --param feeder1.uri=/tmp/tests/file.txt") self.assertIsNotNone(jobId, "Job ID not found.") logging.info('The job ID is %s' % jobId) logging.info("Test if job finishes successfully...") completed = com.Console.waitForCompletion(jobId=jobId, maxTime=30, period=1, verbose=True) ret = com.Console.call("hc j d %s" % jobId) logging.info(ret[1]) self.assertTrue(completed, "Job failed to finish successfully.") # TODO: check time elapsed if completed in >30s logging.info("Test if job finished in less than 30 seconds...") jd = com.JobDetails(jobId) timeElapsed = int(jd.get("job_processing_time_sec")) self.assertGreaterEqual( 50, timeElapsed, "The job %s took %s seconds, which is more than accepted 30s" % (jobId, timeElapsed)) logging.info("Test if there are 4 objects found by unicorn...") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertEqual( len(ret), 4, "Expected 2 objects from unicorn, received %d" % len(ret)) logging.info("Test if there are 2 objects found in CouchDB...") couch = couchdb.Server()['hsn'] self.assertEqual( len(couch), 2, "Expected 2 objects from CouchDB, received %d" % len(couch)) logging.info( "Test if JavaScript contexts are consistent between webclient and js-sta..." ) for objId in couch: self.assertIn( objId, [str(jobId) + ":2:js-sta", str(jobId) + ":2:webclient"], "%s is not allowed object id in CouchDB" % objId) logging.info("Test if there is 1 context detected by webclient...") ret = couch[str(jobId) + ':2:webclient']['details']['value'] i = 0 for c in ret: if c['name'] == u'js_contexts': break i += 1 self.assertEqual( len(ret[i]['value']), 10, "Expected 10 JavaScript context in 1:2:webclient, found %s" % len(ret[i]['value'])) logging.info("Test if there is 1 context analysed by js-sta...") ret = couch[str(jobId) + ':2:js-sta']['details']['value'][0]['value'] self.assertEqual( len(ret), 10, "Expected 10 JavaScript context in 1:2:js-sta, found %s" % len(ret)) # logging.info("Test if content 0 is obfuscated...") # ret = jsonpath(couch[str(jobId) + ':2:js-sta'],"$.details.value[?(@.name='Individual contexts')].value[?(@.name='Context no. 0')].value[?(@.name='classification')].value")[0] # self.assertEqual(ret, u'OBFUSCATED', "Expected context 0 classified as OBFUSCATED, got %s" % ret) logging.info("Test if there is max. 1 HTTP request...") ret = com.Configuration.getConf("/var/log/hsn2/site-80.log") self.assertEqual(len(ret), 1, "Expected 1 HTTP request, got %d" % len(ret)) logging.info("Test if there is 1 file attached to 1:2:webclient...") ret = couch[str(jobId) + ':2:webclient']['_attachments'].keys() self.assertEqual( len(ret), 1, "Expected 1 attachment to 1:2:webclinet, got %d" % len(ret)) handle = com.getUrlHandle('http://localhost:80', method='GET') html = ''.join(handle.readlines()) attachment = ''.join( couch.get_attachment(str(jobId) + ":2:webclient", "%s" % ret[0]).readlines()) msg = "Expected HTML attached to 1:2:webclient is exactly the same as the page from http://localhost, but they differ:\nhtml:\n%s\nattachment:\n%s" % ( html, attachment) self.assertEqual(html, attachment, msg)
def testSimpleJob(self): logging.info("Test if CouchDB responds to HTTP requests...") ret = com.Console.call("curl 127.0.0.1:5984 --stderr /dev/null") logging.info(ret[1]) self.assertIn('{"couchdb":"Welcome","version":"1.2', ret[1], "Couldn't connect to CouchDB.") logging.info("Test if submitted job is accepted...") jobId = com.Console.submitJob("simple --param feeder1.uri=/tmp/tests/file.txt") self.assertIsNotNone(jobId, "Job ID not found.") logging.info("The job ID is %s" % jobId) logging.info("Test if job finishes successfully...") completed = com.Console.waitForCompletion(jobId=jobId, maxTime=30, period=1, verbose=True) ret = com.Console.call("hc j d %s" % jobId) logging.info(ret[1]) self.assertTrue(completed, "Job failed to finish successfully.") # TODO: check time elapsed if completed in >30s logging.info("Test if job finished in less than 30 seconds...") jd = com.JobDetails(jobId) timeElapsed = int(jd.get("job_processing_time_sec")) self.assertGreaterEqual( 50, timeElapsed, "The job %s took %s seconds, which is more than accepted 30s" % (jobId, timeElapsed) ) logging.info("Test if there are 4 objects found by unicorn...") ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg) self.assertEqual(len(ret), 4, "Expected 2 objects from unicorn, received %d" % len(ret)) logging.info("Test if there are 2 objects found in CouchDB...") couch = couchdb.Server()["hsn"] self.assertEqual(len(couch), 2, "Expected 2 objects from CouchDB, received %d" % len(couch)) logging.info("Test if JavaScript contexts are consistent between webclient and js-sta...") for objId in couch: self.assertIn( objId, [str(jobId) + ":2:js-sta", str(jobId) + ":2:webclient"], "%s is not allowed object id in CouchDB" % objId, ) logging.info("Test if there is 1 context detected by webclient...") ret = couch[str(jobId) + ":2:webclient"]["details"]["value"] i = 0 for c in ret: if c["name"] == u"js_contexts": break i += 1 self.assertEqual( len(ret[i]["value"]), 10, "Expected 10 JavaScript context in 1:2:webclient, found %s" % len(ret[i]["value"]) ) logging.info("Test if there is 1 context analysed by js-sta...") ret = couch[str(jobId) + ":2:js-sta"]["details"]["value"][0]["value"] self.assertEqual(len(ret), 10, "Expected 10 JavaScript context in 1:2:js-sta, found %s" % len(ret)) # logging.info("Test if content 0 is obfuscated...") # ret = jsonpath(couch[str(jobId) + ':2:js-sta'],"$.details.value[?(@.name='Individual contexts')].value[?(@.name='Context no. 0')].value[?(@.name='classification')].value")[0] # self.assertEqual(ret, u'OBFUSCATED', "Expected context 0 classified as OBFUSCATED, got %s" % ret) logging.info("Test if there is max. 1 HTTP request...") ret = com.Configuration.getConf("/var/log/hsn2/site-80.log") self.assertEqual(len(ret), 1, "Expected 1 HTTP request, got %d" % len(ret)) logging.info("Test if there is 1 file attached to 1:2:webclient...") ret = couch[str(jobId) + ":2:webclient"]["_attachments"].keys() self.assertEqual(len(ret), 1, "Expected 1 attachment to 1:2:webclinet, got %d" % len(ret)) handle = com.getUrlHandle("http://localhost:80", method="GET") html = "".join(handle.readlines()) attachment = "".join(couch.get_attachment(str(jobId) + ":2:webclient", "%s" % ret[0]).readlines()) msg = ( "Expected HTML attached to 1:2:webclient is exactly the same as the page from http://localhost, but they differ:\nhtml:\n%s\nattachment:\n%s" % (html, attachment) ) self.assertEqual(html, attachment, msg)