def testCompletedWithMatches(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-simple.json yara1.rules_filename=/tmp/tests/resources/yara/rules1.yar"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
details = com.JobDetails(jobId)
self.assertEqual(details["job_status"],
"COMPLETED",
name1="job_status")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertTrue(ret[1].isSet("yara_time_start"))
self.assertTrue(ret[1].isSet("yara_time_stop"))
self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start >= 0)
self.assertTrue(ret[1].isSet("yara_matches_found"))
self.assertTrue(ret[1].isSet("yara_matches_list"))
self.assertTrue(ret[1].yara_matches_found)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].yara_matches_list.key),
method="GET")
matches = ow.fromYaraMatchesList(contHandle)
self.assertTrue(
len(matches) == 2,
"There should be two yara matches for test-file.txt")
self.assertRuleMatches(matches, 'TextExample', 'default')
self.assertRuleMatches(matches, 'AnotherTextExample', 'default')
def testPackersUpx(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-upx.json yara1.rules_filename=/tmp/tests/resources/yara/packers.yar"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
details = com.JobDetails(jobId)
self.assertEqual(details["job_status"],
"COMPLETED",
name1="job_status")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertTrue(ret[1].isSet("yara_time_start"))
self.assertTrue(ret[1].isSet("yara_time_stop"))
self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start > 0,
"Expected yara running for more than 0 milliseconds.")
self.assertTrue(ret[1].isSet("yara_matches_found"))
self.assertTrue(ret[1].isSet("yara_matches_list"))
self.assertTrue(ret[1].yara_matches_found)
contHandle = com.getUrlHandle("http://localhost:8080/data/%s/%d" %
(jobId, ret[1].yara_matches_list.key),
method="GET")
matches = ow.fromYaraMatchesList(contHandle)
self.assertTrue(
len(matches) == 1, "There should be one yara match for calc.upx")
self.assertRuleMatches(matches, 'UPX', 'default')
def testFailed(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-simple.json")
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertFalse(finished, "Job should fail.")
details = com.JobDetails(jobId)
self.assertTrue(details["job_status"] in ("FAILED", "ABORTED"))
self.assertEqual(
details["job_error_message"],
"Both rules_filename and rules_string are missing. One of them is required.",
name1="job_error_message")
def performanceMultiThreadCrawlShallow(self, num=10):
'''
Jobs submitted sequentially, workflow: crawl-shallow.hwl.
Execution control dependent on framework.
'''
passed = True
com.Configuration.setWorkflow(
"/tmp/tests/workflows/performance/crawl-shallow.hwl")
jlog = com.JmeterLog()
totalLabel = "Total Time for %s" % self._testMethodName
totalUid = uuid.uuid4()
jlog.start(totalLabel, totalUid)
uidList = dict()
label = self._testMethodName
for i in range(num):
uid = i #uuid.uuid4()
jlog.start(label, uid)
jobId = com.Console.submitJob(
"crawl-shallow feeder.url=http://js.honeysploit.hsn/dangerous2.html"
)
uidList[uid] = jobId
self.allJobsFinished()
jlog.stop(totalLabel, totalUid)
jlog.success(totalLabel, totalUid, True)
jlog.addsample(totalLabel, totalUid)
for item in uidList.iteritems():
uid = item[0]
jobId = item[1]
if jobId is not None:
details = com.JobDetails(jobId)
duration = int(details.get("job_processing_time_sec")) * 1000
finished = "COMPLETED" in details.get("job_status")
else:
finished = False
duration = 0
if finished is False:
passed = False
jlog.stop(label, uid, duration)
jlog.success(label, uid, bool(finished))
jlog.addsample(label, uid)
com.Configuration.setConf("/tmp/tests/out/%s.jml" % label,
jlog.toxml())
self.assertTrue(
passed, "Seems like one or more jobs failed to submit/complete")
def testCompletedNoMatches(self):
com.Configuration.setWorkflow(
"/tmp/tests/workflows/integration/yara-1.hwl")
jobId = com.Console.submitJob(
"yara-1 feed.uri=/tmp/tests/resources/json/yara-simple.json yara1.rules_filename=/tmp/tests/resources/yara/rules2.yar"
)
self.assertIsNotNone(jobId, "Returned job id is none.")
finished = com.Console.waitForCompletion(jobId, 5, 1, True)
self.assertTrue(finished, "Job failed or took too long.")
details = com.JobDetails(jobId)
self.assertEqual(details["job_status"],
"COMPLETED",
name1="job_status")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertTrue(ret[1].isSet("yara_time_start"))
self.assertTrue(ret[1].isSet("yara_time_stop"))
self.assertTrue(ret[1].yara_time_stop - ret[1].yara_time_start >= 0)
self.assertTrue(ret[1].isSet("yara_matches_found"))
self.assertFalse(ret[1].isSet("yara_matches_list"))
self.assertFalse(ret[1].yara_matches_found)
def testSimpleJob(self):
logging.info("Test if CouchDB responds to HTTP requests...")
ret = com.Console.call("curl 127.0.0.1:5984 --stderr /dev/null")
logging.info(ret[1])
self.assertIn('{"couchdb":"Welcome","version":"1.2', ret[1],
"Couldn't connect to CouchDB.")
logging.info("Test if submitted job is accepted...")
jobId = com.Console.submitJob(
"simple --param feeder1.uri=/tmp/tests/file.txt")
self.assertIsNotNone(jobId, "Job ID not found.")
logging.info('The job ID is %s' % jobId)
logging.info("Test if job finishes successfully...")
completed = com.Console.waitForCompletion(jobId=jobId,
maxTime=30,
period=1,
verbose=True)
ret = com.Console.call("hc j d %s" % jobId)
logging.info(ret[1])
self.assertTrue(completed, "Job failed to finish successfully.")
# TODO: check time elapsed if completed in >30s
logging.info("Test if job finished in less than 30 seconds...")
jd = com.JobDetails(jobId)
timeElapsed = int(jd.get("job_processing_time_sec"))
self.assertGreaterEqual(
50, timeElapsed,
"The job %s took %s seconds, which is more than accepted 30s" %
(jobId, timeElapsed))
logging.info("Test if there are 4 objects found by unicorn...")
ret = com.Console.getDumpAsObjects(jobId, agg=self.testHelp.agg)
self.assertEqual(
len(ret), 4,
"Expected 2 objects from unicorn, received %d" % len(ret))
logging.info("Test if there are 2 objects found in CouchDB...")
couch = couchdb.Server()['hsn']
self.assertEqual(
len(couch), 2,
"Expected 2 objects from CouchDB, received %d" % len(couch))
logging.info(
"Test if JavaScript contexts are consistent between webclient and js-sta..."
)
for objId in couch:
self.assertIn(
objId, [str(jobId) + ":2:js-sta",
str(jobId) + ":2:webclient"],
"%s is not allowed object id in CouchDB" % objId)
logging.info("Test if there is 1 context detected by webclient...")
ret = couch[str(jobId) + ':2:webclient']['details']['value']
i = 0
for c in ret:
if c['name'] == u'js_contexts':
break
i += 1
self.assertEqual(
len(ret[i]['value']), 10,
"Expected 10 JavaScript context in 1:2:webclient, found %s" %
len(ret[i]['value']))
logging.info("Test if there is 1 context analysed by js-sta...")
ret = couch[str(jobId) + ':2:js-sta']['details']['value'][0]['value']
self.assertEqual(
len(ret), 10,
"Expected 10 JavaScript context in 1:2:js-sta, found %s" %
len(ret))
# logging.info("Test if content 0 is obfuscated...")
# ret = jsonpath(couch[str(jobId) + ':2:js-sta'],"$.details.value[?(@.name='Individual contexts')].value[?(@.name='Context no. 0')].value[?(@.name='classification')].value")[0]
# self.assertEqual(ret, u'OBFUSCATED', "Expected context 0 classified as OBFUSCATED, got %s" % ret)
logging.info("Test if there is max. 1 HTTP request...")
ret = com.Configuration.getConf("/var/log/hsn2/site-80.log")
self.assertEqual(len(ret), 1,
"Expected 1 HTTP request, got %d" % len(ret))
logging.info("Test if there is 1 file attached to 1:2:webclient...")
ret = couch[str(jobId) + ':2:webclient']['_attachments'].keys()
self.assertEqual(
len(ret), 1,
"Expected 1 attachment to 1:2:webclinet, got %d" % len(ret))
handle = com.getUrlHandle('http://localhost:80', method='GET')
html = ''.join(handle.readlines())
attachment = ''.join(
couch.get_attachment(str(jobId) + ":2:webclient",
"%s" % ret[0]).readlines())
msg = "Expected HTML attached to 1:2:webclient is exactly the same as the page from http://localhost, but they differ:\nhtml:\n%s\nattachment:\n%s" % (
html, attachment)
self.assertEqual(html, attachment, msg)