def _generate_test_job_with_file(params, filename):
test_file = f"{settings.PROJECT_LOCATION}/test_files/{filename}"
with open(test_file, "rb") as f:
django_file = File(f)
params["file"] = django_file
params["md5"] = hashlib.md5(django_file.file.read()).hexdigest()
test_job = Job(**params)
test_job.save()
return test_job
def _generate_test_job_with_file(params, filename):
test_file = "{}/test_files/{}".format(settings.PROJECT_LOCATION, filename)
with open(test_file, "rb") as f:
django_file = File(f)
params['file'] = django_file
params['md5'] = hashlib.md5(django_file.file.read()).hexdigest()
test_job = Job(**params)
test_job.save()
return test_job
def setUp(self):
params = self.get_params()
params["md5"] = hashlib.md5(
params["observable_name"].encode("utf-8")).hexdigest()
test_job = Job(**params)
test_job.save()
self.job_id = test_job.id
self.observable_name = test_job.observable_name
self.observable_classification = test_job.observable_classification
def setUp(self):
super().setUp()
# get params
params = self.get_params()
# save job instance
self.test_job = Job(**params)
# overwrite if set in env var
if len(self.analyzers_to_test):
self.test_job.analyzers_to_execute = self.analyzers_to_test
self._read_file_save_job(filename=params["file_name"])
class _FileAnalyzersScriptsTestCase(_AbstractAnalyzersScriptTestCase):
# define runtime configs
runtime_configuration = {
"VirusTotal_v2_Scan_File": {"wait_for_scan_anyway": True, "max_tries": 1},
"VirusTotal_v3_Scan_File": {"max_tries": 1, "poll_distance": 1},
"VirusTotal_v3_Get_File": {"max_tries": 1, "poll_distance": 1},
"VirusTotal_v3_Get_File_And_Scan": {
"max_tries": 1,
"poll_distance": 1,
"force_active_scan": True,
"force_active_scan_if_old": False,
},
"Cuckoo_Scan": {"max_poll_tries": 1, "max_post_tries": 1},
"PEframe_Scan": {"max_tries": 1},
"MWDB_Scan": {
"upload_file": True,
"max_tries": 1,
},
"Doc_Info_Experimental": {
"additional_passwords_to_check": ["testpassword"],
"experimental": True,
},
}
@classmethod
def get_params(cls):
return {
**super().get_params(),
"is_sample": True,
}
def setUp(self):
super().setUp()
# get params
params = self.get_params()
# save job instance
self.test_job = Job(**params)
# overwrite if set in env var
if len(self.analyzers_to_test):
self.test_job.analyzers_to_execute = self.analyzers_to_test
self._read_file_save_job(filename=params["file_name"])
def _read_file_save_job(self, filename: str):
test_file = f"{settings.PROJECT_LOCATION}/test_files/{filename}"
with open(test_file, "rb") as f:
self.test_job.file = File(f)
self.test_job.md5 = hashlib.md5(f.read()).hexdigest()
self.test_job.save()
def get_observable_data(job_id):
job_object = Job.object_by_job_id(job_id)
observable_name = job_object.observable_name
observable_classification = job_object.observable_classification
return observable_name, observable_classification
def get_filepath_filename(job_id):
# this function allows to minimize access to the database
# in this way the analyzers could not touch the DB until the end of the analysis
job_object = Job.object_by_job_id(job_id)
filename = job_object.file_name
file_path = job_object.file.path
return file_path, filename
def setUp(self):
params = {
"source": "test",
"is_sample": False,
"observable_name": os.environ.get("TEST_MD5", ""),
"observable_classification": "hash",
"force_privacy": False,
"analyzers_requested": ["test"]
}
params["md5"] = hashlib.md5(
params['observable_name'].encode('utf-8')).hexdigest()
test_job = Job(**params)
test_job.save()
self.job_id = test_job.id
self.observable_name = test_job.observable_name
self.observable_classification = test_job.observable_classification
def setUp(self):
params = {
"source": "test",
"is_sample": False,
"observable_name": os.environ.get("TEST_DOMAIN", "www.google.com"),
"observable_classification": "domain",
"force_privacy": False,
"analyzers_requested": ["test"],
}
params["md5"] = hashlib.md5(
params["observable_name"].encode("utf-8")).hexdigest()
test_job = Job(**params)
test_job.save()
self.job_id = test_job.id
self.observable_name = test_job.observable_name
self.observable_classification = test_job.observable_classification
def set_job_status(job_id, status, errors=None):
message = f"setting job_id {job_id} to status {status}"
if status == "failed":
logger.error(message)
else:
logger.info(message)
job_object = Job.object_by_job_id(job_id)
if errors:
job_object.errors.extend(errors)
job_object.status = status
job_object.save()
class _ObservableAnalyzersScriptsTestCase(_AbstractAnalyzersScriptTestCase):
# define runtime configs
runtime_configuration = {
"Triage_Search": {
"max_tries": 1,
},
"VirusTotal_v3_Get_Observable": {
"max_tries": 1,
"poll_distance": 1,
},
}
@classmethod
def get_params(cls):
return {
**super().get_params(),
"is_sample": False,
}
def setUp(self):
super().setUp()
# init job instance
params = self.get_params()
params["md5"] = hashlib.md5(
params["observable_name"].encode("utf-8")
).hexdigest()
self.test_job = Job(**params)
# overwrite if not set in env var
if len(self.analyzers_to_test):
self.test_job.analyzers_to_execute = self.analyzers_to_test
else:
self.test_job.analyzers_to_execute = [
config.name
for config in self.analyzer_configs.values()
if config.is_observable_type_supported(
params["observable_classification"]
)
]
# save job
self.test_job.save()
def setUp(self):
params = {
"source":
"test",
"is_sample":
False,
"observable_name":
os.environ.get("TEST_MD5", "446c5fbb11b9ce058450555c1c27153c"),
"observable_classification":
"hash",
"force_privacy":
False,
"analyzers_requested": ["test"],
}
params["md5"] = hashlib.md5(
params["observable_name"].encode("utf-8")).hexdigest()
test_job = Job(**params)
test_job.save()
self.job_id = test_job.id
self.observable_name = test_job.observable_name
self.observable_classification = test_job.observable_classification
def setUp(self):
super().setUp()
# init job instance
params = self.get_params()
params["md5"] = hashlib.md5(
params["observable_name"].encode("utf-8")
).hexdigest()
self.test_job = Job(**params)
# overwrite if not set in env var
if len(self.analyzers_to_test):
self.test_job.analyzers_to_execute = self.analyzers_to_test
else:
self.test_job.analyzers_to_execute = [
config.name
for config in self.analyzer_configs.values()
if config.is_observable_type_supported(
params["observable_classification"]
)
]
# save job
self.test_job.save()
def set_report_and_cleanup(job_id, report):
analyzer_name = report.get("name", "")
job_repr = f"({analyzer_name}, job_id: #{job_id})"
logger.info(f"STARTING set_report_and_cleanup for <-- {job_repr}.")
job_object = None
try:
with transaction.atomic():
job_object = Job.object_by_job_id(job_id, transaction=True)
job_object.analysis_reports.append(report)
job_object.save(update_fields=["analysis_reports"])
if job_object.status == "failed":
raise AlreadyFailedJobException()
num_analysis_reports = len(job_object.analysis_reports)
num_analyzers_to_execute = len(job_object.analyzers_to_execute)
logger.info(
f"REPORT: num analysis reports:{num_analysis_reports}, "
f"num analyzer to execute:{num_analyzers_to_execute}"
f" <-- {job_repr}."
)
# check if it was the last analysis...
# ..In case, set the analysis as "reported" or "failed"
if num_analysis_reports == num_analyzers_to_execute:
status_to_set = "reported_without_fails"
# set status "failed" in case all analyzers failed
failed_analyzers = 0
for analysis_report in job_object.analysis_reports:
if not analysis_report.get("success", False):
failed_analyzers += 1
if failed_analyzers == num_analysis_reports:
status_to_set = "failed"
elif failed_analyzers >= 1:
status_to_set = "reported_with_fails"
set_job_status(job_id, status_to_set)
job_object.finished_analysis_time = get_now()
job_object.save(update_fields=["finished_analysis_time"])
except AlreadyFailedJobException:
logger.error(
f"job_id {job_id} status failed. Do not process the report {report}"
)
except Exception as e:
logger.exception(f"job_id: {job_id}, Error: {e}")
set_job_status(job_id, "failed", errors=[str(e)])
job_object.finished_analysis_time = get_now()
job_object.save(update_fields=["finished_analysis_time"])
