def _pyintelowl_logic(args, logger):
md5 = None
results = []
elapsed_time = None
get_configuration_only = False
try:
filename = None
binary = None
if args.command == "file":
if not os.path.exists(args.file):
raise IntelOwlClientException(f"{args.file} does not exists")
with open(args.file, "rb") as f:
binary = f.read()
filename = os.path.basename(f.name)
md5 = hashlib.md5(binary).hexdigest()
elif args.command == "observable":
args.value = args.value.lower()
md5 = hashlib.md5(args.value.encode("utf-8")).hexdigest()
elif args.get_configuration:
get_configuration_only = True
else:
raise IntelOwlClientException(
"you must specify the type of the analysis: [observable, file]"
)
pyintelowl_client = IntelOwl(args.api_token_file, args.certificate,
args.instance, args.debug)
if get_configuration_only:
api_request_result = pyintelowl_client.get_analyzer_configs()
errors = api_request_result.get("errors", [])
if errors:
logger.error(
f"API get_analyzer_configs failed. Errors: {errors}")
analyzers_config = api_request_result.get("answer", {})
logger.info(
f"extracted analyzer_configuration: {analyzers_config}")
pprint(analyzers_config)
exit(0)
analysis_available = False
if not args.skip_check_analysis_availability:
job_id_to_get = None
# first step: ask analysis availability
logger.info(
f"[STARTED] request ask_analysis_availability for md5: {md5}, analyzers: {args.analyzers_list}"
)
api_request_result = pyintelowl_client.ask_analysis_availability(
md5,
args.analyzers_list,
args.run_all_available_analyzers,
args.check_reported_analysis_too,
)
errors = api_request_result.get("errors", [])
if errors:
raise IntelOwlClientException(
f"API ask_analysis_availability failed. Errors: {errors}")
answer = api_request_result.get("answer", {})
status = answer.get("status", None)
if not status:
raise IntelOwlClientException(
"API ask_analysis_availability gave result without status!?!?"
f" Answer: {answer}")
if status != "not_available":
analysis_available = True
job_id_to_get = answer.get("job_id", "")
if job_id_to_get:
logger.info(
f"[INFO] already existing Job(#{job_id_to_get}, md5: {md5},"
f" status: {status}) with analyzers: {args.analyzers_list}"
)
else:
raise IntelOwlClientException(
"API ask_analysis_availability gave result without job_id!?!?"
f" Answer:{answer}")
# second step: in case there are no analysis available, start a new analysis
if not analysis_available:
if args.command == "file":
api_request_result = pyintelowl_client.send_file_analysis_request(
md5,
args.analyzers_list,
filename,
binary,
args.force_privacy,
args.private_job,
args.disable_external_analyzers,
args.run_all_available_analyzers,
)
elif args.command == "observable":
api_request_result = pyintelowl_client.send_observable_analysis_request(
md5,
args.analyzers_list,
args.value,
args.force_privacy,
args.private_job,
args.disable_external_analyzers,
args.run_all_available_analyzers,
)
else:
raise NotImplementedError()
# both cases share the same logic for the management of the result retrieved from the API
errors = api_request_result.get("errors", [])
if errors:
raise IntelOwlClientException(
f"API send_analysis_request failed. Errors: {errors}")
answer = api_request_result.get("answer", {})
logger.info(
f"[INFO] md5: {md5} received response from intel_owl: {answer}"
)
status = answer.get("status", None)
if not status:
raise IntelOwlClientException(
"API send_analysis_request gave result without status!?"
f" Answer:{answer}")
if status != "accepted":
raise IntelOwlClientException(
f"API send_analysis_request gave unexpected result status: {status}"
)
job_id_to_get = answer.get("job_id", None)
analyzers_running = answer.get("analyzers_running", None)
warnings = answer.get("warnings", [])
if job_id_to_get:
logger.info(
f"[STARTED] Job(#{job_id_to_get}, md5: {md5}, status: {status})"
f" -> analyzers: {analyzers_running}. Warnings: {warnings}"
)
else:
raise IntelOwlClientException(
f"API send_analysis_request gave result without job_id!?!?"
f"answer:{answer}")
# third step: at this moment we must have a job_id to check for results
polling_max_tries = 60 * 20
polling_interval = 1
logger.info("[STARTED] polling...")
for chance in range(polling_max_tries):
time.sleep(polling_interval)
api_request_result = pyintelowl_client.ask_analysis_result(
job_id_to_get)
errors = api_request_result.get("errors", [])
if errors:
raise IntelOwlClientException(
f"API ask_analysis_result failed. Errors: {errors}")
answer = api_request_result.get("answer", {})
status = answer.get("status", None)
if not status:
raise IntelOwlClientException(
f"API ask_analysis_result gave result without status!?!?"
f" job_id:{job_id_to_get} answer:{answer}")
if status in ["invalid_id", "not_available"]:
raise IntelOwlClientException(
f"API send_analysis_request gave status {status}")
if status == "running":
continue
if status == "pending":
logger.warning(
f"API ask_analysis_result check job in status 'pending'. Maybe it is stuck"
f"job_id:{job_id_to_get} md5:{md5} analyzer_list:{args.analyzers_list}"
)
elif status in [
"reported_without_fails", "reported_with_fails", "failed"
]:
logger.info(
f"[FINISHED] Job(#{job_id_to_get}, md5: {md5}, status: {status})"
f" -> analyzer_list:{args.analyzers_list}")
results = answer.get("results", [])
elapsed_time = answer.get("elapsed_time_in_seconds", [])
break
if not results:
raise IntelOwlClientException(
f"[ENDED] Reached polling timeout without results. Job_id: {job_id_to_get}"
)
except IntelOwlClientException as e:
logger.error(f"Error: {e} md5: {md5}")
except requests.exceptions.HTTPError as e:
logger.exception(e)
except IntelOwlInvalidAPITokenException as e:
logger.exception(e)
exit(-1)
except Exception as e:
logger.exception(e)
logger.info(f"Elapsed time: {elapsed_time}")
logger.info("Results:")
if args.show_colors:
checkers = Checkers(results, args.value)
observable = get_observable_classification(args.value)
if "domain" in observable:
checkers.check_domain()
elif "hash" in observable:
checkers.check_hash()
elif "url" in observable:
checkers.check_url()
else:
checkers.check_ip()
else:
pprint(results)
def setUp(self):
self.client = IntelOwl(secrets.get_secret("TEST_TOKEN"), False,
"http://nginx", True)
def intel_owl_client():
parser = argparse.ArgumentParser(description='Intel Owl classic client.')
parser.add_argument("-k",
"--api-key",
required=True,
help="your Intel Owl API key")
parser.add_argument("-c",
"--certificate",
default=False,
help="path to Intel Owl certificate")
parser.add_argument("-i",
"--instance",
required=True,
help="your instance URL")
parser.add_argument("-d",
"--debug",
action="store_true",
default=False,
help="debug mode")
parser.add_argument("-l", "--log-to-file", help="log to specified file")
parser.add_argument("-a",
"--analyzers-list",
required=True,
action='append',
help="list of analyzers to launch")
parser.add_argument("-p",
"--force-privacy",
action="store_true",
default=False,
help="disable analyzers that could impact privacy")
parser.add_argument("-e",
"--disable-external-analyzers",
action="store_true",
default=False,
help="disable analyzers that use external services")
parser.add_argument(
"-r",
"--check-reported-analysis-too",
action="store_true",
default=False,
help="check reported analysis too, not only 'running' ones")
parser.add_argument("-s",
"--skip-check-analysis-availability",
action="store_true",
default=False,
help="skip check analysis availability")
subparsers = parser.add_subparsers(help='choose type of analysis',
dest='command')
parser_sample = subparsers.add_parser('file', help='File analysis')
parser_observable = subparsers.add_parser('observable',
help='Observables analysis')
parser_sample.add_argument("-f",
"--file",
required=True,
help="file to analyze")
parser_observable.add_argument("-v",
"--value",
required=True,
help="observable to analyze")
args = parser.parse_args()
logger = get_logger(args.debug, args.log_to_file)
md5 = None
results = []
elapsed_time = None
try:
filename = None
binary = None
if args.command == 'file':
if not os.path.exists(args.file):
raise IntelOwlClientException("{} does not exists".format(
args.file))
with open(args.file, "rb") as f:
binary = f.read()
filename = os.path.basename(f.name)
md5 = hashlib.md5(binary).hexdigest()
elif args.command == 'observable':
md5 = hashlib.md5(args.value.encode('utf-8')).hexdigest()
else:
raise IntelOwlClientException(
"you must specify the type of the analysis: [observable, file]"
)
pyintelowl_client = IntelOwl(args.api_key, args.certificate,
args.instance, args.debug)
analysis_available = False
if not args.skip_check_analysis_availability:
job_id_to_get = None
# first step: ask analysis availability
logger.info(
"about to request ask_analysis_availability for md5: {}, analyzers: {}"
"".format(md5, args.analyzers_list))
api_request_result = pyintelowl_client.ask_analysis_availability(
md5, args.analyzers_list, args.check_reported_analysis_too)
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException(
"API ask_analysis_availability failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
status = answer.get('status', '')
if not status:
raise IntelOwlClientException(
"API ask_analysis_availability gave result without status!?!?"
"answer:{}".format(answer))
elif status != 'not_available':
analysis_available = True
job_id_to_get = answer.get('job_id', '')
if job_id_to_get:
logger.info(
"found already existing job with id {} and status {} for md5 {} and analyzers {}"
"".format(job_id_to_get, status, md5,
args.analyzers_list))
else:
raise IntelOwlClientException(
"API ask_analysis_availability gave result without job_id!?!? answer:{}"
"".format(answer))
# second step: in case there are no analysis available, start a new analysis
if not analysis_available:
if args.command == 'file':
api_request_result = pyintelowl_client.send_file_analysis_request(
md5, args.analyzers_list, filename, binary,
args.force_privacy, args.disable_external_analyzers)
elif args.command == 'observable':
api_request_result = pyintelowl_client.send_observable_analysis_request(
md5, args.analyzers_list, args.value, args.force_privacy,
args.disable_external_analyzers)
else:
raise NotImplementedError()
# both cases share the same logic for the management of the result retrieved from the API
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException(
"API send_analysis_request failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
logger.info("md5 {} received response from intel_owl: {}".format(
md5, answer))
status = answer.get('status', '')
if not status:
raise IntelOwlClientException(
"API send_analysis_request gave result without status!?!? answer:{}"
"".format(answer))
elif status != "accepted":
raise IntelOwlClientException(
"API send_analysis_request gave unexpected result status:{}"
"".format(status))
else:
job_id_to_get = answer.get('job_id', '')
analyzers_running = answer.get('analyzers_running', '')
warnings = answer.get('warnings', [])
if job_id_to_get:
logger.info(
"started job with id {} and status {} for md5 {} and analyzers {}. Warnings:{}"
"".format(job_id_to_get, status, md5,
analyzers_running, warnings))
else:
raise IntelOwlClientException(
"API send_analysis_request gave result without job_id!?!?"
"answer:{}".format(answer))
# third step: at this moment we must have a job_id to check for results
polling_max_tries = 60 * 20
polling_interval = 1
logger.info("started polling")
for chance in range(polling_max_tries):
time.sleep(polling_interval)
api_request_result = pyintelowl_client.ask_analysis_result(
job_id_to_get)
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException(
"API ask_analysis_result failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
status = answer.get('status', '')
if not status:
raise IntelOwlClientException(
"API ask_analysis_result gave result without status!?!? job_id:{} answer:{}"
"".format(job_id_to_get, answer))
elif status in ['invalid_id', 'not_available']:
raise IntelOwlClientException(
"API send_analysis_request gave status {}".format(status))
elif status == 'running':
continue
elif status == 'pending':
logger.warning(
"API ask_analysis_result check job in status 'pending'. Maybe it is stuck"
"job_id:{} md5:{} analyzer_list:{}".format(
job_id_to_get, md5, args.analyzers_list))
elif status in [
'reported_without_fails', 'reported_with_fails', 'failed'
]:
logger.info("job_id {} Analysis finished. Status: {} "
"md5:{} analyzer_list:{}".format(
job_id_to_get, status, md5,
args.analyzers_list))
results = answer.get('results', [])
elapsed_time = answer.get('elapsed_time_in_seconds', [])
break
if not results:
raise IntelOwlClientException(
"reached polling timeout without results. Job_id {}"
"".format(job_id_to_get))
except IntelOwlClientException as e:
logger.error("Error:{} md5:{}".format(e, md5))
except requests.exceptions.HTTPError as e:
logger.error(e)
except Exception as e:
logger.exception(e)
logger.info("elapsed time: {}".format(elapsed_time))
logger.info("results:")
pprint.pprint(results)
class ApiTests(TestCase):
def setUp(self):
self.client = IntelOwl(secrets.get_secret("TEST_TOKEN"), False,
"http://nginx", True)
def test_ask_analysis_availability(self):
md5 = os.environ.get("TEST_MD5", "")
analyzers_needed = ["Fortiguard", "CIRCLPassiveDNS"]
api_request_result = self.client.ask_analysis_availability(
md5, analyzers_needed)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def test_send_corrupted_sample_pe(self):
filename = "non_valid_pe.exe"
test_file = "{}/test_files/{}".format(settings.PROJECT_LOCATION,
filename)
with open(test_file, "rb") as f:
binary = f.read()
md5 = hashlib.md5(binary).hexdigest()
analyzers_requested = [
"File_Info", "PE_Info", "Strings_Info_Classic", "Signature_Info"
]
api_request_result = self.client.send_file_analysis_request(
md5, analyzers_requested, filename, binary, False)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def test_send_analysis_request_sample(self):
filename = "file.exe"
test_file = "{}/test_files/{}".format(settings.PROJECT_LOCATION,
filename)
with open(test_file, "rb") as f:
binary = f.read()
md5 = hashlib.md5(binary).hexdigest()
analyzers_requested = [
"Yara_Scan", "HybridAnalysis_Get_File", "Cuckoo_ScanClassic",
"Intezer_Scan", "VirusTotal_v3_Get_File",
"VirusTotal_v3_Scan_File", "File_Info", "PE_Info", "Doc_Info",
"PDF_Info", "Strings_Info_Classic", "Strings_Info_ML"
]
api_request_result = self.client.send_file_analysis_request(
md5, analyzers_requested, filename, binary, False)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def test_send_analysis_request_domain(self):
analyzers_requested = [
"Fortiguard", "CIRCLPassiveDNS", "GoogleSafebrowsing",
"Robtex_Forward_PDNS_Query", "OTXQuery",
"VirusTotal_v3_Get_Observable", "HybridAnalysis_Get_Observable"
]
observable_name = os.environ.get("TEST_DOMAIN", "")
md5 = hashlib.md5(observable_name.encode('utf-8')).hexdigest()
api_request_result = self.client.send_observable_analysis_request(
md5, analyzers_requested, observable_name, False)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def test_send_analysis_request_ip(self):
analyzers_requested = [
"TorProject", "AbuseIPDB", "Shodan", "MaxMindGeoIP",
"CIRCLPassiveSSL", "GreyNoiseAlpha", "GoogleSafebrowsing",
"Robtex_IP_Query", "Robtex_Reverse_PDNS_Query", "TalosReputation",
"OTXQuery", "VirusTotal_Get_v2_Observable",
"HybridAnalysis_Get_Observable", "Hunter", "HoneyDB"
]
observable_name = os.environ.get("TEST_IP", "")
md5 = hashlib.md5(observable_name.encode('utf-8')).hexdigest()
api_request_result = self.client.send_observable_analysis_request(
md5, analyzers_requested, observable_name, False)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def test_ask_analysis_result(self):
# put your test job_id
job_id = os.environ.get("TEST_JOB_ID", "")
api_request_result = self.client.ask_analysis_result(job_id)
answer = api_request_result.get('answer', {})
print(answer)
errors = api_request_result.get('errors', [])
self.assertFalse(errors)
def _pyintelowl_logic(args, logger):
md5 = None
results = []
elapsed_time = None
get_configuration_only = False
try:
filename = None
binary = None
if args.command == 'file':
if not os.path.exists(args.file):
raise IntelOwlClientException("{} does not exists".format(args.file))
with open(args.file, "rb") as f:
binary = f.read()
filename = os.path.basename(f.name)
md5 = hashlib.md5(binary).hexdigest()
elif args.command == 'observable':
args.value = args.value.lower()
md5 = hashlib.md5(args.value.encode('utf-8')).hexdigest()
elif args.get_configuration:
get_configuration_only = True
else:
raise IntelOwlClientException("you must specify the type of the analysis: [observable, file]")
pyintelowl_client = IntelOwl(args.api_token_file, args.certificate, args.instance, args.debug)
if get_configuration_only:
api_request_result = pyintelowl_client.get_analyzer_configs()
errors = api_request_result.get('errors', [])
if errors:
logger.error("API get_analyzer_configs failed. Errors: {}".format(errors))
analyzers_config = api_request_result.get('answer', {})
logger.info("extracted analyzer_configuration: {}".format(analyzers_config))
pprint(analyzers_config)
exit(0)
analysis_available = False
if not args.skip_check_analysis_availability:
job_id_to_get = None
# first step: ask analysis availability
logger.info("about to request ask_analysis_availability for md5: {}, analyzers: {}"
"".format(md5, args.analyzers_list))
api_request_result = \
pyintelowl_client.ask_analysis_availability(md5, args.analyzers_list,
args.run_all_available_analyzers,
args.check_reported_analysis_too)
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException("API ask_analysis_availability failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
status = answer.get('status', '')
if not status:
raise IntelOwlClientException("API ask_analysis_availability gave result without status!?!?"
"answer:{}".format(answer))
if status != 'not_available':
analysis_available = True
job_id_to_get = answer.get('job_id', '')
if job_id_to_get:
logger.info("found already existing job with id {} and status {} for md5 {} and analyzers {}"
"".format(job_id_to_get, status, md5, args.analyzers_list))
else:
raise IntelOwlClientException(
"API ask_analysis_availability gave result without job_id!?!? answer:{}"
"".format(answer))
# second step: in case there are no analysis available, start a new analysis
if not analysis_available:
if args.command == 'file':
api_request_result = pyintelowl_client.send_file_analysis_request(md5, args.analyzers_list, filename,
binary, args.force_privacy,
args.disable_external_analyzers,
args.run_all_available_analyzers)
elif args.command == 'observable':
api_request_result = pyintelowl_client.send_observable_analysis_request(md5, args.analyzers_list,
args.value, args.force_privacy,
args.disable_external_analyzers,
args.run_all_available_analyzers)
else:
raise NotImplementedError()
# both cases share the same logic for the management of the result retrieved from the API
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException("API send_analysis_request failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
logger.info("md5 {} received response from intel_owl: {}".format(md5, answer))
status = answer.get('status', '')
if not status:
raise IntelOwlClientException(
"API send_analysis_request gave result without status!?!? answer:{}"
"".format(answer))
if status != "accepted":
raise IntelOwlClientException("API send_analysis_request gave unexpected result status:{}"
"".format(status))
job_id_to_get = answer.get('job_id', '')
analyzers_running = answer.get('analyzers_running', '')
warnings = answer.get('warnings', [])
if job_id_to_get:
logger.info("started job with id {} and status {} for md5 {} and analyzers {}. Warnings:{}"
"".format(job_id_to_get, status, md5, analyzers_running, warnings))
else:
raise IntelOwlClientException("API send_analysis_request gave result without job_id!?!?"
"answer:{}".format(answer))
# third step: at this moment we must have a job_id to check for results
polling_max_tries = 60 * 20
polling_interval = 1
logger.info("started polling")
for chance in range(polling_max_tries):
time.sleep(polling_interval)
api_request_result = pyintelowl_client.ask_analysis_result(job_id_to_get)
errors = api_request_result.get('errors', [])
if errors:
raise IntelOwlClientException("API ask_analysis_result failed. Errors: {}"
"".format(errors))
answer = api_request_result.get('answer', {})
status = answer.get('status', '')
if not status:
raise IntelOwlClientException(
"API ask_analysis_result gave result without status!?!? job_id:{} answer:{}"
"".format(job_id_to_get, answer))
if status in ['invalid_id', 'not_available']:
raise IntelOwlClientException("API send_analysis_request gave status {}".format(status))
if status == 'running':
continue
if status == 'pending':
logger.warning("API ask_analysis_result check job in status 'pending'. Maybe it is stuck"
"job_id:{} md5:{} analyzer_list:{}".format(job_id_to_get, md5, args.analyzers_list))
elif status in ['reported_without_fails', 'reported_with_fails', 'failed']:
logger.info("job_id {} Analysis finished. Status: {} "
"md5:{} analyzer_list:{}".format(job_id_to_get, status, md5, args.analyzers_list))
results = answer.get('results', [])
elapsed_time = answer.get('elapsed_time_in_seconds', [])
break
if not results:
raise IntelOwlClientException("reached polling timeout without results. Job_id {}"
"".format(job_id_to_get))
except IntelOwlClientException as e:
logger.error("Error:{} md5:{}".format(e, md5))
except requests.exceptions.HTTPError as e:
logger.exception(e)
except IntelOwlInvalidAPITokenException as e:
logger.exception(e)
exit(-1)
except Exception as e:
logger.exception(e)
logger.info("elapsed time: {}".format(elapsed_time))
logger.info("results:")
if args.show_json:
pprint(results)
sys.exit()
checkers = Checkers(results, args.value)
observable = get_observable_classification(args.value)
if 'domain' in observable:
checkers.check_domain()
elif 'hash' in observable:
checkers.check_hash()
else:
checkers.check_ip()