def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_version = additional_config_params.get("greynoise_api_version",
"v1")
if api_version == "v1":
url = "https://api.greynoise.io/v1/query/ip"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
data = {"ip": observable_name}
response = requests.post(url, data=data, headers=headers)
response.raise_for_status()
elif api_version == "v2":
url = f"https://api.greynoise.io/v2/noise/context/{observable_name}"
api_key_name = additional_config_params.get(
"api_key_name", "GREYNOISE_API_KEY")
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("GREYNOISE_API_KEY not specified.")
headers = {"Accept": "application/json", "key": api_key}
response = requests.get(url, headers=headers)
response.raise_for_status()
else:
raise AnalyzerRunException(
"Invalid API Version. Supported are: v1 (free) & v2 (paid).")
result = response.json()
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
result = _urlhaus_get_report(observable_name,
observable_classification)
report['report'] = result
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
# You should save CIRCL credentials with this template: "<user>|<pwd>"
credentials = secrets.get_secret("CIRCL_CREDENTIALS")
if not credentials:
raise AnalyzerRunException("no credentials retrieved")
split_credentials = credentials.split('|')
if len(split_credentials) != 2:
raise AnalyzerRunException(
"CIRCL credentials not properly configured."
"Template to use: '<user>|<pwd>'")
user = split_credentials[0]
pwd = split_credentials[1]
pdns = pypdns.PyPDNS(basic_auth=(user, pwd))
domain = observable_name
if observable_classification == 'url':
domain = urlparse(observable_name).hostname
result = pdns.query(domain)
for result_item in result:
keys_to_decode = ['time_first', 'time_last']
for key_to_decode in keys_to_decode:
time_extracted = result_item.get(key_to_decode, None)
if time_extracted and isinstance(time_extracted,
datetime.datetime):
result_item[key_to_decode] = time_extracted.strftime(
"%Y-%m-%d %H:%M:%S")
# pprint.pprint(result)
report['report'] = result
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
"""Run ActiveDNS analyzer
Admit:
* additional_config_params[service]: google - Google DoH (DNS over HTTPS)
* additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS)
* additional_config_params[service]: classic - classic DNS query
Google and CloudFlare return an IP (or NXDOMAIN) from a domain.
Classic support also reverse lookup (domain from IP)
:param analyzer_name: Analyzer configuration in analyzer_config.json
:type analyzer_name: str
:param job_id: job identifier
:type job_id: str
:param observable_name: analyzed observable
:type observable_name: str
:param observable_classification: observable classification (allow: ip or domain) ip only classic
:type observable_classification: str
:param additional_config_params: params service to select the service
:type additional_config_params: dict
:return: report: name: observable_name, resolution: ip,NXDOMAIN, ''
:rtype: report: dict
"""
logger.info(f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}")
report = general.get_basic_report_template(analyzer_name)
try:
dns_type = additional_config_params.get('service', '')
if dns_type == 'google':
_doh_google(job_id, analyzer_name, observable_classification, observable_name, report)
elif dns_type == 'cloudflare':
_doh_cloudflare(job_id, analyzer_name, observable_classification, observable_name,
report)
elif dns_type == 'classic':
_classic_dns(job_id, analyzer_name, observable_classification, observable_name, report)
else:
raise AnalyzerConfigurationException(f'Service selected: {dns_type} is not available')
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \
f"observable_name:{observable_name} Analyzer error {e}"
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \
f"observable_name:{observable_name} Unexpected error {e}"
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
general.set_report_and_cleanup(job_id, report)
logger.info(f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}")
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
rt_value = additional_config_params.get("rt_value", "")
params = {"q": observable_name}
if rt_value:
params["rt"] = rt_value
if observable_classification == "domain":
uri = "domain.php"
elif observable_classification == "ip":
uri = "host.php"
else:
raise AnalyzerRunException(
"not supported observable type {}. Supported are IP and Domain"
"".format(observable_classification))
try:
response = requests.get(base_url + uri, params=params)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
robtex_analysis = additional_config_params.get('robtex_analysis',
'ip_query')
if robtex_analysis == 'ip_query':
uri = 'ipquery/{}'.format(observable_name)
elif robtex_analysis == 'reverse_pdns':
uri = 'pdns/reverse/{}'.format(observable_name)
elif robtex_analysis == 'forward_pdns':
domain = observable_name
if observable_classification == 'url':
domain = urlparse(observable_name).hostname
uri = 'pdns/forward/{}'.format(domain)
else:
raise AnalyzerRunException(
"not supported analysis type {}.".format(robtex_analysis))
try:
response = requests.get(base_url + uri)
response.raise_for_status()
result = response.text.split('\r\n')
except requests.ConnectionError as e:
raise AnalyzerRunException("connection error: {}".format(e))
else:
loaded_results = []
for item in result:
if len(item) > 0:
loaded_results.append(json.loads(item))
# pprint.pprint(loaded_results)
report['report'] = loaded_results
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
# You should save CIRCL credentials with this template: "<user>|<pwd>"
credentials = secrets.get_secret("CIRCL_CREDENTIALS")
if not credentials:
raise AnalyzerRunException("no credentials retrieved")
split_credentials = credentials.split('|')
if len(split_credentials) != 2:
raise AnalyzerRunException("CIRCL credentials not properly configured."
"Template to use: '<user>|<pwd>'")
user = split_credentials[0]
pwd = split_credentials[1]
pssl = pypssl.PyPSSL(basic_auth=(user, pwd))
result = pssl.query(observable_name)
certificates = []
if result.get(observable_name, {}):
certificates = list(result.get(observable_name).get('certificates', []))
parsed_result = {'ip': observable_name, 'certificates': []}
for cert in certificates:
subject = result.get(observable_name).get('subjects', {}).get(cert, {}).get('values', [])
if subject:
parsed_result['certificates'].append({'fingerprint': cert, 'subject': subject[0]})
# pprint.pprint(parsed_result)
report['report'] = parsed_result
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info(
"started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "")
if not api_key_name:
api_key_name = "AUTH0_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
headers = {"X-Auth-Token": api_key}
url = "https://signals.api.auth0.com/v2.0/ip/{}".format(observable_name)
response = requests.get(url, headers=headers)
response.raise_for_status()
json_response = response.json()
# pprint.pprint(json_response)
report["report"] = json_response
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info(
"ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info(
"started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "HUNTER_API_KEY")
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no Hunter.io API Key retrieved")
url = (
"https://api.hunter.io/v2/domain-search?"
f"domain={observable_name}&api_key={api_key}"
)
response = requests.get(url)
response.raise_for_status()
json_response = response.json()
report["report"] = json_response
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info(
"ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "")
if not api_key_name:
api_key_name = "ABUSEIPDB_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
headers = {"Key": api_key, "Accept": "application/json"}
params = {
"ipAddress": observable_name,
"maxAgeInDays": 180,
"verbose": True
}
url = "https://api.abuseipdb.com/api/v2/check"
response = requests.get(url, params=params, headers=headers)
response.raise_for_status()
json_response = response.json()
# pprint.pprint(json_response)
report["report"] = json_response
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, filepath, filename, md5,
additional_config_params):
logger.info("started analyzer {} job_id {}"
"".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
# cuckoo installation can be with or without the api_token
# it depends on version and configuration
api_key_name = additional_config_params.get("api_key_name", "")
if api_key_name:
api_key = secrets.get_secret(api_key_name)
else:
api_key = None
logger.info("job_id {} md5 {} analyzer {} no API key set"
"".format(job_id, md5, analyzer_name))
cuckoo_url = secrets.get_secret("CUCKOO_URL")
if not cuckoo_url:
raise AnalyzerRunException("cuckoo URL missing")
cuckoo_analysis = CuckooAnalysis(api_key, cuckoo_url)
binary = general.get_binary(job_id)
if not binary:
raise AnalyzerRunException("is the binary empty?!")
_cuckoo_scan_file(cuckoo_analysis, additional_config_params, filename,
md5, binary)
result = cuckoo_analysis.report
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id))
return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params):
logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
results = {}
peepdf_analysis = []
pdf_parser = peepdf.PDFCore.PDFParser()
ret, pdf = pdf_parser.parse(filepath, True)
if ret:
peepdf_analysis["status_code"] = ret
else:
stats = pdf.getStats()
for version in stats.get("Versions", []):
version_dict = {
"events": version.get("Events", {}),
"actions": version.get("Actions", {}),
"urls": version.get("URLs", []),
"uris": version.get("URIs", []),
"elements": version.get("Elements", {}),
"vulns": version.get("Vulns", []),
"objects_with_js_code": version.get("Objects with JS code", []),
}
peepdf_analysis.append(version_dict)
results["peepdf"] = peepdf_analysis
# pprint.pprint(results)
report["report"] = results
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e)
)
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e)
)
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_id_name = additional_config_params.get("api_id_name",
"CENSYS_API_ID")
api_secret_name = additional_config_params.get("api_secret_name",
"CENSYS_API_SECRET")
api_id = secrets.get_secret(api_id_name)
api_secret = secrets.get_secret(api_secret_name)
if not (api_id and api_secret):
raise AnalyzerRunException("no api credentials retrieved")
result = _censys_get_report(
(api_id, api_secret),
observable_name,
observable_classification,
additional_config_params,
)
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "")
if not api_key_name:
api_key_name = "VT_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
result = vt_get_report(
api_key,
observable_name,
observable_classification,
additional_config_params,
job_id,
)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
# pprint.pprint(report)
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key = secrets.get_secret("GSF_KEY")
if not api_key:
raise AnalyzerRunException(
"no api key retrieved. job_id {}, analyzer {}".format(
job_id, analyzer_name))
sb_instance = SafeBrowsing(api_key)
response = sb_instance.lookup_urls([observable_name])
if observable_name in response and isinstance(
response[observable_name], dict):
result = response[observable_name]
else:
raise AnalyzerRunException(
"result not expected: {}".format(response))
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params):
logger.info("started analyzer {} job_id {}"
"".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
results = {}
peepdf_analysis = []
pdf_parser = peepdf.PDFCore.PDFParser()
ret, pdf = pdf_parser.parse(filepath, True)
if ret:
peepdf_analysis['status_code'] = ret
else:
stats = pdf.getStats()
for version in stats.get('Versions', []):
version_dict = {
'events': version.get('Events', {}),
'actions': version.get('Actions', {}),
'urls': version.get('URLs', []),
'uris': version.get('URIs', []),
'elements': version.get('Elements', {}),
'vulns': version.get('Vulns', []),
'objects_with_js_code': version.get('Objects with JS code', [])
}
peepdf_analysis.append(version_dict)
results['peepdf'] = peepdf_analysis
# pprint.pprint(results)
report['report'] = results
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" \
"".format(job_id, analyzer_name, md5, filename, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" \
"".format(job_id, analyzer_name, md5, filename, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {}"
"".format(analyzer_name, job_id))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
if observable_classification == "hash":
filehash = observable_name
else:
raise AnalyzerRunException(
f"not supported observable type {observable_classification}."
f" Supported: hash only")
post_data = {"query": "get_info", "hash": filehash}
url = "https://mb-api.abuse.ch/api/v1/"
response = requests.post(url, data=post_data)
response.raise_for_status()
json_response = response.json()
report["report"] = json_response
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
try:
if not os.path.isfile(database_location):
updater(additional_config_params)
reader = maxminddb.open_database(database_location)
maxmind_result = reader.get(observable_name)
reader.close()
except maxminddb.InvalidDatabaseError as e:
error_message = "invalid database error: {}".format(e)
logger.exception(error_message)
maxmind_result = {"error": error_message}
if not maxmind_result:
maxmind_result = {}
# pprint.pprint(maxmind_result)
report["report"] = maxmind_result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("finished analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, filepath, filename, md5,
additional_config_params):
logger.info("started analyzer {} job_id {}"
"".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
results = {}
results["magic"] = magic.from_file(filepath)
results["mimetype"] = magic.from_file(filepath, mime=True)
results["filetype"] = pyexifinfo.fileType(filepath)
exif_report = pyexifinfo.get_json(filepath)
if exif_report:
exif_report_cleaned = {
key: value
for key, value in exif_report[0].items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
results["exiftool"] = exif_report_cleaned
binary = general.get_binary(job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(filepath).decode()
report["report"] = results
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id))
return report
def run(analyzer_name, job_id, filepath, filename, md5,
additional_config_params):
logger.info("started analyzer {} job_id {}"
"".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "")
if not api_key_name:
api_key_name = "INTEZER_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
intezer_token = os.environ.get("INTEZER_TOKEN", "")
intezer_token_date = os.environ.get("INTEZER_TOKEN_DATE", "")
today = get_now_date_only()
if not intezer_token or intezer_token_date != today:
intezer_token = _get_access_token(api_key)
if not intezer_token:
raise AnalyzerRunException("token extraction failed")
binary = general.get_binary(job_id)
result = _intezer_scan_file(intezer_token, md5, filename, binary,
additional_config_params)
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
result = {"found": False}
if not os.path.isfile(database_location):
updater()
with open(database_location, "r") as f:
db = f.read()
db_list = db.split("\n")
# pprint.pprint(db_list)
if observable_name in db_list:
result["found"] = True
# pprint.pprint(result)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("finished analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
domain = observable_name
if observable_classification == 'url':
domain = urlparse(observable_name).hostname
try:
url = 'https://freeapi.robtex.com/pdns/forward/{}'.format(domain)
response = requests.get(url)
response.raise_for_status()
result = response.text.split('\r\n')
except requests.ConnectionError as e:
raise AnalyzerRunException("connection error: {}".format(e))
else:
loaded_results = []
for item in result:
if len(item) > 0:
loaded_results.append(json.loads(item))
# pprint.pprint(loaded_results)
report['report'] = loaded_results
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get('api_key_name', 'HONEYDB_API_KEY')
api_id_name = additional_config_params.get('api_id_name', 'HONEYDB_API_ID')
api_key = secrets.get_secret(api_key_name)
api_id = secrets.get_secret(api_id_name)
if not api_key:
raise AnalyzerRunException("no HoneyDB API Key retrieved")
if not api_id:
raise AnalyzerRunException("no HoneyDB API ID retrieved")
headers = {
'X-HoneyDb-ApiKey': api_key,
'X-HoneyDb-ApiId': api_id
}
url = f'https://honeydb.io/api/twitter-threat-feed/{observable_name}'
response = requests.get(url, headers=headers)
response.raise_for_status()
json_response = response.json()
report['report'] = json_response
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
pattern = re.compile(r"(?:Category: )([\w\s]+)")
baseurl = "https://www.fortiguard.com/webfilter?q="
url = baseurl + observable_name
response = requests.get(url)
response.raise_for_status()
category_match = re.search(pattern, str(response.content), flags=0)
dict_response = {
"category": category_match.group(1) if category_match else ""
}
report["report"] = dict_response
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
observable_to_analyze = observable_name
if observable_classification == "url":
observable_to_analyze = urlparse(observable_name).hostname
url = base_url + observable_to_analyze
response = requests.get(url)
response.raise_for_status()
result = response.json()
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get('api_key_name', '')
if not api_key_name:
api_key_name = "SHODAN_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
params = {'key': api_key, 'minify': True}
url = 'https://api.shodan.io/shodan/host/{}'.format(observable_name)
response = requests.get(url, params=params)
response.raise_for_status()
json_response = response.json()
# pprint.pprint(json_response)
report['report'] = json_response
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get('api_key_name', '')
if not api_key_name:
api_key_name = "KEY_NAME"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
result = {}
# do something here
# pprint.pprint(result)
report['report'] = result
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(analyzer_name, job_id, filepath, filename, md5,
additional_config_params):
logger.info(f"started analyzer {analyzer_name} job_id {job_id}")
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "")
if not api_key_name:
api_key_name = "VT_KEY"
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException("no api key retrieved")
result = vt_scan_file(api_key, md5, job_id, additional_config_params)
report["report"] = result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
# pprint.pprint(report)
general.set_report_and_cleanup(job_id, report)
logger.info(f"ended analyzer {analyzer_name} job_id {job_id}")
return report
def run(analyzer_name, job_id, observable_name, observable_classification,
additional_config_params):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
url = 'https://api.greynoise.io/v1/query/ip'
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
data = {'ip': observable_name}
response = requests.post(url, data=data, headers=headers)
response.raise_for_status()
result = response.json()
# pprint.pprint(result)
report['report'] = result
except AnalyzerRunException as e:
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
"".format(job_id, analyzer_name, observable_name, e)
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
else:
report['success'] = True
general.set_report_and_cleanup(job_id, report, logger)
logger.info("ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info(
f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}"
)
report = general.get_basic_report_template(analyzer_name)
try:
results = {}
if observable_classification != "hash":
raise AnalyzerRunException(
f"observable type {observable_classification} not supported"
)
results["found"] = False
# reference: https://team-cymru.com/community-services/mhr/
# if the resolution works, this means that the file is reported
# as malware by Cymru
resolutions = []
try:
query_to_perform = f"{observable_name}.malware.hash.cymru.com"
domains = socket.gethostbyaddr(query_to_perform)
resolutions = domains[2]
except (socket.gaierror, socket.herror):
logger.info(f"observable {observable_name} not found in HMR DB")
if resolutions:
results["found"] = True
results["resolution_data"] = resolutions
report["report"] = results
except AnalyzerRunException as e:
error_message = (
f"job_id:{job_id} analyzer:{analyzer_name}"
f" observable_name:{observable_name} Analyzer error {e}"
)
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
f"job_id:{job_id} analyzer:{analyzer_name}"
f" observable_name:{observable_name} Unexpected error {e}"
)
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info(
f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}"
)
return report
