def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_version = additional_config_params.get("greynoise_api_version", "v1") if api_version == "v1": url = "https://api.greynoise.io/v1/query/ip" headers = {"Content-Type": "application/x-www-form-urlencoded"} data = {"ip": observable_name} response = requests.post(url, data=data, headers=headers) response.raise_for_status() elif api_version == "v2": url = f"https://api.greynoise.io/v2/noise/context/{observable_name}" api_key_name = additional_config_params.get( "api_key_name", "GREYNOISE_API_KEY") api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("GREYNOISE_API_KEY not specified.") headers = {"Accept": "application/json", "key": api_key} response = requests.get(url, headers=headers) response.raise_for_status() else: raise AnalyzerRunException( "Invalid API Version. Supported are: v1 (free) & v2 (paid).") result = response.json() report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: result = _urlhaus_get_report(observable_name, observable_classification) report['report'] = result except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: # You should save CIRCL credentials with this template: "<user>|<pwd>" credentials = secrets.get_secret("CIRCL_CREDENTIALS") if not credentials: raise AnalyzerRunException("no credentials retrieved") split_credentials = credentials.split('|') if len(split_credentials) != 2: raise AnalyzerRunException( "CIRCL credentials not properly configured." "Template to use: '<user>|<pwd>'") user = split_credentials[0] pwd = split_credentials[1] pdns = pypdns.PyPDNS(basic_auth=(user, pwd)) domain = observable_name if observable_classification == 'url': domain = urlparse(observable_name).hostname result = pdns.query(domain) for result_item in result: keys_to_decode = ['time_first', 'time_last'] for key_to_decode in keys_to_decode: time_extracted = result_item.get(key_to_decode, None) if time_extracted and isinstance(time_extracted, datetime.datetime): result_item[key_to_decode] = time_extracted.strftime( "%Y-%m-%d %H:%M:%S") # pprint.pprint(result) report['report'] = result except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): """Run ActiveDNS analyzer Admit: * additional_config_params[service]: google - Google DoH (DNS over HTTPS) * additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS) * additional_config_params[service]: classic - classic DNS query Google and CloudFlare return an IP (or NXDOMAIN) from a domain. Classic support also reverse lookup (domain from IP) :param analyzer_name: Analyzer configuration in analyzer_config.json :type analyzer_name: str :param job_id: job identifier :type job_id: str :param observable_name: analyzed observable :type observable_name: str :param observable_classification: observable classification (allow: ip or domain) ip only classic :type observable_classification: str :param additional_config_params: params service to select the service :type additional_config_params: dict :return: report: name: observable_name, resolution: ip,NXDOMAIN, '' :rtype: report: dict """ logger.info(f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}") report = general.get_basic_report_template(analyzer_name) try: dns_type = additional_config_params.get('service', '') if dns_type == 'google': _doh_google(job_id, analyzer_name, observable_classification, observable_name, report) elif dns_type == 'cloudflare': _doh_cloudflare(job_id, analyzer_name, observable_classification, observable_name, report) elif dns_type == 'classic': _classic_dns(job_id, analyzer_name, observable_classification, observable_name, report) else: raise AnalyzerConfigurationException(f'Service selected: {dns_type} is not available') except (AnalyzerConfigurationException, AnalyzerRunException) as e: error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \ f"observable_name:{observable_name} Analyzer error {e}" logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \ f"observable_name:{observable_name} Unexpected error {e}" logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False general.set_report_and_cleanup(job_id, report) logger.info(f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}") return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: rt_value = additional_config_params.get("rt_value", "") params = {"q": observable_name} if rt_value: params["rt"] = rt_value if observable_classification == "domain": uri = "domain.php" elif observable_classification == "ip": uri = "host.php" else: raise AnalyzerRunException( "not supported observable type {}. Supported are IP and Domain" "".format(observable_classification)) try: response = requests.get(base_url + uri, params=params) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: robtex_analysis = additional_config_params.get('robtex_analysis', 'ip_query') if robtex_analysis == 'ip_query': uri = 'ipquery/{}'.format(observable_name) elif robtex_analysis == 'reverse_pdns': uri = 'pdns/reverse/{}'.format(observable_name) elif robtex_analysis == 'forward_pdns': domain = observable_name if observable_classification == 'url': domain = urlparse(observable_name).hostname uri = 'pdns/forward/{}'.format(domain) else: raise AnalyzerRunException( "not supported analysis type {}.".format(robtex_analysis)) try: response = requests.get(base_url + uri) response.raise_for_status() result = response.text.split('\r\n') except requests.ConnectionError as e: raise AnalyzerRunException("connection error: {}".format(e)) else: loaded_results = [] for item in result: if len(item) > 0: loaded_results.append(json.loads(item)) # pprint.pprint(loaded_results) report['report'] = loaded_results except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: # You should save CIRCL credentials with this template: "<user>|<pwd>" credentials = secrets.get_secret("CIRCL_CREDENTIALS") if not credentials: raise AnalyzerRunException("no credentials retrieved") split_credentials = credentials.split('|') if len(split_credentials) != 2: raise AnalyzerRunException("CIRCL credentials not properly configured." "Template to use: '<user>|<pwd>'") user = split_credentials[0] pwd = split_credentials[1] pssl = pypssl.PyPSSL(basic_auth=(user, pwd)) result = pssl.query(observable_name) certificates = [] if result.get(observable_name, {}): certificates = list(result.get(observable_name).get('certificates', [])) parsed_result = {'ip': observable_name, 'certificates': []} for cert in certificates: subject = result.get(observable_name).get('subjects', {}).get(cert, {}).get('values', []) if subject: parsed_result['certificates'].append({'fingerprint': cert, 'subject': subject[0]}) # pprint.pprint(parsed_result) report['report'] = parsed_result except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info( "started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name) ) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "") if not api_key_name: api_key_name = "AUTH0_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") headers = {"X-Auth-Token": api_key} url = "https://signals.api.auth0.com/v2.0/ip/{}".format(observable_name) response = requests.get(url, headers=headers) response.raise_for_status() json_response = response.json() # pprint.pprint(json_response) report["report"] = json_response except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e) ) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e) ) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info( "ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name) ) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info( "started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name) ) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "HUNTER_API_KEY") api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no Hunter.io API Key retrieved") url = ( "https://api.hunter.io/v2/domain-search?" f"domain={observable_name}&api_key={api_key}" ) response = requests.get(url) response.raise_for_status() json_response = response.json() report["report"] = json_response except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e) ) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e) ) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info( "ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name) ) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "") if not api_key_name: api_key_name = "ABUSEIPDB_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") headers = {"Key": api_key, "Accept": "application/json"} params = { "ipAddress": observable_name, "maxAgeInDays": 180, "verbose": True } url = "https://api.abuseipdb.com/api/v2/check" response = requests.get(url, params=params, headers=headers) response.raise_for_status() json_response = response.json() # pprint.pprint(json_response) report["report"] = json_response except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: # cuckoo installation can be with or without the api_token # it depends on version and configuration api_key_name = additional_config_params.get("api_key_name", "") if api_key_name: api_key = secrets.get_secret(api_key_name) else: api_key = None logger.info("job_id {} md5 {} analyzer {} no API key set" "".format(job_id, md5, analyzer_name)) cuckoo_url = secrets.get_secret("CUCKOO_URL") if not cuckoo_url: raise AnalyzerRunException("cuckoo URL missing") cuckoo_analysis = CuckooAnalysis(api_key, cuckoo_url) binary = general.get_binary(job_id) if not binary: raise AnalyzerRunException("is the binary empty?!") _cuckoo_scan_file(cuckoo_analysis, additional_config_params, filename, md5, binary) result = cuckoo_analysis.report # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: results = {} peepdf_analysis = [] pdf_parser = peepdf.PDFCore.PDFParser() ret, pdf = pdf_parser.parse(filepath, True) if ret: peepdf_analysis["status_code"] = ret else: stats = pdf.getStats() for version in stats.get("Versions", []): version_dict = { "events": version.get("Events", {}), "actions": version.get("Actions", {}), "urls": version.get("URLs", []), "uris": version.get("URIs", []), "elements": version.get("Elements", {}), "vulns": version.get("Vulns", []), "objects_with_js_code": version.get("Objects with JS code", []), } peepdf_analysis.append(version_dict) results["peepdf"] = peepdf_analysis # pprint.pprint(results) report["report"] = results except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e) ) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e) ) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_id_name = additional_config_params.get("api_id_name", "CENSYS_API_ID") api_secret_name = additional_config_params.get("api_secret_name", "CENSYS_API_SECRET") api_id = secrets.get_secret(api_id_name) api_secret = secrets.get_secret(api_secret_name) if not (api_id and api_secret): raise AnalyzerRunException("no api credentials retrieved") result = _censys_get_report( (api_id, api_secret), observable_name, observable_classification, additional_config_params, ) # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "") if not api_key_name: api_key_name = "VT_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") result = vt_get_report( api_key, observable_name, observable_classification, additional_config_params, job_id, ) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True # pprint.pprint(report) general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key = secrets.get_secret("GSF_KEY") if not api_key: raise AnalyzerRunException( "no api key retrieved. job_id {}, analyzer {}".format( job_id, analyzer_name)) sb_instance = SafeBrowsing(api_key) response = sb_instance.lookup_urls([observable_name]) if observable_name in response and isinstance( response[observable_name], dict): result = response[observable_name] else: raise AnalyzerRunException( "result not expected: {}".format(response)) # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: results = {} peepdf_analysis = [] pdf_parser = peepdf.PDFCore.PDFParser() ret, pdf = pdf_parser.parse(filepath, True) if ret: peepdf_analysis['status_code'] = ret else: stats = pdf.getStats() for version in stats.get('Versions', []): version_dict = { 'events': version.get('Events', {}), 'actions': version.get('Actions', {}), 'urls': version.get('URLs', []), 'uris': version.get('URIs', []), 'elements': version.get('Elements', {}), 'vulns': version.get('Vulns', []), 'objects_with_js_code': version.get('Objects with JS code', []) } peepdf_analysis.append(version_dict) results['peepdf'] = peepdf_analysis # pprint.pprint(results) report['report'] = results except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" \ "".format(job_id, analyzer_name, md5, filename, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" \ "".format(job_id, analyzer_name, md5, filename, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: if observable_classification == "hash": filehash = observable_name else: raise AnalyzerRunException( f"not supported observable type {observable_classification}." f" Supported: hash only") post_data = {"query": "get_info", "hash": filehash} url = "https://mb-api.abuse.ch/api/v1/" response = requests.post(url, data=post_data) response.raise_for_status() json_response = response.json() report["report"] = json_response except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: try: if not os.path.isfile(database_location): updater(additional_config_params) reader = maxminddb.open_database(database_location) maxmind_result = reader.get(observable_name) reader.close() except maxminddb.InvalidDatabaseError as e: error_message = "invalid database error: {}".format(e) logger.exception(error_message) maxmind_result = {"error": error_message} if not maxmind_result: maxmind_result = {} # pprint.pprint(maxmind_result) report["report"] = maxmind_result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("finished analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: results = {} results["magic"] = magic.from_file(filepath) results["mimetype"] = magic.from_file(filepath, mime=True) results["filetype"] = pyexifinfo.fileType(filepath) exif_report = pyexifinfo.get_json(filepath) if exif_report: exif_report_cleaned = { key: value for key, value in exif_report[0].items() if not (key.startswith("File") or key.startswith("SourceFile")) } results["exiftool"] = exif_report_cleaned binary = general.get_binary(job_id) results["md5"] = hashlib.md5(binary).hexdigest() results["sha1"] = hashlib.sha1(binary).hexdigest() results["sha256"] = hashlib.sha256(binary).hexdigest() results["ssdeep"] = pydeep.hash_file(filepath).decode() report["report"] = results except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "") if not api_key_name: api_key_name = "INTEZER_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") intezer_token = os.environ.get("INTEZER_TOKEN", "") intezer_token_date = os.environ.get("INTEZER_TOKEN_DATE", "") today = get_now_date_only() if not intezer_token or intezer_token_date != today: intezer_token = _get_access_token(api_key) if not intezer_token: raise AnalyzerRunException("token extraction failed") binary = general.get_binary(job_id) result = _intezer_scan_file(intezer_token, md5, filename, binary, additional_config_params) # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: result = {"found": False} if not os.path.isfile(database_location): updater() with open(database_location, "r") as f: db = f.read() db_list = db.split("\n") # pprint.pprint(db_list) if observable_name in db_list: result["found"] = True # pprint.pprint(result) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("finished analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: domain = observable_name if observable_classification == 'url': domain = urlparse(observable_name).hostname try: url = 'https://freeapi.robtex.com/pdns/forward/{}'.format(domain) response = requests.get(url) response.raise_for_status() result = response.text.split('\r\n') except requests.ConnectionError as e: raise AnalyzerRunException("connection error: {}".format(e)) else: loaded_results = [] for item in result: if len(item) > 0: loaded_results.append(json.loads(item)) # pprint.pprint(loaded_results) report['report'] = loaded_results except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get('api_key_name', 'HONEYDB_API_KEY') api_id_name = additional_config_params.get('api_id_name', 'HONEYDB_API_ID') api_key = secrets.get_secret(api_key_name) api_id = secrets.get_secret(api_id_name) if not api_key: raise AnalyzerRunException("no HoneyDB API Key retrieved") if not api_id: raise AnalyzerRunException("no HoneyDB API ID retrieved") headers = { 'X-HoneyDb-ApiKey': api_key, 'X-HoneyDb-ApiId': api_id } url = f'https://honeydb.io/api/twitter-threat-feed/{observable_name}' response = requests.get(url, headers=headers) response.raise_for_status() json_response = response.json() report['report'] = json_response except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: pattern = re.compile(r"(?:Category: )([\w\s]+)") baseurl = "https://www.fortiguard.com/webfilter?q=" url = baseurl + observable_name response = requests.get(url) response.raise_for_status() category_match = re.search(pattern, str(response.content), flags=0) dict_response = { "category": category_match.group(1) if category_match else "" } report["report"] = dict_response except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: observable_to_analyze = observable_name if observable_classification == "url": observable_to_analyze = urlparse(observable_name).hostname url = base_url + observable_to_analyze response = requests.get(url) response.raise_for_status() result = response.json() report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get('api_key_name', '') if not api_key_name: api_key_name = "SHODAN_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") params = {'key': api_key, 'minify': True} url = 'https://api.shodan.io/shodan/host/{}'.format(observable_name) response = requests.get(url, params=params) response.raise_for_status() json_response = response.json() # pprint.pprint(json_response) report['report'] = json_response except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get('api_key_name', '') if not api_key_name: api_key_name = "KEY_NAME" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") result = {} # do something here # pprint.pprint(result) report['report'] = result except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info(f"started analyzer {analyzer_name} job_id {job_id}") report = general.get_basic_report_template(analyzer_name) try: api_key_name = additional_config_params.get("api_key_name", "") if not api_key_name: api_key_name = "VT_KEY" api_key = secrets.get_secret(api_key_name) if not api_key: raise AnalyzerRunException("no api key retrieved") result = vt_scan_file(api_key, md5, job_id, additional_config_params) report["report"] = result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True # pprint.pprint(report) general.set_report_and_cleanup(job_id, report) logger.info(f"ended analyzer {analyzer_name} job_id {job_id}") return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: url = 'https://api.greynoise.io/v1/query/ip' headers = {'Content-Type': 'application/x-www-form-urlencoded'} data = {'ip': observable_name} response = requests.post(url, data=data, headers=headers) response.raise_for_status() result = response.json() # pprint.pprint(result) report['report'] = result except AnalyzerRunException as e: error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \ "".format(job_id, analyzer_name, observable_name, e) logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False else: report['success'] = True general.set_report_and_cleanup(job_id, report, logger) logger.info("ended analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info( f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}" ) report = general.get_basic_report_template(analyzer_name) try: results = {} if observable_classification != "hash": raise AnalyzerRunException( f"observable type {observable_classification} not supported" ) results["found"] = False # reference: https://team-cymru.com/community-services/mhr/ # if the resolution works, this means that the file is reported # as malware by Cymru resolutions = [] try: query_to_perform = f"{observable_name}.malware.hash.cymru.com" domains = socket.gethostbyaddr(query_to_perform) resolutions = domains[2] except (socket.gaierror, socket.herror): logger.info(f"observable {observable_name} not found in HMR DB") if resolutions: results["found"] = True results["resolution_data"] = resolutions report["report"] = results except AnalyzerRunException as e: error_message = ( f"job_id:{job_id} analyzer:{analyzer_name}" f" observable_name:{observable_name} Analyzer error {e}" ) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( f"job_id:{job_id} analyzer:{analyzer_name}" f" observable_name:{observable_name} Unexpected error {e}" ) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info( f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}" ) return report