def run(self): if self.use_proxy and not self.proxy: raise AnalyzerConfigurationException( "No proxy retrieved when use_proxy is true.") if self.output not in ["image", "json"]: raise AnalyzerConfigurationException( "output param can only be 'image' or 'json'") try: if isinstance(self.extra_api_params, dict): params = self.extra_api_params else: params = {} params["url"] = self.observable_name params["token"] = self.__api_key params["output"] = self.output if self.use_proxy: params["proxy"] = self.proxy resp = requests.get(self.base_url, params=params) resp.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) if self.output == "image": try: b64_img = base64.b64encode(resp.content).decode("utf-8") return {"screenshot": b64_img} except Exception as err: raise AnalyzerRunException( f"Failed to convert to base64 string {err}") return resp.json()
def set_config(self, additional_config_params): api_key_name = additional_config_params.get("api_key_name", "HONEYDB_API_KEY") api_id_name = additional_config_params.get("api_id_name", "HONEYDB_API_ID") self.analysis_type = additional_config_params.get( "honeydb_analysis", "all") self.endpoints = [ "scan_twitter", "ip_query", "ip_history", "internet_scanner", "ip_info", ] if self.analysis_type not in self.endpoints and self.analysis_type != "all": raise AnalyzerConfigurationException( f"analysis_type is not valid: {self.analysis_type}") self.__api_key = secrets.get_secret(api_key_name) self.__api_id = secrets.get_secret(api_id_name) if not self.__api_key: raise AnalyzerConfigurationException( "No HoneyDB API Key retrieved") if not self.__api_id: raise AnalyzerConfigurationException("No HoneyDB API ID retrieved") self.headers = { "X-HoneyDb-ApiKey": self.__api_key, "X-HoneyDb-ApiId": self.__api_id, } self.result = {}
def run(self): if not self.__api_key: raise AnalyzerConfigurationException( f"No API key retrieved with name: {self.api_key_name}." ) if not self.__api_password: raise AnalyzerConfigurationException( f"No API password retrieved with name: {self.api_password_name}." ) auth = HTTPBasicAuth(self.__api_key, self.__api_password) endpoints = self._get_endpoints() result = {} for endpoint in endpoints: try: if self.observable_classification == "url": observable_to_check = quote_plus(self.observable_name) else: observable_to_check = self.observable_name url = f"{self.base_url}/{endpoint}/{observable_to_check}" response = requests.get(url, auth=auth) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result[endpoint] = response.json() return result
def run(self): if not self.__api_key: raise AnalyzerConfigurationException( f"No API key retrieved with name: {self.api_key_name}.") if self.analysis_type == "search": params = {"key": self.__api_key, "minify": True} uri = f"shodan/host/{self.observable_name}" elif self.analysis_type == "honeyscore": params = { "key": self.__api_key, } uri = f"labs/honeyscore/{self.observable_name}" else: raise AnalyzerConfigurationException( f"analysis type: '{self.analysis_type}' not suported." "Supported are: 'search', 'honeyscore'.") try: response = requests.get(self.base_url + uri, params=params) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() if self.analysis_type == "honeyscore": return {"honeyscore": result} return result
def run(self): self.__prepare_args() try: if self.search_type == "WiFi Network": uri = f"/api/v3/detail/wifi/{self.args.get('wifiNetworkId', None)}" elif self.search_type == "CDMA Network": uri = ( f"/api/v3/detail/cell/CDMA/{self.args.get('sid', None)}/" f"{self.args.get('nid', None)}/{self.args.get('bsid', None)}" ) elif self.search_type == "Bluetooth Network": uri = f"/api/v3/detail/bt/{self.args.get('btNetworkId', None)}" elif self.search_type == "GSM/LTE/WCDMA Network": uri = ( "/api/v3/detail/cell/" f"{self.args.get('type', None)}/{self.args.get('operator', None)}/" f"{self.args.get('lac', None)}/{self.args.get('cid', None)}" ) else: raise AnalyzerConfigurationException( f"search type: '{self.search_type}' not supported." "Supported are: 'WiFi Network', 'CDMA Network', " "'Bluetooth Network', 'GSM/LTE/WCDMA Network'") response = requests.get( self.base_url + uri, headers={"Authorization": "Basic " + self.__api_key}, ) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() return result
def download_iplist(self, list_name): if ".ipset" not in list_name and ".netset" not in list_name: raise AnalyzerConfigurationException( f"extension missing from {list_name} (add .ipset or .netset to name)" ) try: iplist_location = f"{db_path}/{list_name}" data_cleaned = "" logger.info( f"starting download of {list_name} from firehol iplist") url = f"https://iplists.firehol.org/files/{list_name}" r = requests.get(url) r.raise_for_status() data_extracted = r.content.decode() for line in data_extracted.splitlines(): if not line.startswith("#"): data_cleaned += f"{line}\n" with open(iplist_location, "w") as f: f.write(data_cleaned) if not os.path.exists(iplist_location): raise AnalyzerRunException( f"failed extraction of {list_name} iplist") logger.info(f"ended download of {list_name} from firehol iplist") except Exception as e: traceback.print_exc() logger.exception(e)
def __build_zoomeye_url(self): if self.observable_classification == "ip": self.query += f" ip:{self.observable_name}" else: self.query += f" hostname:{self.observable_name}" self.search_type = "host" if self.search_type == "host" or self.search_type == "web": self.url = self.base_url + self.search_type + "/search?query=" self.url += self.query if self.page: self.url += f"&page={self.page}" if self.facets: self.url += f"&facet={','.join(self.facets)}" elif self.search_type == "both": self.url = self.base_url + "both/search?" if self.history: self.url += f"history={self.history}&" self.url += f"ip={self.observable_name}" else: raise AnalyzerConfigurationException( f"search type: '{self.search_type}' not suported." "Supported are: 'host', 'web', 'both'")
def __raise_in_case_bad_request(name, resp, params_to_check=None) -> bool: """ Raises: :class: `AnalyzerRunException`, if bad status code or no key in response """ if params_to_check is None: params_to_check = ["key"] # different error messages for different cases if resp.status_code == 404: raise AnalyzerConfigurationException( f"{name} docker container is not running." ) if resp.status_code == 400: err = resp.json().get("error", "") raise AnalyzerRunException(err) if resp.status_code == 500: raise AnalyzerRunException( f"Internal Server Error in {name} docker container" ) # check to make sure there was a valid params in response for param in params_to_check: param_value = resp.json().get(param, None) if not param_value: raise AnalyzerRunException( "Unexpected Error. " f"Please check log files under /var/log/intel_owl/{name.lower()}/" ) # just in case couldn't catch the error manually resp.raise_for_status() return True
def run(self): ip = self.observable_name result = {} if not self.list_names: raise AnalyzerConfigurationException( "list_names is empty in custom analyzer config, add an iplist") for list_name in self.list_names: result[list_name] = False self.check_iplist_status(list_name) with open(f"{db_path}/{list_name}", "r") as f: db = f.read() db_list = db.split("\n") for ip_or_subnet in db_list: if ip_or_subnet != "": if ipaddress.ip_address(ip) in ipaddress.ip_network( ip_or_subnet): result[list_name] = True break return result
def _docker_run(self, req_data, req_files=None): """ Helper function that takes of care of requesting new analysis, reading response, polling for result and exception handling for a docker based analyzer. Args: req_data (Dict): Dict of request JSON. req_files (Dict, optional): Dict of files to send. Defaults to None. Raises: AnalyzerConfigurationException: In case docker service is not running AnalyzerRunException: Any other error Returns: Dict: Final analysis results """ # handle in case this is a test if hasattr(self, "is_test") and getattr(self, "is_test"): # only happens in case of testing self.report["success"] = True return {} # step #1: request new analysis args = req_data.get("args", []) logger.debug(f"Making request with arguments: {args} <- {self.__repr__()}") try: if req_files: form_data = {"request_json": json.dumps(req_data)} resp1 = requests.post(self.url, files=req_files, data=form_data) else: resp1 = requests.post(self.url, json=req_data) except requests.exceptions.ConnectionError: raise AnalyzerConfigurationException( f"{self.name} docker container is not running." ) # step #2: raise AnalyzerRunException in case of error assert self.__raise_in_case_bad_request(self.name, resp1) # step #3: if no error, continue and try to fetch result key = resp1.json().get("key") final_resp = self.__poll_for_result(key) err = final_resp.get("error", None) report = final_resp.get("report", None) if not report: raise AnalyzerRunException(f"Report is empty. Reason: {err}") if isinstance(report, dict): return report try: report = json.loads(report) except json.JSONDecodeError: raise AnalyzerRunException(str(err)) return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params): """Run ActiveDNS analyzer Admit: * additional_config_params[service]: google - Google DoH (DNS over HTTPS) * additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS) * additional_config_params[service]: classic - classic DNS query Google and CloudFlare return an IP (or NXDOMAIN) from a domain. Classic support also reverse lookup (domain from IP) :param analyzer_name: Analyzer configuration in analyzer_config.json :type analyzer_name: str :param job_id: job identifier :type job_id: str :param observable_name: analyzed observable :type observable_name: str :param observable_classification: observable classification (allow: ip or domain) ip only classic :type observable_classification: str :param additional_config_params: params service to select the service :type additional_config_params: dict :return: report: name: observable_name, resolution: ip,NXDOMAIN, '' :rtype: report: dict """ logger.info(f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}") report = general.get_basic_report_template(analyzer_name) try: dns_type = additional_config_params.get('service', '') if dns_type == 'google': _doh_google(job_id, analyzer_name, observable_classification, observable_name, report) elif dns_type == 'cloudflare': _doh_cloudflare(job_id, analyzer_name, observable_classification, observable_name, report) elif dns_type == 'classic': _classic_dns(job_id, analyzer_name, observable_classification, observable_name, report) else: raise AnalyzerConfigurationException(f'Service selected: {dns_type} is not available') except (AnalyzerConfigurationException, AnalyzerRunException) as e: error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \ f"observable_name:{observable_name} Analyzer error {e}" logger.error(error_message) report['errors'].append(error_message) report['success'] = False except Exception as e: traceback.print_exc() error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \ f"observable_name:{observable_name} Unexpected error {e}" logger.exception(error_message) report['errors'].append(str(e)) report['success'] = False general.set_report_and_cleanup(job_id, report) logger.info(f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}") return report
def run(self): misp_instance = pymisp.PyMISP( url=self.__url_name, key=self.__api_key, ssl=self.ssl_check, debug=self.debug, timeout=5, ) now = datetime.datetime.now() date_from = now - datetime.timedelta(days=self.from_days) params = { "limit": self.limit, "enforce_warninglist": self.enforce_warninglist, } if self.strict_search: params["value"] = self.observable_name else: string_wildcard = f"%{self.observable_name}%" params["searchall"] = string_wildcard if self.from_days != 0: params["date_from"] = date_from.strftime("%Y-%m-%d %H:%M:%S") if self.filter_on_type: params["type_attribute"] = [self.observable_classification] if self.observable_classification == self.ObservableTypes.HASH: params["type_attribute"] = ["md5", "sha1", "sha256"] if self.observable_classification == self.ObservableTypes.IP: params["type_attribute"] = [ "ip-dst", "ip-src", "ip-src|port", "ip-dst|port", "domain|ip", ] elif self.observable_classification == self.ObservableTypes.DOMAIN: params["type_attribute"] = [ self.observable_classification, "domain|ip" ] elif self.observable_classification == self.ObservableTypes.HASH: params["type_attribute"] = ["md5", "sha1", "sha256"] elif self.observable_classification == self.ObservableTypes.URL: params["type_attribute"] = [self.observable_classification] elif self.observable_classification == self.ObservableTypes.GENERIC: pass else: raise AnalyzerConfigurationException( f"Observable {self.observable_classification} not supported." "Currently supported are: ip, domain, hash, url, generic.") result_search = misp_instance.search(**params) if isinstance(result_search, dict): errors = result_search.get("errors", []) if errors: raise AnalyzerRunException(errors) return { "result_search": result_search, "instance_url": self.__url_name }
def run(self): params = {} uri = "" if self.analysis_type == "intelligence": self.active = None if self.must_active: self.active = "active" params = { "value__contains": self.observable_name, "limit": self.limit, "status": self.active, "confidence__gt": self.minimal_confidence, "modified_ts__gte": self.modified_after, } # If value = None don't enter in filter uri = "v2/intelligence/" elif self.analysis_type == "confidence": params = {"type": "confidence", "value": self.observable_name} uri = "v1/inteldetails/confidence_trend/" elif self.analysis_type == "passive_dns": if self.observable_classification == self.ObservableTypes.IP: uri = f"v1/pdns/ip/{self.observable_name}" elif self.observable_classification == self.ObservableTypes.DOMAIN: uri = f"v1/pdns/domain/{self.observable_name}" else: raise AnalyzerConfigurationException( f"Observable {self.observable_classification} not supported." "Currently supported are: ip, domain." ) else: raise AnalyzerConfigurationException( f"Analysis type: {self.analysis_type} not supported." "Currently supported are: intelligence, confidence,passive_dns." ) try: api_header = {"Authorization": f"apikey {self.__api_user}:{self.__api_key}"} response = requests.get( self.base_url + uri, params=params, headers=api_header ) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() return result
def run(self): if self.analysis_type == "search": self.__triage_search() elif self.analysis_type == "submit": self.__triage_submit() else: raise AnalyzerConfigurationException( f"analysis type '{self.analysis_type}' not supported." "Supported are: 'search', 'submit'.") return self.final_report
def run(self): if self.dns_type == "google": return self.__doh_google() if self.dns_type == "cloudflare": return self.__doh_cloudflare() if self.dns_type == "cloudflare_malware": return self.__doh_cloudflare_malware() if self.dns_type == "classic": return self.__classic_dns() raise AnalyzerConfigurationException( f"Service selected: {self.dns_type} is not available")
def run(self): if not self.cuckoo_url: raise AnalyzerConfigurationException("cuckoo URL missing") binary = self.read_file_bytes() if not binary: raise AnalyzerRunException("is the binary empty?!") self.__cuckoo_request_scan(binary) self.__cuckoo_poll_result() result = self.__cuckoo_retrieve_and_create_report() return result
def set_params(self, params): self.endpoint = params.get("endpoint", "public") if self.endpoint == "private": self.base_url = self.private_url self.__api_key = self._secrets["api_key_name"] self.report_type = params.get("report_type", "overview") if self.report_type not in ["overview", "complete"]: raise AnalyzerConfigurationException( f"report_type must be 'overview' or 'complete' " f"but it is '{self.report_type}'") self.max_tries = params.get("max_tries", 200) self.poll_distance = 3
def set_params(self, params): self._query_type = params.get("query_type", "phonebook") if self._query_type not in ["phonebook", "intelligent"]: raise AnalyzerConfigurationException( f"{self._query_type} not supported") self.url = self.base_url + f"/{self._query_type}/search" self._rows_limit = int(params.get("rows_limit", 1000)) self._max_tries = int(params.get("max_tries", 10)) self._poll_distance = int(params.get("poll_distance", 3)) self._timeout = int(params.get("timeout", 10)) self._datefrom = params.get("datefrom", "") self._dateto = params.get("dateto", "") self.__api_key = self._secrets["api_key_name"]
def run(self): self.session = requests.Session() self.session.headers = {"Authorization": f"Bearer {self.__api_key}"} if self.analysis_type == "search": response = self.__triage_search() elif self.analysis_type == "submit": response = self.__triage_submit() else: raise AnalyzerConfigurationException( f"analysis type '{self.analysis_type}' not supported." "Supported are: 'search', 'submit'.") return response
def run(self): if not self.__api_key: raise AnalyzerConfigurationException( f"No API key retrieved with name: {self.api_key_name}.") try: response = requests.get( self.base_url + self.observable_name, params={"token": self.__api_key}, ) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() return result
def set_params(self, params): self.analysis_type = params.get("honeydb_analysis", "all") self.endpoints = [ "scan_twitter", "ip_query", "ip_history", "internet_scanner", "ip_info", ] if self.analysis_type not in self.endpoints and self.analysis_type != "all": raise AnalyzerConfigurationException( f"analysis_type is not valid: {self.analysis_type}") # set secrets self.__api_key = self._secrets["api_key_name"] self.__api_id = self._secrets["api_id_name"] self.headers = { "X-HoneyDb-ApiKey": self.__api_key, "X-HoneyDb-ApiId": self.__api_id, } self.result = {}
def run(self): self.__auth = secrets.get_secret(self.api_key_name) if not self.__auth: raise AnalyzerConfigurationException( f"No secret retrieved for `{self.api_key_name}`") # try to identify search operator operator = self.__identify_search_operator() if operator: value = f"{operator}:{self.observable_name}" else: # if operator couldn't be identified, we can query without it value = self.observable_name # execute searches entries = self.__search(value) return { "query_value": value, "pages_queried": self.pages, "entries": entries, }
def run(self): api_key = secrets.get_secret(self._api_key_name) if not api_key: raise AnalyzerConfigurationException( f"No API key retrieved with name: '{self._api_key_name}'") session = requests.Session() session.headers.update({ "x-key": api_key, "User-Agent": "IntelOwl/v1.x" }) params = { "term": self.observable_name, "buckets": [], "lookuplevel": 0, "maxresults": self._rows_limit, "timeout": 10, "sort": 4, "media": 0, "terminate": [], "target": 0, } # POST the search term --> Fetch the 'id' --> GET the results using the 'id' r = session.post(self.base_url, json=params) r.raise_for_status() search_id = r.json().get("id", None) if not search_id: raise AnalyzerRunException( f"Failed to request search. Status code: {r.status_code}.") time.sleep(15) r = session.get( f"{self.base_url}/result?id={search_id}&limit={self._rows_limit}&offset=-1" ) r.raise_for_status() selectors = r.json()["selectors"] parsed_selectors = self.__pb_search_results(selectors) return {"id": search_id, **parsed_selectors}
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): """Run ActiveDNS analyzer Admit: * additional_config_params[service]: google - Google DoH (DNS over HTTPS) * additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS) * additional_config_params[service]: cloudflare_malware - CloudFlare DoH (DNS over HTTPS) with Malware filtering * additional_config_params[service]: classic - classic DNS query Google and CloudFlare return an IP (or NXDOMAIN) from a domain. Classic support also reverse lookup (domain from IP) :param analyzer_name: str Analyzer configuration in analyzer_config.json :param job_id: str job identifier :param observable_name: str analyzed observable :param observable_classification: str observable classification (allow: ip or domain) ip only classic :param additional_config_params: dict params service to select the service :return: report: dict name: observable_name, resolution: ip,NXDOMAIN, '' """ logger.info( f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}" ) report = general.get_basic_report_template(analyzer_name) try: dns_type = additional_config_params.get("service", "") if dns_type == "google": _doh_google( job_id, analyzer_name, observable_classification, observable_name, report, ) elif dns_type == "cloudflare": _doh_cloudflare( job_id, analyzer_name, observable_classification, observable_name, report, ) elif dns_type == "cloudflare_malware": _doh_cloudflare_malware( job_id, analyzer_name, observable_classification, observable_name, report, ) elif dns_type == "classic": _classic_dns( job_id, analyzer_name, observable_classification, observable_name, report, ) else: raise AnalyzerConfigurationException( f"Service selected: {dns_type} is not available") except (AnalyzerConfigurationException, AnalyzerRunException) as e: error_message = ( f"job_id:{job_id} analyzer:{analyzer_name} " f"observable_name:{observable_name} Analyzer error {e}") logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( f"job_id:{job_id} analyzer:{analyzer_name} " f"observable_name:{observable_name} Unexpected error {e}") logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info( f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}" ) return report
def start_analyzers(analyzers_to_execute, analyzers_config, job_id, md5, is_sample): set_job_status(job_id, "running") if is_sample: file_path, filename = get_filepath_filename(job_id) else: observable_name, observable_classification = get_observable_data(job_id) for analyzer in analyzers_to_execute: try: analyzer_module = analyzers_config[analyzer].get("python_module", "") if not analyzer_module: message = ( f"no python_module available in config for {analyzer} analyzer?!" ) raise AnalyzerConfigurationException(message) additional_config_params = analyzers_config[analyzer].get( "additional_config_params", {} ) # run analyzer with a celery task asynchronously if is_sample: # check if we should run the hash instead of the binary run_hash = analyzers_config[analyzer].get("run_hash", "") if run_hash: # check which kind of hash the analyzer needs run_hash_type = analyzers_config[analyzer].get( "run_hash_type", "md5" ) if run_hash_type == "md5": hash_value = md5 elif run_hash_type == "sha256": hash_value = generate_sha256(job_id) else: error_message = ( f"only md5 and sha256 are supported " f"but you asked {run_hash_type}. job_id: {job_id}" ) raise AnalyzerConfigurationException(error_message) # run the analyzer with the hash args = [ analyzer, job_id, hash_value, "hash", additional_config_params, ] getattr(tasks, analyzer_module).apply_async( args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE ) else: # run the analyzer with the binary args = [ analyzer, job_id, file_path, filename, md5, additional_config_params, ] getattr(tasks, analyzer_module).apply_async( args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE ) else: # observables analyzer case args = [ analyzer, job_id, observable_name, observable_classification, additional_config_params, ] getattr(tasks, analyzer_module).apply_async( args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE ) except (AnalyzerConfigurationException, AnalyzerRunException) as e: error_message = "job_id {}. analyzer: {}. error: {}".format( job_id, analyzer, e ) logger.error(error_message) set_failed_analyzer(analyzer, job_id, error_message)
def start_analyzers( analyzers_to_execute, analyzers_config, runtime_configuration, job_id, md5, is_sample, ): set_job_status(job_id, "running") if is_sample: file_path, filename = get_filepath_filename(job_id) else: observable_name, observable_classification = get_observable_data( job_id) for analyzer in analyzers_to_execute: ac = analyzers_config[analyzer] try: module = ac.get("python_module", None) if not module: raise AnalyzerConfigurationException( f"no python_module available in config for {analyzer} analyzer?!" ) additional_config_params = ac.get("additional_config_params", {}) adjust_analyzer_config(runtime_configuration, additional_config_params, analyzer, job_id, md5) # get celery queue queue = ac.get("queue", "default") if queue not in CELERY_QUEUES: logger.error( f"Analyzer {analyzers_to_execute} has a wrong queue." f" Setting to default") queue = "default" # construct arguments if is_sample: # check if we should run the hash instead of the binary run_hash = ac.get("run_hash", False) if run_hash: # check which kind of hash the analyzer needs run_hash_type = ac.get("run_hash_type", "md5") if run_hash_type == "md5": hash_value = md5 elif run_hash_type == "sha256": hash_value = generate_sha256(job_id) else: error_message = ( f"only md5 and sha256 are supported " f"but you asked {run_hash_type}. job_id: {job_id}") raise AnalyzerConfigurationException(error_message) # run the analyzer with the hash args = [ f"observable_analyzers.{module}", analyzer, job_id, hash_value, "hash", additional_config_params, ] else: # run the analyzer with the binary args = [ f"file_analyzers.{module}", analyzer, job_id, file_path, filename, md5, additional_config_params, ] else: # observables analyzer case args = [ f"observable_analyzers.{module}", analyzer, job_id, observable_name, observable_classification, additional_config_params, ] # run analyzer with a celery task asynchronously stl = ac.get("soft_time_limit", 300) t_id = uuid() celery_app.send_task( "run_analyzer", args=args, queue=queue, soft_time_limit=stl, task_id=t_id, ) # to track task_id by job_id task_ids = cache.get(job_id) if isinstance(task_ids, list): task_ids.append(t_id) else: task_ids = [t_id] cache.set(job_id, task_ids) except (AnalyzerConfigurationException, AnalyzerRunException) as e: err_msg = f"({analyzer}, job_id #{job_id}) -> Error: {e}" logger.error(err_msg) set_failed_analyzer(analyzer, job_id, err_msg)
def start_analyzers( analyzers_to_execute, analyzers_config, runtime_configuration, job_id, md5, is_sample, ): set_job_status(job_id, "running") if is_sample: file_path, filename = get_filepath_filename(job_id) else: observable_name, observable_classification = get_observable_data( job_id) for analyzer in analyzers_to_execute: ac = analyzers_config[analyzer] try: module = ac.get("python_module", None) if not module: raise AnalyzerConfigurationException( f"no python_module available in config for {analyzer} analyzer?!" ) additional_config_params = ac.get("additional_config_params", {}) adjust_analyzer_config(runtime_configuration, additional_config_params, analyzer) # construct arguments if is_sample: # check if we should run the hash instead of the binary run_hash = ac.get("run_hash", False) if run_hash: # check which kind of hash the analyzer needs run_hash_type = ac.get("run_hash_type", "md5") if run_hash_type == "md5": hash_value = md5 elif run_hash_type == "sha256": hash_value = generate_sha256(job_id) else: error_message = ( f"only md5 and sha256 are supported " f"but you asked {run_hash_type}. job_id: {job_id}") raise AnalyzerConfigurationException(error_message) # run the analyzer with the hash args = [ analyzer, job_id, hash_value, "hash", additional_config_params, ] else: # run the analyzer with the binary args = [ analyzer, job_id, file_path, filename, md5, additional_config_params, ] else: # observables analyzer case args = [ analyzer, job_id, observable_name, observable_classification, additional_config_params, ] # run analyzer with a celery task asynchronously tasks.analyzer_run.apply_async( args=[ac["type"], module, *args], queue=settings.CELERY_TASK_DEFAULT_QUEUE, ) except (AnalyzerConfigurationException, AnalyzerRunException) as e: error_message = f"job_id {job_id}. analyzer: {analyzer}. error: {e}" logger.error(error_message) set_failed_analyzer(analyzer, job_id, error_message)
def _raise_container_not_running(self) -> None: raise AnalyzerConfigurationException( f"{self.name} docker container is not running.\n" f"You have to enable it using the appropriate " f"parameter when executing start.py." )
def run(self): if not self.__api_key: raise AnalyzerConfigurationException( f"No API key retrieved with name: {self.api_key_name}.") return self._scan_binary()
def run(self): result = {} headers = {"Content-Type": "application/json"} if self.__api_key: headers["Authorization"] = self.__api_key else: warning = f"No API key retrieved with name: {self.api_key_name}" logger.info( f"{warning}. Continuing without API key..." f" <- {self.__repr__()}" ) self.report["errors"].append(warning) if self.analysis_type == "dfi_search": if self.observable_classification == "hash": uri = ( f"/api/dfi/search/hash/{self.hash_type}?hash={self.observable_name}" ) elif self.observable_classification in ["ip", "url", "domain"]: uri = ( f"/api/dfi/search/ioc/{self.observable_classification}" f"?keyword={self.observable_name}" ) elif self.observable_classification == "generic": try: type_, value = self.observable_name.split(":") except ValueError: self.generic_identifier_mode = "auto" type_ = self.type_of_generic() value = self.observable_name if type_ not in ["email", "filename", "registry", "xmpid"]: raise AnalyzerRunException(f"Unknown Type: {type_}") uri = f"/api/dfi/search/ioc/{type_}?keyword={value}" else: raise AnalyzerRunException() elif self.analysis_type == "iocdb_search": uri = f"/api/iocdb/search?keyword={self.observable_name}" elif self.analysis_type == "repdb_search": uri = f"/api/repdb/search?keyword={self.observable_name}" else: raise AnalyzerConfigurationException( f"analysis type: '{self.analysis_type}' not suported." "Supported are: 'dfi_search', 'iocdb_search', 'repdb_search'." ) try: response = requests.get(self.base_url + uri, headers=headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) result = response.json() if ( self.analysis_type == "dfi_search" and self.observable_classification == "hash" ): result["hash_type"] = self.hash_type if self.generic_identifier_mode == "auto": result["type_of_generic"] = self.type_of_generic() return result