def run(self):
if self.use_proxy and not self.proxy:
raise AnalyzerConfigurationException(
"No proxy retrieved when use_proxy is true.")
if self.output not in ["image", "json"]:
raise AnalyzerConfigurationException(
"output param can only be 'image' or 'json'")
try:
if isinstance(self.extra_api_params, dict):
params = self.extra_api_params
else:
params = {}
params["url"] = self.observable_name
params["token"] = self.__api_key
params["output"] = self.output
if self.use_proxy:
params["proxy"] = self.proxy
resp = requests.get(self.base_url, params=params)
resp.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
if self.output == "image":
try:
b64_img = base64.b64encode(resp.content).decode("utf-8")
return {"screenshot": b64_img}
except Exception as err:
raise AnalyzerRunException(
f"Failed to convert to base64 string {err}")
return resp.json()
def set_config(self, additional_config_params):
api_key_name = additional_config_params.get("api_key_name",
"HONEYDB_API_KEY")
api_id_name = additional_config_params.get("api_id_name",
"HONEYDB_API_ID")
self.analysis_type = additional_config_params.get(
"honeydb_analysis", "all")
self.endpoints = [
"scan_twitter",
"ip_query",
"ip_history",
"internet_scanner",
"ip_info",
]
if self.analysis_type not in self.endpoints and self.analysis_type != "all":
raise AnalyzerConfigurationException(
f"analysis_type is not valid: {self.analysis_type}")
self.__api_key = secrets.get_secret(api_key_name)
self.__api_id = secrets.get_secret(api_id_name)
if not self.__api_key:
raise AnalyzerConfigurationException(
"No HoneyDB API Key retrieved")
if not self.__api_id:
raise AnalyzerConfigurationException("No HoneyDB API ID retrieved")
self.headers = {
"X-HoneyDb-ApiKey": self.__api_key,
"X-HoneyDb-ApiId": self.__api_id,
}
self.result = {}
def run(self):
if not self.__api_key:
raise AnalyzerConfigurationException(
f"No API key retrieved with name: {self.api_key_name}."
)
if not self.__api_password:
raise AnalyzerConfigurationException(
f"No API password retrieved with name: {self.api_password_name}."
)
auth = HTTPBasicAuth(self.__api_key, self.__api_password)
endpoints = self._get_endpoints()
result = {}
for endpoint in endpoints:
try:
if self.observable_classification == "url":
observable_to_check = quote_plus(self.observable_name)
else:
observable_to_check = self.observable_name
url = f"{self.base_url}/{endpoint}/{observable_to_check}"
response = requests.get(url, auth=auth)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result[endpoint] = response.json()
return result
def run(self):
if not self.__api_key:
raise AnalyzerConfigurationException(
f"No API key retrieved with name: {self.api_key_name}.")
if self.analysis_type == "search":
params = {"key": self.__api_key, "minify": True}
uri = f"shodan/host/{self.observable_name}"
elif self.analysis_type == "honeyscore":
params = {
"key": self.__api_key,
}
uri = f"labs/honeyscore/{self.observable_name}"
else:
raise AnalyzerConfigurationException(
f"analysis type: '{self.analysis_type}' not suported."
"Supported are: 'search', 'honeyscore'.")
try:
response = requests.get(self.base_url + uri, params=params)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
if self.analysis_type == "honeyscore":
return {"honeyscore": result}
return result
def run(self):
self.__prepare_args()
try:
if self.search_type == "WiFi Network":
uri = f"/api/v3/detail/wifi/{self.args.get('wifiNetworkId', None)}"
elif self.search_type == "CDMA Network":
uri = (
f"/api/v3/detail/cell/CDMA/{self.args.get('sid', None)}/"
f"{self.args.get('nid', None)}/{self.args.get('bsid', None)}"
)
elif self.search_type == "Bluetooth Network":
uri = f"/api/v3/detail/bt/{self.args.get('btNetworkId', None)}"
elif self.search_type == "GSM/LTE/WCDMA Network":
uri = (
"/api/v3/detail/cell/"
f"{self.args.get('type', None)}/{self.args.get('operator', None)}/"
f"{self.args.get('lac', None)}/{self.args.get('cid', None)}"
)
else:
raise AnalyzerConfigurationException(
f"search type: '{self.search_type}' not supported."
"Supported are: 'WiFi Network', 'CDMA Network', "
"'Bluetooth Network', 'GSM/LTE/WCDMA Network'")
response = requests.get(
self.base_url + uri,
headers={"Authorization": "Basic " + self.__api_key},
)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
return result
def download_iplist(self, list_name):
if ".ipset" not in list_name and ".netset" not in list_name:
raise AnalyzerConfigurationException(
f"extension missing from {list_name} (add .ipset or .netset to name)"
)
try:
iplist_location = f"{db_path}/{list_name}"
data_cleaned = ""
logger.info(
f"starting download of {list_name} from firehol iplist")
url = f"https://iplists.firehol.org/files/{list_name}"
r = requests.get(url)
r.raise_for_status()
data_extracted = r.content.decode()
for line in data_extracted.splitlines():
if not line.startswith("#"):
data_cleaned += f"{line}\n"
with open(iplist_location, "w") as f:
f.write(data_cleaned)
if not os.path.exists(iplist_location):
raise AnalyzerRunException(
f"failed extraction of {list_name} iplist")
logger.info(f"ended download of {list_name} from firehol iplist")
except Exception as e:
traceback.print_exc()
logger.exception(e)
def __build_zoomeye_url(self):
if self.observable_classification == "ip":
self.query += f" ip:{self.observable_name}"
else:
self.query += f" hostname:{self.observable_name}"
self.search_type = "host"
if self.search_type == "host" or self.search_type == "web":
self.url = self.base_url + self.search_type + "/search?query="
self.url += self.query
if self.page:
self.url += f"&page={self.page}"
if self.facets:
self.url += f"&facet={','.join(self.facets)}"
elif self.search_type == "both":
self.url = self.base_url + "both/search?"
if self.history:
self.url += f"history={self.history}&"
self.url += f"ip={self.observable_name}"
else:
raise AnalyzerConfigurationException(
f"search type: '{self.search_type}' not suported."
"Supported are: 'host', 'web', 'both'")
def __raise_in_case_bad_request(name, resp, params_to_check=None) -> bool:
"""
Raises:
:class: `AnalyzerRunException`, if bad status code or no key in response
"""
if params_to_check is None:
params_to_check = ["key"]
# different error messages for different cases
if resp.status_code == 404:
raise AnalyzerConfigurationException(
f"{name} docker container is not running."
)
if resp.status_code == 400:
err = resp.json().get("error", "")
raise AnalyzerRunException(err)
if resp.status_code == 500:
raise AnalyzerRunException(
f"Internal Server Error in {name} docker container"
)
# check to make sure there was a valid params in response
for param in params_to_check:
param_value = resp.json().get(param, None)
if not param_value:
raise AnalyzerRunException(
"Unexpected Error. "
f"Please check log files under /var/log/intel_owl/{name.lower()}/"
)
# just in case couldn't catch the error manually
resp.raise_for_status()
return True
def run(self):
ip = self.observable_name
result = {}
if not self.list_names:
raise AnalyzerConfigurationException(
"list_names is empty in custom analyzer config, add an iplist")
for list_name in self.list_names:
result[list_name] = False
self.check_iplist_status(list_name)
with open(f"{db_path}/{list_name}", "r") as f:
db = f.read()
db_list = db.split("\n")
for ip_or_subnet in db_list:
if ip_or_subnet != "":
if ipaddress.ip_address(ip) in ipaddress.ip_network(
ip_or_subnet):
result[list_name] = True
break
return result
def _docker_run(self, req_data, req_files=None):
"""
Helper function that takes of care of requesting new analysis,
reading response, polling for result and exception handling for a
docker based analyzer.
Args:
req_data (Dict): Dict of request JSON.
req_files (Dict, optional): Dict of files to send. Defaults to None.
Raises:
AnalyzerConfigurationException: In case docker service is not running
AnalyzerRunException: Any other error
Returns:
Dict: Final analysis results
"""
# handle in case this is a test
if hasattr(self, "is_test") and getattr(self, "is_test"):
# only happens in case of testing
self.report["success"] = True
return {}
# step #1: request new analysis
args = req_data.get("args", [])
logger.debug(f"Making request with arguments: {args} <- {self.__repr__()}")
try:
if req_files:
form_data = {"request_json": json.dumps(req_data)}
resp1 = requests.post(self.url, files=req_files, data=form_data)
else:
resp1 = requests.post(self.url, json=req_data)
except requests.exceptions.ConnectionError:
raise AnalyzerConfigurationException(
f"{self.name} docker container is not running."
)
# step #2: raise AnalyzerRunException in case of error
assert self.__raise_in_case_bad_request(self.name, resp1)
# step #3: if no error, continue and try to fetch result
key = resp1.json().get("key")
final_resp = self.__poll_for_result(key)
err = final_resp.get("error", None)
report = final_resp.get("report", None)
if not report:
raise AnalyzerRunException(f"Report is empty. Reason: {err}")
if isinstance(report, dict):
return report
try:
report = json.loads(report)
except json.JSONDecodeError:
raise AnalyzerRunException(str(err))
return report
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
"""Run ActiveDNS analyzer
Admit:
* additional_config_params[service]: google - Google DoH (DNS over HTTPS)
* additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS)
* additional_config_params[service]: classic - classic DNS query
Google and CloudFlare return an IP (or NXDOMAIN) from a domain.
Classic support also reverse lookup (domain from IP)
:param analyzer_name: Analyzer configuration in analyzer_config.json
:type analyzer_name: str
:param job_id: job identifier
:type job_id: str
:param observable_name: analyzed observable
:type observable_name: str
:param observable_classification: observable classification (allow: ip or domain) ip only classic
:type observable_classification: str
:param additional_config_params: params service to select the service
:type additional_config_params: dict
:return: report: name: observable_name, resolution: ip,NXDOMAIN, ''
:rtype: report: dict
"""
logger.info(f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}")
report = general.get_basic_report_template(analyzer_name)
try:
dns_type = additional_config_params.get('service', '')
if dns_type == 'google':
_doh_google(job_id, analyzer_name, observable_classification, observable_name, report)
elif dns_type == 'cloudflare':
_doh_cloudflare(job_id, analyzer_name, observable_classification, observable_name,
report)
elif dns_type == 'classic':
_classic_dns(job_id, analyzer_name, observable_classification, observable_name, report)
else:
raise AnalyzerConfigurationException(f'Service selected: {dns_type} is not available')
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \
f"observable_name:{observable_name} Analyzer error {e}"
logger.error(error_message)
report['errors'].append(error_message)
report['success'] = False
except Exception as e:
traceback.print_exc()
error_message = f"job_id:{job_id} analyzer:{analyzer_name} " \
f"observable_name:{observable_name} Unexpected error {e}"
logger.exception(error_message)
report['errors'].append(str(e))
report['success'] = False
general.set_report_and_cleanup(job_id, report)
logger.info(f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}")
return report
def run(self):
misp_instance = pymisp.PyMISP(
url=self.__url_name,
key=self.__api_key,
ssl=self.ssl_check,
debug=self.debug,
timeout=5,
)
now = datetime.datetime.now()
date_from = now - datetime.timedelta(days=self.from_days)
params = {
"limit": self.limit,
"enforce_warninglist": self.enforce_warninglist,
}
if self.strict_search:
params["value"] = self.observable_name
else:
string_wildcard = f"%{self.observable_name}%"
params["searchall"] = string_wildcard
if self.from_days != 0:
params["date_from"] = date_from.strftime("%Y-%m-%d %H:%M:%S")
if self.filter_on_type:
params["type_attribute"] = [self.observable_classification]
if self.observable_classification == self.ObservableTypes.HASH:
params["type_attribute"] = ["md5", "sha1", "sha256"]
if self.observable_classification == self.ObservableTypes.IP:
params["type_attribute"] = [
"ip-dst",
"ip-src",
"ip-src|port",
"ip-dst|port",
"domain|ip",
]
elif self.observable_classification == self.ObservableTypes.DOMAIN:
params["type_attribute"] = [
self.observable_classification, "domain|ip"
]
elif self.observable_classification == self.ObservableTypes.HASH:
params["type_attribute"] = ["md5", "sha1", "sha256"]
elif self.observable_classification == self.ObservableTypes.URL:
params["type_attribute"] = [self.observable_classification]
elif self.observable_classification == self.ObservableTypes.GENERIC:
pass
else:
raise AnalyzerConfigurationException(
f"Observable {self.observable_classification} not supported."
"Currently supported are: ip, domain, hash, url, generic.")
result_search = misp_instance.search(**params)
if isinstance(result_search, dict):
errors = result_search.get("errors", [])
if errors:
raise AnalyzerRunException(errors)
return {
"result_search": result_search,
"instance_url": self.__url_name
}
def run(self):
params = {}
uri = ""
if self.analysis_type == "intelligence":
self.active = None
if self.must_active:
self.active = "active"
params = {
"value__contains": self.observable_name,
"limit": self.limit,
"status": self.active,
"confidence__gt": self.minimal_confidence,
"modified_ts__gte": self.modified_after,
} # If value = None don't enter in filter
uri = "v2/intelligence/"
elif self.analysis_type == "confidence":
params = {"type": "confidence", "value": self.observable_name}
uri = "v1/inteldetails/confidence_trend/"
elif self.analysis_type == "passive_dns":
if self.observable_classification == self.ObservableTypes.IP:
uri = f"v1/pdns/ip/{self.observable_name}"
elif self.observable_classification == self.ObservableTypes.DOMAIN:
uri = f"v1/pdns/domain/{self.observable_name}"
else:
raise AnalyzerConfigurationException(
f"Observable {self.observable_classification} not supported."
"Currently supported are: ip, domain."
)
else:
raise AnalyzerConfigurationException(
f"Analysis type: {self.analysis_type} not supported."
"Currently supported are: intelligence, confidence,passive_dns."
)
try:
api_header = {"Authorization": f"apikey {self.__api_user}:{self.__api_key}"}
response = requests.get(
self.base_url + uri, params=params, headers=api_header
)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
return result
def run(self):
if self.analysis_type == "search":
self.__triage_search()
elif self.analysis_type == "submit":
self.__triage_submit()
else:
raise AnalyzerConfigurationException(
f"analysis type '{self.analysis_type}' not supported."
"Supported are: 'search', 'submit'.")
return self.final_report
def run(self):
if self.dns_type == "google":
return self.__doh_google()
if self.dns_type == "cloudflare":
return self.__doh_cloudflare()
if self.dns_type == "cloudflare_malware":
return self.__doh_cloudflare_malware()
if self.dns_type == "classic":
return self.__classic_dns()
raise AnalyzerConfigurationException(
f"Service selected: {self.dns_type} is not available")
def run(self):
if not self.cuckoo_url:
raise AnalyzerConfigurationException("cuckoo URL missing")
binary = self.read_file_bytes()
if not binary:
raise AnalyzerRunException("is the binary empty?!")
self.__cuckoo_request_scan(binary)
self.__cuckoo_poll_result()
result = self.__cuckoo_retrieve_and_create_report()
return result
def set_params(self, params):
self.endpoint = params.get("endpoint", "public")
if self.endpoint == "private":
self.base_url = self.private_url
self.__api_key = self._secrets["api_key_name"]
self.report_type = params.get("report_type", "overview")
if self.report_type not in ["overview", "complete"]:
raise AnalyzerConfigurationException(
f"report_type must be 'overview' or 'complete' "
f"but it is '{self.report_type}'")
self.max_tries = params.get("max_tries", 200)
self.poll_distance = 3
def set_params(self, params):
self._query_type = params.get("query_type", "phonebook")
if self._query_type not in ["phonebook", "intelligent"]:
raise AnalyzerConfigurationException(
f"{self._query_type} not supported")
self.url = self.base_url + f"/{self._query_type}/search"
self._rows_limit = int(params.get("rows_limit", 1000))
self._max_tries = int(params.get("max_tries", 10))
self._poll_distance = int(params.get("poll_distance", 3))
self._timeout = int(params.get("timeout", 10))
self._datefrom = params.get("datefrom", "")
self._dateto = params.get("dateto", "")
self.__api_key = self._secrets["api_key_name"]
def run(self):
self.session = requests.Session()
self.session.headers = {"Authorization": f"Bearer {self.__api_key}"}
if self.analysis_type == "search":
response = self.__triage_search()
elif self.analysis_type == "submit":
response = self.__triage_submit()
else:
raise AnalyzerConfigurationException(
f"analysis type '{self.analysis_type}' not supported."
"Supported are: 'search', 'submit'.")
return response
def run(self):
if not self.__api_key:
raise AnalyzerConfigurationException(
f"No API key retrieved with name: {self.api_key_name}.")
try:
response = requests.get(
self.base_url + self.observable_name,
params={"token": self.__api_key},
)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
return result
def set_params(self, params):
self.analysis_type = params.get("honeydb_analysis", "all")
self.endpoints = [
"scan_twitter",
"ip_query",
"ip_history",
"internet_scanner",
"ip_info",
]
if self.analysis_type not in self.endpoints and self.analysis_type != "all":
raise AnalyzerConfigurationException(
f"analysis_type is not valid: {self.analysis_type}")
# set secrets
self.__api_key = self._secrets["api_key_name"]
self.__api_id = self._secrets["api_id_name"]
self.headers = {
"X-HoneyDb-ApiKey": self.__api_key,
"X-HoneyDb-ApiId": self.__api_id,
}
self.result = {}
def run(self):
self.__auth = secrets.get_secret(self.api_key_name)
if not self.__auth:
raise AnalyzerConfigurationException(
f"No secret retrieved for `{self.api_key_name}`")
# try to identify search operator
operator = self.__identify_search_operator()
if operator:
value = f"{operator}:{self.observable_name}"
else:
# if operator couldn't be identified, we can query without it
value = self.observable_name
# execute searches
entries = self.__search(value)
return {
"query_value": value,
"pages_queried": self.pages,
"entries": entries,
}
def run(self):
api_key = secrets.get_secret(self._api_key_name)
if not api_key:
raise AnalyzerConfigurationException(
f"No API key retrieved with name: '{self._api_key_name}'")
session = requests.Session()
session.headers.update({
"x-key": api_key,
"User-Agent": "IntelOwl/v1.x"
})
params = {
"term": self.observable_name,
"buckets": [],
"lookuplevel": 0,
"maxresults": self._rows_limit,
"timeout": 10,
"sort": 4,
"media": 0,
"terminate": [],
"target": 0,
}
# POST the search term --> Fetch the 'id' --> GET the results using the 'id'
r = session.post(self.base_url, json=params)
r.raise_for_status()
search_id = r.json().get("id", None)
if not search_id:
raise AnalyzerRunException(
f"Failed to request search. Status code: {r.status_code}.")
time.sleep(15)
r = session.get(
f"{self.base_url}/result?id={search_id}&limit={self._rows_limit}&offset=-1"
)
r.raise_for_status()
selectors = r.json()["selectors"]
parsed_selectors = self.__pb_search_results(selectors)
return {"id": search_id, **parsed_selectors}
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
"""Run ActiveDNS analyzer
Admit:
* additional_config_params[service]: google - Google DoH (DNS over HTTPS)
* additional_config_params[service]: cloudflare - CloudFlare DoH (DNS over HTTPS)
* additional_config_params[service]: cloudflare_malware -
CloudFlare DoH (DNS over HTTPS) with Malware filtering
* additional_config_params[service]: classic - classic DNS query
Google and CloudFlare return an IP (or NXDOMAIN) from a domain.
Classic support also reverse lookup (domain from IP)
:param analyzer_name: str
Analyzer configuration in analyzer_config.json
:param job_id: str
job identifier
:param observable_name: str
analyzed observable
:param observable_classification: str
observable classification (allow: ip or domain) ip only classic
:param additional_config_params: dict
params service to select the service
:return: report: dict
name: observable_name, resolution: ip,NXDOMAIN, ''
"""
logger.info(
f"started analyzer {analyzer_name} job_id {job_id} observable {observable_name}"
)
report = general.get_basic_report_template(analyzer_name)
try:
dns_type = additional_config_params.get("service", "")
if dns_type == "google":
_doh_google(
job_id,
analyzer_name,
observable_classification,
observable_name,
report,
)
elif dns_type == "cloudflare":
_doh_cloudflare(
job_id,
analyzer_name,
observable_classification,
observable_name,
report,
)
elif dns_type == "cloudflare_malware":
_doh_cloudflare_malware(
job_id,
analyzer_name,
observable_classification,
observable_name,
report,
)
elif dns_type == "classic":
_classic_dns(
job_id,
analyzer_name,
observable_classification,
observable_name,
report,
)
else:
raise AnalyzerConfigurationException(
f"Service selected: {dns_type} is not available")
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
error_message = (
f"job_id:{job_id} analyzer:{analyzer_name} "
f"observable_name:{observable_name} Analyzer error {e}")
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
f"job_id:{job_id} analyzer:{analyzer_name} "
f"observable_name:{observable_name} Unexpected error {e}")
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info(
f"ended analyzer {analyzer_name} job_id {job_id} observable {observable_name}"
)
return report
def start_analyzers(analyzers_to_execute, analyzers_config, job_id, md5, is_sample):
set_job_status(job_id, "running")
if is_sample:
file_path, filename = get_filepath_filename(job_id)
else:
observable_name, observable_classification = get_observable_data(job_id)
for analyzer in analyzers_to_execute:
try:
analyzer_module = analyzers_config[analyzer].get("python_module", "")
if not analyzer_module:
message = (
f"no python_module available in config for {analyzer} analyzer?!"
)
raise AnalyzerConfigurationException(message)
additional_config_params = analyzers_config[analyzer].get(
"additional_config_params", {}
)
# run analyzer with a celery task asynchronously
if is_sample:
# check if we should run the hash instead of the binary
run_hash = analyzers_config[analyzer].get("run_hash", "")
if run_hash:
# check which kind of hash the analyzer needs
run_hash_type = analyzers_config[analyzer].get(
"run_hash_type", "md5"
)
if run_hash_type == "md5":
hash_value = md5
elif run_hash_type == "sha256":
hash_value = generate_sha256(job_id)
else:
error_message = (
f"only md5 and sha256 are supported "
f"but you asked {run_hash_type}. job_id: {job_id}"
)
raise AnalyzerConfigurationException(error_message)
# run the analyzer with the hash
args = [
analyzer,
job_id,
hash_value,
"hash",
additional_config_params,
]
getattr(tasks, analyzer_module).apply_async(
args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE
)
else:
# run the analyzer with the binary
args = [
analyzer,
job_id,
file_path,
filename,
md5,
additional_config_params,
]
getattr(tasks, analyzer_module).apply_async(
args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE
)
else:
# observables analyzer case
args = [
analyzer,
job_id,
observable_name,
observable_classification,
additional_config_params,
]
getattr(tasks, analyzer_module).apply_async(
args=args, queue=settings.CELERY_TASK_DEFAULT_QUEUE
)
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
error_message = "job_id {}. analyzer: {}. error: {}".format(
job_id, analyzer, e
)
logger.error(error_message)
set_failed_analyzer(analyzer, job_id, error_message)
def start_analyzers(
analyzers_to_execute,
analyzers_config,
runtime_configuration,
job_id,
md5,
is_sample,
):
set_job_status(job_id, "running")
if is_sample:
file_path, filename = get_filepath_filename(job_id)
else:
observable_name, observable_classification = get_observable_data(
job_id)
for analyzer in analyzers_to_execute:
ac = analyzers_config[analyzer]
try:
module = ac.get("python_module", None)
if not module:
raise AnalyzerConfigurationException(
f"no python_module available in config for {analyzer} analyzer?!"
)
additional_config_params = ac.get("additional_config_params", {})
adjust_analyzer_config(runtime_configuration,
additional_config_params, analyzer, job_id,
md5)
# get celery queue
queue = ac.get("queue", "default")
if queue not in CELERY_QUEUES:
logger.error(
f"Analyzer {analyzers_to_execute} has a wrong queue."
f" Setting to default")
queue = "default"
# construct arguments
if is_sample:
# check if we should run the hash instead of the binary
run_hash = ac.get("run_hash", False)
if run_hash:
# check which kind of hash the analyzer needs
run_hash_type = ac.get("run_hash_type", "md5")
if run_hash_type == "md5":
hash_value = md5
elif run_hash_type == "sha256":
hash_value = generate_sha256(job_id)
else:
error_message = (
f"only md5 and sha256 are supported "
f"but you asked {run_hash_type}. job_id: {job_id}")
raise AnalyzerConfigurationException(error_message)
# run the analyzer with the hash
args = [
f"observable_analyzers.{module}",
analyzer,
job_id,
hash_value,
"hash",
additional_config_params,
]
else:
# run the analyzer with the binary
args = [
f"file_analyzers.{module}",
analyzer,
job_id,
file_path,
filename,
md5,
additional_config_params,
]
else:
# observables analyzer case
args = [
f"observable_analyzers.{module}",
analyzer,
job_id,
observable_name,
observable_classification,
additional_config_params,
]
# run analyzer with a celery task asynchronously
stl = ac.get("soft_time_limit", 300)
t_id = uuid()
celery_app.send_task(
"run_analyzer",
args=args,
queue=queue,
soft_time_limit=stl,
task_id=t_id,
)
# to track task_id by job_id
task_ids = cache.get(job_id)
if isinstance(task_ids, list):
task_ids.append(t_id)
else:
task_ids = [t_id]
cache.set(job_id, task_ids)
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
err_msg = f"({analyzer}, job_id #{job_id}) -> Error: {e}"
logger.error(err_msg)
set_failed_analyzer(analyzer, job_id, err_msg)
def start_analyzers(
analyzers_to_execute,
analyzers_config,
runtime_configuration,
job_id,
md5,
is_sample,
):
set_job_status(job_id, "running")
if is_sample:
file_path, filename = get_filepath_filename(job_id)
else:
observable_name, observable_classification = get_observable_data(
job_id)
for analyzer in analyzers_to_execute:
ac = analyzers_config[analyzer]
try:
module = ac.get("python_module", None)
if not module:
raise AnalyzerConfigurationException(
f"no python_module available in config for {analyzer} analyzer?!"
)
additional_config_params = ac.get("additional_config_params", {})
adjust_analyzer_config(runtime_configuration,
additional_config_params, analyzer)
# construct arguments
if is_sample:
# check if we should run the hash instead of the binary
run_hash = ac.get("run_hash", False)
if run_hash:
# check which kind of hash the analyzer needs
run_hash_type = ac.get("run_hash_type", "md5")
if run_hash_type == "md5":
hash_value = md5
elif run_hash_type == "sha256":
hash_value = generate_sha256(job_id)
else:
error_message = (
f"only md5 and sha256 are supported "
f"but you asked {run_hash_type}. job_id: {job_id}")
raise AnalyzerConfigurationException(error_message)
# run the analyzer with the hash
args = [
analyzer,
job_id,
hash_value,
"hash",
additional_config_params,
]
else:
# run the analyzer with the binary
args = [
analyzer,
job_id,
file_path,
filename,
md5,
additional_config_params,
]
else:
# observables analyzer case
args = [
analyzer,
job_id,
observable_name,
observable_classification,
additional_config_params,
]
# run analyzer with a celery task asynchronously
tasks.analyzer_run.apply_async(
args=[ac["type"], module, *args],
queue=settings.CELERY_TASK_DEFAULT_QUEUE,
)
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
error_message = f"job_id {job_id}. analyzer: {analyzer}. error: {e}"
logger.error(error_message)
set_failed_analyzer(analyzer, job_id, error_message)
def _raise_container_not_running(self) -> None:
raise AnalyzerConfigurationException(
f"{self.name} docker container is not running.\n"
f"You have to enable it using the appropriate "
f"parameter when executing start.py."
)
def run(self):
if not self.__api_key:
raise AnalyzerConfigurationException(
f"No API key retrieved with name: {self.api_key_name}.")
return self._scan_binary()
def run(self):
result = {}
headers = {"Content-Type": "application/json"}
if self.__api_key:
headers["Authorization"] = self.__api_key
else:
warning = f"No API key retrieved with name: {self.api_key_name}"
logger.info(
f"{warning}. Continuing without API key..." f" <- {self.__repr__()}"
)
self.report["errors"].append(warning)
if self.analysis_type == "dfi_search":
if self.observable_classification == "hash":
uri = (
f"/api/dfi/search/hash/{self.hash_type}?hash={self.observable_name}"
)
elif self.observable_classification in ["ip", "url", "domain"]:
uri = (
f"/api/dfi/search/ioc/{self.observable_classification}"
f"?keyword={self.observable_name}"
)
elif self.observable_classification == "generic":
try:
type_, value = self.observable_name.split(":")
except ValueError:
self.generic_identifier_mode = "auto"
type_ = self.type_of_generic()
value = self.observable_name
if type_ not in ["email", "filename", "registry", "xmpid"]:
raise AnalyzerRunException(f"Unknown Type: {type_}")
uri = f"/api/dfi/search/ioc/{type_}?keyword={value}"
else:
raise AnalyzerRunException()
elif self.analysis_type == "iocdb_search":
uri = f"/api/iocdb/search?keyword={self.observable_name}"
elif self.analysis_type == "repdb_search":
uri = f"/api/repdb/search?keyword={self.observable_name}"
else:
raise AnalyzerConfigurationException(
f"analysis type: '{self.analysis_type}' not suported."
"Supported are: 'dfi_search', 'iocdb_search', 'repdb_search'."
)
try:
response = requests.get(self.base_url + uri, headers=headers)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
if (
self.analysis_type == "dfi_search"
and self.observable_classification == "hash"
):
result["hash_type"] = self.hash_type
if self.generic_identifier_mode == "auto":
result["type_of_generic"] = self.type_of_generic()
return result
