def analyze_ip(ip, results):
"""Specific analyzer for Ip observables."""
links = set()
result = db.data.infos_byip(ip.value)
if result is None:
return []
if results is not None:
results.update(raw=pformat(result))
if 'as_name' in result:
asn = AutonomousSystem.get_or_create(
value=result['as_name'],
as_num=result['as_num'],
)
links.update(ip.active_link_to(asn, 'asn#', 'IVRE - MaxMind'))
if 'country_code' in result:
ip.geoip = {"country": result['country_code']}
if 'region_code' in result:
ip.geoip['region'] = ' / '.join(result['region_code'])
if 'city' in result:
ip.geoip['city'] = result['city']
ip.save()
if all(context['source'] != 'ivre_maxmind' for context in ip.context):
result['source'] = 'ivre_maxmind'
ip.add_context(result)
return list(links)
def analyze_ip(ip, results):
"""Specific analyzer for Ip observables."""
links = set()
result = db.data.infos_byip(ip.value)
if result is None:
return []
if results is not None:
results.update(raw=pformat(result))
if "as_name" in result:
asn = AutonomousSystem.get_or_create(
value=result["as_name"],
as_num=result["as_num"],
)
links.update(ip.active_link_to(asn, "asn#", "IVRE - MaxMind"))
if "country_code" in result:
ip.geoip = {"country": result["country_code"]}
if "region_code" in result:
ip.geoip["region"] = " / ".join(result["region_code"])
if "city" in result:
ip.geoip["city"] = result["city"]
ip.save()
if all(context["source"] != "ivre_maxmind" for context in ip.context):
result["source"] = "ivre_maxmind"
ip.add_context(result)
return list(links)
def analyze(self, line):
_id = line['id']
_ = line['firstseen']
url = line['url']
_status = line['status']
_hash = line['hash']
country = line['country']
asn = line['as']
tags = ["collected_by_honeypot"]
context = {"source": self.name, "country": country}
url_obs = None
if url:
try:
url_obs = Url.get_or_create(value=url.rstrip())
url_obs.add_context(context)
url_obs.tag(tags)
url_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if _hash and len(_hash) > 16:
try:
hash_obs = Hash.get_or_create(value=_hash)
hash_obs.add_context(context)
hash_obs.tag(tags)
hash_obs.add_source(self.name)
if url_obs:
hash_obs.active_link_to(url_obs,
"MD5",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
if asn:
try:
asn = asn.split(" ")[0].replace("AS", "")
asn_obs = AutonomousSystem.get_or_create(value=asn)
asn_obs.add_context(context)
asn_obs.tag(tags)
asn_obs.add_source(self.name)
asn_obs.active_link_to(url_obs,
"ASN",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, line):
first_seen, _type, family, hostname, url, status, registrar, ips, asns, countries = line
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": parser.parse(first_seen),
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
url_obs = False
hostname_obs = False
try:
url_obs = Url.get_or_create(value=url.rstrip())
url_obs.add_context(context)
url_obs.tag(tags)
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
try:
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
for ip in ips.split("|"):
if ip != hostname and ip is not None and ip != '':
try:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.active_link_to((url_obs, hostname),
"ip",
self.name,
clean_old=False)
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid Observable: {}".format(e))
for asn in asns.split("|"):
try:
asn_obs = AutonomousSystem.get_or_create(value=asn)
asn_obs.active_link_to((hostname, ip_obs),
"asn",
self.name,
clean_old=False)
except (ObservableValidationError,
UnicodeEncodeError) as e:
logging.error("Invalid Observable: {}".format(e))
def _process_asn_data(page, observable):
links = set()
if page['page'].get('asn'):
asn = AutonomousSystem.get_or_create(value=page['page']['asn'].replace("AS", ""))
links.update(asn.active_link_to(observable, 'asn#', 'UrlScanIo Query'))
if page['page'].get('asnname'):
asnname = Text.get_or_create(value=page['page']['asnname'])
links.update(asnname.active_link_to(observable,
'asn_name', 'UrlScanIoQuerycanIo Query'))
if page['page'].get('server'):
server = Text.get_or_create(value=page['page']['server'])
links.update(server.active_link_to(observable,
'server', 'UrlScanIo Query'))
return list(links)
def analyze(self, line):
_id, _, url, _status, _hash, country, asn = tuple(line)
tags = ["collected_by_honeypot"]
context = {"source": self.name}
if url:
try:
url_obs = Url.get_or_create(value=url.rstrip())
url_obs.add_context(context)
url_obs.tag(tags)
url_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if _hash:
try:
hash_obs = Hash.get_or_create(value=_hash)
hash_obs.add_context(context)
hash_obs.tag(tags)
hash_obs.add_source(self.name)
hash_obs.active_link_to(url_obs,
"MD5",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
if asn:
try:
asn = asn.split(" ")[0].replace("AS", "")
asn_obs = AutonomousSystem.get_or_create(value=asn)
asn_obs.add_context(context)
asn_obs.tag(tags)
asn_obs.add_source(self.name)
asn_obs.active_link_to(url_obs,
"ASN",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
def _process_asn_data(page, observable):
links = set()
if page["page"].get("asn"):
asn = AutonomousSystem.get_or_create(
value=page["page"]["asn"].replace("AS", ""))
links.update(
asn.active_link_to(observable, "asn#", "UrlScanIo Query"))
if page["page"].get("asnname"):
asnname = Text.get_or_create(value=page["page"]["asnname"])
links.update(
asnname.active_link_to(observable, "asn_name",
"UrlScanIoQuerycanIo Query"))
if page["page"].get("server"):
server = Text.get_or_create(value=page["page"]["server"])
links.update(
server.active_link_to(observable, "server", "UrlScanIo Query"))
return list(links)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
json_string = json.dumps(result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = AutonomousSystem.get_or_create(
value=result['asn'].replace("AS", ""))
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings["shodan_api_key"])
json_string = json.dumps(
result, sort_keys=True, indent=4, separators=(",", ": ")
)
results.update(raw=json_string)
if "tags" in result and result["tags"] is not None:
ip.tag(result["tags"])
if "asn" in result and result["asn"] is not None:
o_asn = AutonomousSystem.get_or_create(
value=result["asn"].replace("AS", "")
)
links.update(o_asn.active_link_to(ip, "asn#", "Shodan Query"))
if "hostnames" in result and result["hostnames"] is not None:
for hostname in result["hostnames"]:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, "A record", "Shodan Query"))
if "isp" in result and result["isp"] is not None:
o_isp = Company.get_or_create(name=result["isp"])
links.update(ip.active_link_to(o_isp, "hosting", "Shodan Query"))
for context in ip.context:
if context["source"] == "shodan_query":
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result["source"] = "shodan_query"
ip.add_context(result)
return list(links)
def analyze(self, item):
context = {"source": self.name}
_id = item["_id"]
domain = item["domain"]
ip = item["ip"]
asn = item["asn"]
country_code = item["country_code"]
panel_url = item["panel_index"]
panel_path = item["panel_path"]
panel_version = item["panel_version"]
status = item["status"]
feeder = item["feeder"]
first_seen = item["first_seen"]
context["first_seen"] = first_seen
if feeder:
context["feeder"] = feeder
context["status"] = status
if item["data"]:
context["data"] = item["data"]
context["country"] = country_code
context["_id"] = _id
context["panel_version"] = panel_version
context["panel_path"] = panel_path
try:
hostname = None
url = None
ip_obs = None
asn_obs = None
if domain:
hostname = Hostname.get_or_create(value=domain)
hostname.add_context(context)
hostname.tag("azorult")
if ip:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.add_context(context)
ip_obs.tag("azorult")
if panel_url:
url = Url.get_or_create(value=panel_url)
url.add_context(context)
url.tag("azorult")
if asn:
asn_obs = AutonomousSystem.get_or_create(value=asn)
asn_obs.add_context(context)
asn_obs.tag("azorult")
if hostname and ip_obs:
hostname.active_link_to(ip_obs, "IP", self.name)
if asn_obs and ip_obs:
asn_obs.active_link_to(ip_obs, "AS", self.name)
if url and hostname:
url.active_link_to(hostname, "hostname", self.name)
except ObservableValidationError as e:
logging.error(e)
except TypeError as e:
logging.error(item)