def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
bulk = params.pop('observables', [])
_refang = params.pop('refang', False)
for item in bulk:
value = item['value']
tags = item.get('tags', [])
if _refang:
obs = self.objectmanager.add_text(refang(value), tags)
else:
obs = self.objectmanager.add_text(value, tags)
self._modify_observable(obs, {
'source': item.get('source'),
'context': item.get('context'),
})
added.append(obs)
return render(added)
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
bulk = params.pop('observables', [])
_refang = params.pop('refang', False)
for item in bulk:
value = item['value']
tags = item.get('tags', [])
if _refang:
obs = self.objectmanager.add_text(refang(value), tags)
else:
obs = self.objectmanager.add_text(value, tags)
self._modify_observable(
obs, {
'source': item.get('source'),
'context': item.get('context'),
})
added.append(obs)
return render(added)
def new(self):
"""Create a new Observable
Create a new Observable from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
:<json boolean refang: If set, the observable will be refanged before being added to the database
"""
params = request.json
if "id" in params:
obs = self.objectmanager.objects.get(id=params.pop("id"))
else:
forced_type = params.pop("force_type", None)
try:
if params.pop("refang", None):
obs = self.objectmanager.add_text(
refang(params.pop("value")), force_type=forced_type
)
else:
obs = self.objectmanager.add_text(
params.pop("value"), force_type=forced_type
)
except (GenericYetiError, ObservableValidationError) as e:
logging.error(e)
abort(400)
return render(self._modify_observable(obs, params))
def normalize(self):
self.value = refang(self.value)
if iptools.ipv4.validate_ip(self.value):
self.value = iptools.ipv4.hex2ip(iptools.ipv4.ip2hex(self.value))
self.version = 4
elif iptools.ipv6.validate_ip(self.value):
self.value = iptools.ipv6.long2ip(iptools.ipv6.ip2long(self.value))
self.version = 6
def normalize(self):
self.value = refang(self.value)
if iptools.ipv4.validate_ip(self.value):
self.value = iptools.ipv4.hex2ip(iptools.ipv4.ip2hex(self.value))
self.version = 4
elif iptools.ipv6.validate_ip(self.value):
self.value = iptools.ipv6.long2ip(iptools.ipv6.ip2long(self.value))
self.version = 6
def normalize(self):
self.value = refang(self.value.lower())
try:
self.idna = unicode(idna.encode(self.value))
except idna.core.InvalidCodepoint:
pass
except Exception, e:
raise ObservableValidationError(e.message)
def check_type(txt):
url = refang(txt)
match = re.match("^" + Url.regex + "$", url, re.VERBOSE)
if match:
url = match.group(1)
if url.find('/') != -1:
return match.group(1)
else:
return None
def check_type(txt):
url = refang(txt)
match = re.match("^" + Url.regex + "$", url, re.VERBOSE)
if match:
url = match.group(1)
if url.find('/') != -1:
return match.group(1)
else:
return None
def clean(self):
"""Ensures that URLs are canonized before saving"""
self.value = refang(self.value.strip())
try:
if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http://
self.value = u"http://{}".format(self.value)
self.value = urlnorm.norm(self.value)
except urlnorm.InvalidUrl:
raise ObservableValidationError("Invalid URL: {}".format(self.value))
def normalize(self):
self.value = refang(self.value.lower())
# Remove trailing dot if existing
if self.value.endswith("."):
self.value = self.value[:-1]
try:
self.idna = unicode(idna.encode(self.value))
except idna.core.InvalidCodepoint:
pass
except Exception, e:
raise ObservableValidationError(e.message)
def normalize(self):
self.value = refang(self.value.lower())
# Remove trailing dot if existing
if self.value.endswith("."):
self.value = self.value[:-1]
try:
self.idna = self.value
except idna.core.InvalidCodepoint:
pass
except Exception as e:
raise ObservableValidationError(e.with_traceback())
def is_valid(cls, match):
# Check that the domain is not preceded or followed by a '/'
# This ensures that we do not match URLs
if match.group('pre') != '/' and match.group('post') != '/':
# Check that the domain is valid (by checking TLD)
value = refang(match.group('search'))
if len(value) <= 255:
parts = extract(value)
if parts.suffix and parts.domain:
return True
return False
def clean(self):
"""Ensures that URLs are canonized before saving"""
self.value = refang(self.value.strip())
try:
if re.match(r"[^:]+://", self.value) is None:
# if no schema is specified, assume http://
self.value = u"http://{}".format(self.value)
self.value = urlnorm.norm(self.value)
self.parse()
except urlnorm.InvalidUrl:
raise ObservableValidationError("Invalid URL: {}".format(self.value))
except UnicodeDecodeError:
raise ObservableValidationError("Invalid URL (UTF-8 decode error): {}".format(self.value))
def is_valid(cls, match):
# Check that the domain is not preceded or followed by a '/'
# This ensures that we do not match URLs
if match.group("pre") != "/" and match.group("post") != "/":
# Check that the domain is valid (by checking TLD)
value = refang(match.group("search"))
if len(value) <= 255:
parts = tldextract_parser(value)
if parts.suffix and parts.domain:
return True
return False
def check_type(txt):
hostname = refang(txt.lower())
if hostname:
match = re.match("^" + Hostname.regex + "$", hostname)
if match:
if hostname.endswith('.'):
hostname = hostname[:-1]
parts = extract(hostname)
if parts.suffix and parts.domain:
return hostname
return False
def check_type(txt):
hostname = refang(txt.lower())
if hostname:
match = re.match("^" + Hostname.regex + "$", hostname)
if match:
if hostname.endswith('.'):
hostname = hostname[:-1]
parts = extract(hostname)
if parts.suffix and parts.domain:
return hostname
return False
def new(self):
"""Create a new Observable
Create a new Observable from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
:<json boolean refang: If set, the observable will be refanged before being added to the database
"""
params = request.json
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(params.pop('value')))
else:
obs = self.objectmanager.add_text(params.pop('value'))
return render(self._modify_observable(obs, params))
def new(self):
"""Create a new Observable
Create a new Observable from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
:<json boolean refang: If set, the observable will be refanged before being added to the database
"""
params = request.json
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(params.pop('value')))
else:
obs = self.objectmanager.add_text(params.pop('value'))
return render(self._modify_observable(obs, params))
def normalize(self):
self.value = refang(self.value)
try:
if re.match(r"[^:]+://", self.value) is None:
# if no schema is specified, assume http://
self.value = u"http://{}".format(self.value)
self.value = urlnorm.norm(self.value).replace(' ', '%20')
self.parse()
except urlnorm.InvalidUrl:
raise ObservableValidationError(
"Invalid URL: {}".format(self.value))
except UnicodeDecodeError:
raise ObservableValidationError(
"Invalid URL (UTF-8 decode error): {}".format(self.value))
def normalize(self):
self.value = refang(self.value)
try:
if re.match(r"[^:]+://", self.value) is None:
# if no schema is specified, assume http://
self.value = u"http://{}".format(self.value)
self.value = urlnorm.norm(self.value).replace(' ', '%20')
self.parse()
except urlnorm.InvalidUrl:
raise ObservableValidationError("Invalid URL: {}".format(
self.value))
except UnicodeDecodeError:
raise ObservableValidationError(
"Invalid URL (UTF-8 decode error): {}".format(self.value))
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [String] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
observables = params.pop('observables', [])
for item in observables:
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(item))
else:
obs = self.objectmanager.add_text(item)
added.append(self._modify_observable(obs, params.copy()))
return render(added)
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [String] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
observables = params.pop('observables', [])
for item in observables:
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(item))
else:
obs = self.objectmanager.add_text(item)
added.append(self._modify_observable(obs, params.copy()))
return render(added)
def normalize(self):
self.value = refang(self.value)
if re.match(r"[^:]+://", self.value) is None:
# if no schema is specified, assume http://
self.value = u"http://{}".format(self.value)
try:
self.value = url_normalize(self.value).replace(' ', '%20')
except Exception as e:
raise ObservableValidationError("Invalid URL: {}".format(
self.value))
try:
p = tldextract_parser(self.value)
self.value = self.value.replace(p.fqdn,
p.fqdn.encode("idna").decode(), 1)
self.parse()
except UnicodeDecodeError:
raise ObservableValidationError(
"Invalid URL (UTF-8 decode error): {}".format(self.value))
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
observables = params.pop('observables', [])
for item in observables:
obs = item['value']
tags = item['tags']
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(obs), tags)
else:
obs = self.objectmanager.add_text(obs, tags)
added.append(obs)
return render(added)
def is_valid(cls, match):
value = refang(match.group("search"))
return iptools.ipv4.validate_ip(value) or iptools.ipv6.validate_ip(
value)
def normalize(self):
self.value = refang(self.value.lower())
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables = [refang(observable) for observable in observables if observable]
extended_query = set(observables) | set(derive(observables))
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({"observable": o.info(), "related": [], "suggested_tags": set()})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
[match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables = [
refang(observable) for observable in observables if observable
]
extended_query = set(observables) | set(derive(observables))
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value
not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({
"observable": o.info(),
"related": [],
"suggested_tags": set()
})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
[
match["suggested_tags"].add(tag)
for tag in node.generate_tags() if tag not in o_tags
]
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
def is_valid(cls, match):
value = refang(match.group('search'))
return iptools.ipv4.validate_ip(value) or iptools.ipv6.validate_ip(
value)