def bulk(self): """Bulk-add observables Bulk-add Observables from an array of strings. :<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.) :<json boolean refang: If set, the observables will be refanged before being added to the database """ added = [] params = request.json bulk = params.pop('observables', []) _refang = params.pop('refang', False) for item in bulk: value = item['value'] tags = item.get('tags', []) if _refang: obs = self.objectmanager.add_text(refang(value), tags) else: obs = self.objectmanager.add_text(value, tags) self._modify_observable(obs, { 'source': item.get('source'), 'context': item.get('context'), }) added.append(obs) return render(added)
def bulk(self): """Bulk-add observables Bulk-add Observables from an array of strings. :<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.) :<json boolean refang: If set, the observables will be refanged before being added to the database """ added = [] params = request.json bulk = params.pop('observables', []) _refang = params.pop('refang', False) for item in bulk: value = item['value'] tags = item.get('tags', []) if _refang: obs = self.objectmanager.add_text(refang(value), tags) else: obs = self.objectmanager.add_text(value, tags) self._modify_observable( obs, { 'source': item.get('source'), 'context': item.get('context'), }) added.append(obs) return render(added)
def new(self): """Create a new Observable Create a new Observable from the JSON object passed in the ``POST`` data. :<json object params: JSON object containing fields to set :<json boolean refang: If set, the observable will be refanged before being added to the database """ params = request.json if "id" in params: obs = self.objectmanager.objects.get(id=params.pop("id")) else: forced_type = params.pop("force_type", None) try: if params.pop("refang", None): obs = self.objectmanager.add_text( refang(params.pop("value")), force_type=forced_type ) else: obs = self.objectmanager.add_text( params.pop("value"), force_type=forced_type ) except (GenericYetiError, ObservableValidationError) as e: logging.error(e) abort(400) return render(self._modify_observable(obs, params))
def normalize(self): self.value = refang(self.value) if iptools.ipv4.validate_ip(self.value): self.value = iptools.ipv4.hex2ip(iptools.ipv4.ip2hex(self.value)) self.version = 4 elif iptools.ipv6.validate_ip(self.value): self.value = iptools.ipv6.long2ip(iptools.ipv6.ip2long(self.value)) self.version = 6
def normalize(self): self.value = refang(self.value.lower()) try: self.idna = unicode(idna.encode(self.value)) except idna.core.InvalidCodepoint: pass except Exception, e: raise ObservableValidationError(e.message)
def check_type(txt): url = refang(txt) match = re.match("^" + Url.regex + "$", url, re.VERBOSE) if match: url = match.group(1) if url.find('/') != -1: return match.group(1) else: return None
def clean(self): """Ensures that URLs are canonized before saving""" self.value = refang(self.value.strip()) try: if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http:// self.value = u"http://{}".format(self.value) self.value = urlnorm.norm(self.value) except urlnorm.InvalidUrl: raise ObservableValidationError("Invalid URL: {}".format(self.value))
def normalize(self): self.value = refang(self.value.lower()) # Remove trailing dot if existing if self.value.endswith("."): self.value = self.value[:-1] try: self.idna = unicode(idna.encode(self.value)) except idna.core.InvalidCodepoint: pass except Exception, e: raise ObservableValidationError(e.message)
def normalize(self): self.value = refang(self.value.lower()) # Remove trailing dot if existing if self.value.endswith("."): self.value = self.value[:-1] try: self.idna = self.value except idna.core.InvalidCodepoint: pass except Exception as e: raise ObservableValidationError(e.with_traceback())
def is_valid(cls, match): # Check that the domain is not preceded or followed by a '/' # This ensures that we do not match URLs if match.group('pre') != '/' and match.group('post') != '/': # Check that the domain is valid (by checking TLD) value = refang(match.group('search')) if len(value) <= 255: parts = extract(value) if parts.suffix and parts.domain: return True return False
def clean(self): """Ensures that URLs are canonized before saving""" self.value = refang(self.value.strip()) try: if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http:// self.value = u"http://{}".format(self.value) self.value = urlnorm.norm(self.value) self.parse() except urlnorm.InvalidUrl: raise ObservableValidationError("Invalid URL: {}".format(self.value)) except UnicodeDecodeError: raise ObservableValidationError("Invalid URL (UTF-8 decode error): {}".format(self.value))
def is_valid(cls, match): # Check that the domain is not preceded or followed by a '/' # This ensures that we do not match URLs if match.group("pre") != "/" and match.group("post") != "/": # Check that the domain is valid (by checking TLD) value = refang(match.group("search")) if len(value) <= 255: parts = tldextract_parser(value) if parts.suffix and parts.domain: return True return False
def check_type(txt): hostname = refang(txt.lower()) if hostname: match = re.match("^" + Hostname.regex + "$", hostname) if match: if hostname.endswith('.'): hostname = hostname[:-1] parts = extract(hostname) if parts.suffix and parts.domain: return hostname return False
def new(self): """Create a new Observable Create a new Observable from the JSON object passed in the ``POST`` data. :<json object params: JSON object containing fields to set :<json boolean refang: If set, the observable will be refanged before being added to the database """ params = request.json if params.pop('refang', None): obs = self.objectmanager.add_text(refang(params.pop('value'))) else: obs = self.objectmanager.add_text(params.pop('value')) return render(self._modify_observable(obs, params))
def normalize(self): self.value = refang(self.value) try: if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http:// self.value = u"http://{}".format(self.value) self.value = urlnorm.norm(self.value).replace(' ', '%20') self.parse() except urlnorm.InvalidUrl: raise ObservableValidationError( "Invalid URL: {}".format(self.value)) except UnicodeDecodeError: raise ObservableValidationError( "Invalid URL (UTF-8 decode error): {}".format(self.value))
def normalize(self): self.value = refang(self.value) try: if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http:// self.value = u"http://{}".format(self.value) self.value = urlnorm.norm(self.value).replace(' ', '%20') self.parse() except urlnorm.InvalidUrl: raise ObservableValidationError("Invalid URL: {}".format( self.value)) except UnicodeDecodeError: raise ObservableValidationError( "Invalid URL (UTF-8 decode error): {}".format(self.value))
def bulk(self): """Bulk-add observables Bulk-add Observables from an array of strings. :<json [String] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.) :<json boolean refang: If set, the observables will be refanged before being added to the database """ added = [] params = request.json observables = params.pop('observables', []) for item in observables: if params.pop('refang', None): obs = self.objectmanager.add_text(refang(item)) else: obs = self.objectmanager.add_text(item) added.append(self._modify_observable(obs, params.copy())) return render(added)
def normalize(self): self.value = refang(self.value) if re.match(r"[^:]+://", self.value) is None: # if no schema is specified, assume http:// self.value = u"http://{}".format(self.value) try: self.value = url_normalize(self.value).replace(' ', '%20') except Exception as e: raise ObservableValidationError("Invalid URL: {}".format( self.value)) try: p = tldextract_parser(self.value) self.value = self.value.replace(p.fqdn, p.fqdn.encode("idna").decode(), 1) self.parse() except UnicodeDecodeError: raise ObservableValidationError( "Invalid URL (UTF-8 decode error): {}".format(self.value))
def bulk(self): """Bulk-add observables Bulk-add Observables from an array of strings. :<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.) :<json boolean refang: If set, the observables will be refanged before being added to the database """ added = [] params = request.json observables = params.pop('observables', []) for item in observables: obs = item['value'] tags = item['tags'] if params.pop('refang', None): obs = self.objectmanager.add_text(refang(obs), tags) else: obs = self.objectmanager.add_text(obs, tags) added.append(obs) return render(added)
def is_valid(cls, match): value = refang(match.group("search")) return iptools.ipv4.validate_ip(value) or iptools.ipv6.validate_ip( value)
def normalize(self): self.value = refang(self.value.lower())
def match_observables(observables, save_matches=False, fetch_neighbors=True): # Remove empty observables observables = [refang(observable) for observable in observables if observable] extended_query = set(observables) | set(derive(observables)) data = { "matches": [], "unknown": set(observables), "entities": {}, "known": [], "neighbors": [], } # add to "known" for o in Observable.objects(value__in=list(extended_query)): data['known'].append(o.info()) del_from_set(data['unknown'], o.value) if fetch_neighbors: for link, node in (o.incoming()): if isinstance(node, Observable): if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags: data['neighbors'].append((link.info(), node.info())) for nodes in o.neighbors("Entity").values(): for l, node in nodes: # add node name and link description to indicator node_data = {"entity": node.type, "name": node.name, "link_description": l.description} # uniquely add node information to related entitites ent = data['entities'].get(node.name, node.info()) if 'matches' not in ent: ent['matches'] = {"observables": []} if 'observables' not in ent['matches']: ent['matches']['observables'] = [] info = node.info() o_info = o.info() info['matched_observable'] = { "value": o_info['value'], "tags": [t['name'] for t in o_info['tags']], "human_url": o_info['human_url'], "url": o_info['url'] } if info not in ent['matches']['observables']: ent['matches']['observables'].append(info) data['entities'][node.name] = ent # add to "matches" for o, i in Indicator.search(extended_query): if save_matches: o = Observable.add_text(o) else: o = Observable.guess_type(o)(value=o) try: o.validate() except ObservableValidationError: pass try: o = Observable.objects.get(value=o.value) except Exception: pass match = i.info() match.update({"observable": o.info(), "related": [], "suggested_tags": set()}) for nodes in i.neighbors("Entity").values(): for l, node in nodes: # add node name and link description to indicator node_data = {"entity": node.type, "name": node.name, "link_description": l.description} match["related"].append(node_data) # uniquely add node information to related entitites ent = data['entities'].get(node.name, node.info()) if 'matches' not in ent: ent['matches'] = {"indicators": []} if 'indicators' not in ent['matches']: ent['matches']['indicators'] = [] info = i.info() info['matched_observable'] = o.value if info not in ent['matches']['indicators']: ent['matches']['indicators'].append(info) data['entities'][node.name] = ent o_tags = o.get_tags() [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags] data["matches"].append(match) data['entities'] = data['entities'].values() return data
def match_observables(observables, save_matches=False, fetch_neighbors=True): # Remove empty observables observables = [ refang(observable) for observable in observables if observable ] extended_query = set(observables) | set(derive(observables)) data = { "matches": [], "unknown": set(observables), "entities": {}, "known": [], "neighbors": [], } # add to "known" for o in Observable.objects(value__in=list(extended_query)): data['known'].append(o.info()) del_from_set(data['unknown'], o.value) if fetch_neighbors: for link, node in (o.incoming()): if isinstance(node, Observable): if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags: data['neighbors'].append((link.info(), node.info())) for nodes in o.neighbors("Entity").values(): for l, node in nodes: # add node name and link description to indicator node_data = { "entity": node.type, "name": node.name, "link_description": l.description } # uniquely add node information to related entitites ent = data['entities'].get(node.name, node.info()) if 'matches' not in ent: ent['matches'] = {"observables": []} if 'observables' not in ent['matches']: ent['matches']['observables'] = [] info = node.info() o_info = o.info() info['matched_observable'] = { "value": o_info['value'], "tags": [t['name'] for t in o_info['tags']], "human_url": o_info['human_url'], "url": o_info['url'] } if info not in ent['matches']['observables']: ent['matches']['observables'].append(info) data['entities'][node.name] = ent # add to "matches" for o, i in Indicator.search(extended_query): if save_matches: o = Observable.add_text(o) else: o = Observable.guess_type(o)(value=o) try: o.validate() except ObservableValidationError: pass try: o = Observable.objects.get(value=o.value) except Exception: pass match = i.info() match.update({ "observable": o.info(), "related": [], "suggested_tags": set() }) for nodes in i.neighbors("Entity").values(): for l, node in nodes: # add node name and link description to indicator node_data = { "entity": node.type, "name": node.name, "link_description": l.description } match["related"].append(node_data) # uniquely add node information to related entitites ent = data['entities'].get(node.name, node.info()) if 'matches' not in ent: ent['matches'] = {"indicators": []} if 'indicators' not in ent['matches']: ent['matches']['indicators'] = [] info = i.info() info['matched_observable'] = o.value if info not in ent['matches']['indicators']: ent['matches']['indicators'].append(info) data['entities'][node.name] = ent o_tags = o.get_tags() [ match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags ] data["matches"].append(match) data['entities'] = data['entities'].values() return data
def is_valid(cls, match): value = refang(match.group('search')) return iptools.ipv4.validate_ip(value) or iptools.ipv6.validate_ip( value)