class Analytics: def __init__(self, max_threads=4): self.data = Model() # self.max_threads = threading.Semaphore(app.config['THREADS']) self.active = False self.status = "Inactive" self.websocket = None self.thread = None self.websocket_lock = threading.Lock() self.stack_lock = threading.Lock() self.progress = 0 self.total = 0 self.max_threads = threading.Semaphore(4) def add_text(self, text, tags=[]): added = [] for t in text: elt = None if t.strip() != "": if is_url(t): elt = Url(is_url(t), []) elif is_hostname(t): elt = Hostname(is_hostname(t), []) elif is_ip(t): elt = Ip(is_ip(t), []) if elt: added.append(self.save_element(elt, tags)) if len(added) == 1: return added[0] else: return added def save_element(self, element, tags=[], with_status=False): element.upgrade_tags(tags) return self.data.save(element, with_status=with_status) # graph function def add_artifacts(self, data, tags=[]): artifacts = find_artifacts(data) added = [] for url in artifacts["urls"]: added.append(self.save_element(url, tags)) for hostname in artifacts["hostnames"]: added.append(self.save_element(hostname, tags)) for ip in artifacts["ips"]: added.append(self.save_element(ip, tags)) return added # elements analytics def bulk_asn(self, items=1000): last_analysis = { "$or": [ {"last_analysis": {"$lt": datetime.datetime.utcnow() - datetime.timedelta(days=7)}}, {"last_analysis": None}, ] } nobgp = {"$or": [{"bgp": None}, last_analysis]} total = self.data.elements.find({"$and": [{"type": "ip"}, nobgp]}).count() done = 0 results = [r for r in self.data.elements.find({"$and": [{"type": "ip"}, nobgp]})[:items]] while len(results) > 0: ips = [] debug_output("(getting ASNs for %s IPs - %s/%s done)" % (len(results), done, total), type="analytics") for r in results: ips.append(r) as_info = {} try: as_info = get_net_info_shadowserver(ips) except Exception, e: debug_output("Could not get AS for IPs: %s" % e) if as_info == {}: debug_output("as_info empty", "error") return for ip in as_info: _as = as_info[ip] _ip = self.data.find_one({"value": ip}) if not _ip: return del _as["ip"] for key in _as: if key not in ["type", "value", "tags"]: _ip[key] = _as[key] del _as["bgp"] _as = As.from_dict(_as) # commit any changes to DB _as = self.save_element(_as) _ip = self.save_element(_ip) if _as and _ip: self.data.connect(_ip, _as, "net_info") done += len(results) results = [r for r in self.data.elements.find({"$and": [{"type": "ip"}, nobgp]})[:items]]
class Analytics: def __init__(self): self.data = Model() self.max_threads = Malcom.config.get('MAX_THREADS', 4) self.active = False self.status = "Inactive" self.websocket = None self.thread = None self.websocket_lock = threading.Lock() self.stack_lock = threading.Lock() self.progress = 0 self.total = 0 self.max_threads = threading.Semaphore(self.max_threads) self.worker_threads = {} def add_text(self, text, tags=[]): added = [] for t in text: elt = None if t.strip() != "": if is_url(t): elt = Url(is_url(t), []) elif is_hostname(t): elt = Hostname(is_hostname(t), []) elif is_ip(t): elt = Ip(is_ip(t), []) if elt: added.append(self.save_element(elt, tags)) if len(added) == 1: return added[0] else: return added def save_element(self, element, tags=[], with_status=False): element.upgrade_tags(tags) return self.data.save(element, with_status=with_status) # graph function def add_artifacts(self, data, tags=[]): artifacts = find_artifacts(data) added = [] for url in artifacts['urls']: added.append(self.save_element(url, tags)) for hostname in artifacts['hostnames']: added.append(self.save_element(hostname, tags)) for ip in artifacts['ips']: added.append(self.save_element(ip, tags)) return added # elements analytics def bulk_asn(self, items=1000): last_analysis = { '$or': [ { 'last_analysis': { "$lt": datetime.datetime.utcnow() - datetime.timedelta(days=7) } }, { 'last_analysis': None }, ] } nobgp = {"$or": [{'bgp': None}, last_analysis]} total = self.data.elements.find({ "$and": [{ 'type': 'ip' }, nobgp] }).count() done = 0 results = [ r for r in self.data.elements.find({"$and": [{ 'type': 'ip' }, nobgp]})[:items] ] while len(results) > 0: ips = [] debug_output("(getting ASNs for %s IPs - %s/%s done)" % (len(results), done, total), type='analytics') for r in results: ips.append(r) as_info = {} try: as_info = get_net_info_shadowserver(ips) except Exception, e: debug_output("Could not get AS for IPs: %s" % e) if as_info == {}: debug_output("as_info empty", 'error') return for ip in as_info: _as = as_info[ip] _ip = self.data.find_one({'value': ip}) if not _ip: return del _as['ip'] for key in _as: if key not in ['type', 'value', 'tags']: _ip[key] = _as[key] del _as['bgp'] _as = As.from_dict(_as) # commit any changes to DB _as = self.save_element(_as) _ip['last_analysis'] = datetime.datetime.now() _ip = self.save_element(_ip) if _as and _ip: self.data.connect(_ip, _as, 'net_info') done += len(results) results = [ r for r in self.data.elements.find( {"$and": [{ 'type': 'ip' }, nobgp]})[:items] ]