Esempio n. 1
0
class Analytics:
    def __init__(self, max_threads=4):
        self.data = Model()
        # self.max_threads = threading.Semaphore(app.config['THREADS'])
        self.active = False
        self.status = "Inactive"
        self.websocket = None
        self.thread = None
        self.websocket_lock = threading.Lock()
        self.stack_lock = threading.Lock()
        self.progress = 0
        self.total = 0

        self.max_threads = threading.Semaphore(4)

    def add_text(self, text, tags=[]):
        added = []
        for t in text:
            elt = None
            if t.strip() != "":
                if is_url(t):
                    elt = Url(is_url(t), [])
                elif is_hostname(t):
                    elt = Hostname(is_hostname(t), [])
                elif is_ip(t):
                    elt = Ip(is_ip(t), [])
                if elt:
                    added.append(self.save_element(elt, tags))

        if len(added) == 1:
            return added[0]
        else:
            return added

    def save_element(self, element, tags=[], with_status=False):

        element.upgrade_tags(tags)
        return self.data.save(element, with_status=with_status)

        # graph function

    def add_artifacts(self, data, tags=[]):
        artifacts = find_artifacts(data)

        added = []
        for url in artifacts["urls"]:
            added.append(self.save_element(url, tags))

        for hostname in artifacts["hostnames"]:
            added.append(self.save_element(hostname, tags))

        for ip in artifacts["ips"]:
            added.append(self.save_element(ip, tags))

        return added

        # elements analytics

    def bulk_asn(self, items=1000):

        last_analysis = {
            "$or": [
                {"last_analysis": {"$lt": datetime.datetime.utcnow() - datetime.timedelta(days=7)}},
                {"last_analysis": None},
            ]
        }

        nobgp = {"$or": [{"bgp": None}, last_analysis]}

        total = self.data.elements.find({"$and": [{"type": "ip"}, nobgp]}).count()
        done = 0
        results = [r for r in self.data.elements.find({"$and": [{"type": "ip"}, nobgp]})[:items]]

        while len(results) > 0:

            ips = []
            debug_output("(getting ASNs for %s IPs - %s/%s done)" % (len(results), done, total), type="analytics")

            for r in results:
                ips.append(r)

            as_info = {}

            try:
                as_info = get_net_info_shadowserver(ips)
            except Exception, e:
                debug_output("Could not get AS for IPs: %s" % e)

            if as_info == {}:
                debug_output("as_info empty", "error")
                return

            for ip in as_info:

                _as = as_info[ip]
                _ip = self.data.find_one({"value": ip})

                if not _ip:
                    return

                del _as["ip"]
                for key in _as:
                    if key not in ["type", "value", "tags"]:
                        _ip[key] = _as[key]
                del _as["bgp"]

                _as = As.from_dict(_as)

                # commit any changes to DB
                _as = self.save_element(_as)
                _ip = self.save_element(_ip)

                if _as and _ip:
                    self.data.connect(_ip, _as, "net_info")
            done += len(results)
            results = [r for r in self.data.elements.find({"$and": [{"type": "ip"}, nobgp]})[:items]]
Esempio n. 2
0
class Analytics:
    def __init__(self):
        self.data = Model()
        self.max_threads = Malcom.config.get('MAX_THREADS', 4)
        self.active = False
        self.status = "Inactive"
        self.websocket = None
        self.thread = None
        self.websocket_lock = threading.Lock()
        self.stack_lock = threading.Lock()
        self.progress = 0
        self.total = 0

        self.max_threads = threading.Semaphore(self.max_threads)
        self.worker_threads = {}

    def add_text(self, text, tags=[]):
        added = []
        for t in text:
            elt = None
            if t.strip() != "":
                if is_url(t):
                    elt = Url(is_url(t), [])
                elif is_hostname(t):
                    elt = Hostname(is_hostname(t), [])
                elif is_ip(t):
                    elt = Ip(is_ip(t), [])
                if elt:
                    added.append(self.save_element(elt, tags))

        if len(added) == 1:
            return added[0]
        else:
            return added

    def save_element(self, element, tags=[], with_status=False):
        element.upgrade_tags(tags)
        return self.data.save(element, with_status=with_status)

    # graph function
    def add_artifacts(self, data, tags=[]):
        artifacts = find_artifacts(data)

        added = []
        for url in artifacts['urls']:
            added.append(self.save_element(url, tags))

        for hostname in artifacts['hostnames']:
            added.append(self.save_element(hostname, tags))

        for ip in artifacts['ips']:
            added.append(self.save_element(ip, tags))

        return added

    # elements analytics

    def bulk_asn(self, items=1000):

        last_analysis = {
            '$or': [
                {
                    'last_analysis': {
                        "$lt":
                        datetime.datetime.utcnow() - datetime.timedelta(days=7)
                    }
                },
                {
                    'last_analysis': None
                },
            ]
        }

        nobgp = {"$or": [{'bgp': None}, last_analysis]}

        total = self.data.elements.find({
            "$and": [{
                'type': 'ip'
            }, nobgp]
        }).count()
        done = 0
        results = [
            r
            for r in self.data.elements.find({"$and": [{
                'type': 'ip'
            }, nobgp]})[:items]
        ]

        while len(results) > 0:

            ips = []
            debug_output("(getting ASNs for %s IPs - %s/%s done)" %
                         (len(results), done, total),
                         type='analytics')

            for r in results:
                ips.append(r)

            as_info = {}

            try:
                as_info = get_net_info_shadowserver(ips)
            except Exception, e:
                debug_output("Could not get AS for IPs: %s" % e)

            if as_info == {}:
                debug_output("as_info empty", 'error')
                return

            for ip in as_info:

                _as = as_info[ip]
                _ip = self.data.find_one({'value': ip})

                if not _ip:
                    return

                del _as['ip']
                for key in _as:
                    if key not in ['type', 'value', 'tags']:
                        _ip[key] = _as[key]
                del _as['bgp']

                _as = As.from_dict(_as)

                # commit any changes to DB
                _as = self.save_element(_as)
                _ip['last_analysis'] = datetime.datetime.now()
                _ip = self.save_element(_ip)

                if _as and _ip:
                    self.data.connect(_ip, _as, 'net_info')
            done += len(results)
            results = [
                r for r in self.data.elements.find(
                    {"$and": [{
                        'type': 'ip'
                    }, nobgp]})[:items]
            ]