예제 #1
0
파일: analytics.py 프로젝트: idkwim/malcom
    def bulk_asn(self):
        results = self.data.elements.find({'type': 'ip'})

        #elts = []
        ips = []
        debug_output("(getting ASNs for %s IPs)" % results.count(),
                     type='analytics')

        for r in results:
            ips.append(r)

        as_info = get_net_info_shadowserver(ips)

        if not as_info:
            return

        for ip in as_info:

            _as = as_info[ip]
            _ip = self.data.find_one({'value': ip})

            del _as['ip']
            for key in _as:
                if key not in ['type', 'value', 'context']:
                    _ip[key] = _as[key]
            del _as['bgp']

            _as = As.from_dict(_as)

            # commit any changes to DB
            _as = self.save_element(_as)
            _ip = self.save_element(_ip)

            if _as and _ip:
                self.data.connect(_ip, _as, 'net_info')
예제 #2
0
	def bulk_asn(self):
		results = self.data.elements.find({ 'type': 'ip' })
		
		#elts = []
		ips = []
		debug_output("(getting ASNs for %s IPs)" % results.count(), type='analytics')
		
		for r in results:
			ips.append(r)

		as_info = get_net_info_shadowserver(ips)
		
		if not as_info:
			return

		for ip in as_info:
			
			_as = as_info[ip]
			_ip = self.data.find_one({'value': ip})
			
			del _as['ip']
			for key in _as:
				if key not in ['type', 'value', 'context']:
					_ip[key] = _as[key]
			del _as['bgp']

			_as = As.from_dict(_as)

			# commit any changes to DB
			_as = self.save_element(_as)
			_ip = self.save_element(_ip)

			if _as and _ip:
				self.data.connect(_ip, _as, 'net_info')
예제 #3
0
	def bulk_asn(self, items=1000):

		last_analysis = {'$or': [
									{ 'last_analysis': {"$lt": datetime.datetime.utcnow() - datetime.timedelta(days=7)} },
									{ 'last_analysis': None },
								]
						}

		nobgp = {"$or": [{'bgp': None}, last_analysis ]}

		total = self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]}).count()
		done = 0
		results = [r for r in self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]})[:items]]

		while len(results) > 0:
		
			ips = []
			debug_output("(getting ASNs for %s IPs - %s/%s done)" % (len(results), done, total), type='analytics')
			
			for r in results:
				ips.append(r)

			as_info = {}
			
			try:
				as_info = get_net_info_shadowserver(ips)
			except Exception, e:
				debug_output("Could not get AS for IPs: %s" % e)
			
			if as_info == {}:
				debug_output("as_info empty", 'error')
				return

			for ip in as_info:
				
				_as = as_info[ip]
				_ip = self.data.find_one({'value': ip})

				if not _ip:
					return

				del _as['ip']
				for key in _as:
					if key not in ['type', 'value', 'tags']:
						_ip[key] = _as[key]
				del _as['bgp']

				_as = As.from_dict(_as)

				# commit any changes to DB
				_as = self.save_element(_as)
				_ip = self.save_element(_ip)
			
				if _as and _ip:
					self.data.connect(_ip, _as, 'net_info')
			done += len(results)
			results = [r for r in self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]})[:items]]
예제 #4
0
파일: model.py 프로젝트: idkwim/malcom
def decode_custom_as(document):
	assert document['type'] == "as"
	return As.from_dict(document)
예제 #5
0
class Analytics:
    def __init__(self, max_threads=4):
        self.data = Model()
        #self.max_threads = threading.Semaphore(app.config['THREADS'])
        self.active = False
        self.websocket = None
        self.thread = None
        self.websocket_lock = threading.Lock()
        self.stack_lock = threading.Lock()
        self.progress = 0
        self.total = 0

        self.max_threads = threading.Semaphore(4)

    def add_text(self, text, context=[]):
        added = []
        for t in text:
            elt = None
            if t.strip() != "":
                if is_ip(t):
                    elt = Ip(is_ip(t), [])
                elif is_url(t):
                    elt = Url(is_url(t), [])
                elif is_hostname(t):
                    elt = Hostname(is_hostname(t), [])
                if elt:
                    added.append(self.save_element(elt, context))

        if len(added) == 1:
            return added[0]
        else:
            return added

    def save_element(self, element, context=[], with_status=False):

        element.upgrade_context(context)
        return self.data.save(element, with_status=with_status)

    # graph function
    def add_artifacts(self, data, context=[]):
        artifacts = find_artifacts(data)

        added = []
        for url in artifacts['urls']:
            added.append(self.save_element(url, context))

        for hostname in artifacts['hostnames']:
            added.append(self.save_element(hostname, context))

        for ip in artifacts['ips']:
            added.append(self.save_element(ip, context))

        return added

    # elements analytics

    def bulk_asn(self):
        results = self.data.elements.find({'type': 'ip', 'bgp': None})

        ips = []
        debug_output("(getting ASNs for %s IPs)" % results.count(),
                     type='analytics')

        for r in results:
            ips.append(r)

        ips_chunks = [ips[x:x + 100] for x in xrange(0, len(ips), 100)]

        as_info = {}
        for ips in ips_chunks:
            try:
                as_info = dict(as_info.items() +
                               get_net_info_shadowserver(ips).items())

            except Exception, e:
                pass

        if as_info == {}:
            return

        for ip in as_info:

            _as = as_info[ip]
            _ip = self.data.find_one({'value': ip})

            if not _ip:
                return

            del _as['ip']
            for key in _as:
                if key not in ['type', 'value', 'context']:
                    _ip[key] = _as[key]
            del _as['bgp']

            _as = As.from_dict(_as)

            # commit any changes to DB
            _as = self.save_element(_as)
            _ip = self.save_element(_ip)

            if _as and _ip:
                self.data.connect(_ip, _as, 'net_info')
예제 #6
0
파일: model.py 프로젝트: idkwim/malcom
def decode_custom_as(document):
    assert document['type'] == "as"
    return As.from_dict(document)