def bulk_asn(self):
results = self.data.elements.find({'type': 'ip'})
#elts = []
ips = []
debug_output("(getting ASNs for %s IPs)" % results.count(),
type='analytics')
for r in results:
ips.append(r)
as_info = get_net_info_shadowserver(ips)
if not as_info:
return
for ip in as_info:
_as = as_info[ip]
_ip = self.data.find_one({'value': ip})
del _as['ip']
for key in _as:
if key not in ['type', 'value', 'context']:
_ip[key] = _as[key]
del _as['bgp']
_as = As.from_dict(_as)
# commit any changes to DB
_as = self.save_element(_as)
_ip = self.save_element(_ip)
if _as and _ip:
self.data.connect(_ip, _as, 'net_info')
def bulk_asn(self):
results = self.data.elements.find({ 'type': 'ip' })
#elts = []
ips = []
debug_output("(getting ASNs for %s IPs)" % results.count(), type='analytics')
for r in results:
ips.append(r)
as_info = get_net_info_shadowserver(ips)
if not as_info:
return
for ip in as_info:
_as = as_info[ip]
_ip = self.data.find_one({'value': ip})
del _as['ip']
for key in _as:
if key not in ['type', 'value', 'context']:
_ip[key] = _as[key]
del _as['bgp']
_as = As.from_dict(_as)
# commit any changes to DB
_as = self.save_element(_as)
_ip = self.save_element(_ip)
if _as and _ip:
self.data.connect(_ip, _as, 'net_info')
def bulk_asn(self, items=1000):
last_analysis = {'$or': [
{ 'last_analysis': {"$lt": datetime.datetime.utcnow() - datetime.timedelta(days=7)} },
{ 'last_analysis': None },
]
}
nobgp = {"$or": [{'bgp': None}, last_analysis ]}
total = self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]}).count()
done = 0
results = [r for r in self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]})[:items]]
while len(results) > 0:
ips = []
debug_output("(getting ASNs for %s IPs - %s/%s done)" % (len(results), done, total), type='analytics')
for r in results:
ips.append(r)
as_info = {}
try:
as_info = get_net_info_shadowserver(ips)
except Exception, e:
debug_output("Could not get AS for IPs: %s" % e)
if as_info == {}:
debug_output("as_info empty", 'error')
return
for ip in as_info:
_as = as_info[ip]
_ip = self.data.find_one({'value': ip})
if not _ip:
return
del _as['ip']
for key in _as:
if key not in ['type', 'value', 'tags']:
_ip[key] = _as[key]
del _as['bgp']
_as = As.from_dict(_as)
# commit any changes to DB
_as = self.save_element(_as)
_ip = self.save_element(_ip)
if _as and _ip:
self.data.connect(_ip, _as, 'net_info')
done += len(results)
results = [r for r in self.data.elements.find({ "$and": [{'type': 'ip'}, nobgp]})[:items]]
def decode_custom_as(document): assert document['type'] == "as" return As.from_dict(document)
class Analytics:
def __init__(self, max_threads=4):
self.data = Model()
#self.max_threads = threading.Semaphore(app.config['THREADS'])
self.active = False
self.websocket = None
self.thread = None
self.websocket_lock = threading.Lock()
self.stack_lock = threading.Lock()
self.progress = 0
self.total = 0
self.max_threads = threading.Semaphore(4)
def add_text(self, text, context=[]):
added = []
for t in text:
elt = None
if t.strip() != "":
if is_ip(t):
elt = Ip(is_ip(t), [])
elif is_url(t):
elt = Url(is_url(t), [])
elif is_hostname(t):
elt = Hostname(is_hostname(t), [])
if elt:
added.append(self.save_element(elt, context))
if len(added) == 1:
return added[0]
else:
return added
def save_element(self, element, context=[], with_status=False):
element.upgrade_context(context)
return self.data.save(element, with_status=with_status)
# graph function
def add_artifacts(self, data, context=[]):
artifacts = find_artifacts(data)
added = []
for url in artifacts['urls']:
added.append(self.save_element(url, context))
for hostname in artifacts['hostnames']:
added.append(self.save_element(hostname, context))
for ip in artifacts['ips']:
added.append(self.save_element(ip, context))
return added
# elements analytics
def bulk_asn(self):
results = self.data.elements.find({'type': 'ip', 'bgp': None})
ips = []
debug_output("(getting ASNs for %s IPs)" % results.count(),
type='analytics')
for r in results:
ips.append(r)
ips_chunks = [ips[x:x + 100] for x in xrange(0, len(ips), 100)]
as_info = {}
for ips in ips_chunks:
try:
as_info = dict(as_info.items() +
get_net_info_shadowserver(ips).items())
except Exception, e:
pass
if as_info == {}:
return
for ip in as_info:
_as = as_info[ip]
_ip = self.data.find_one({'value': ip})
if not _ip:
return
del _as['ip']
for key in _as:
if key not in ['type', 'value', 'context']:
_ip[key] = _as[key]
del _as['bgp']
_as = As.from_dict(_as)
# commit any changes to DB
_as = self.save_element(_as)
_ip = self.save_element(_ip)
if _as and _ip:
self.data.connect(_ip, _as, 'net_info')
def decode_custom_as(document):
assert document['type'] == "as"
return As.from_dict(document)