def test_msg_is_not_called_when_interval_not_reached(self):
callback = MagicMock()
message = 'a'
cuckoo = Cuckoo(10, callback)
cuckoo.reset()
cuckoo.msg(message)
self.assertFalse(callback.called)
def __init__(self):
self.session = None
if config.get("submit_to_vt"):
self.vt = Virustotal(config.get("vt_key", optional=True))
else:
self.vt = None
self.cuckoo = Cuckoo(config)
self.do_ip_to_asn_resolution = config.get("do_ip_to_asn_resolution", optional=True, default=True)
def test_msg_resets_timestamp_after_interval_reached(self):
callback = MagicMock()
message = 'a'
cuckoo = Cuckoo(0.1, callback)
cuckoo.reset()
timestamp1 = cuckoo.timestamp
print(cuckoo.timestamp)
self.assertFalse(callback.called)
sleep(0.11)
cuckoo.msg(message)
timestamp2 = cuckoo.timestamp
print(cuckoo.timestamp)
self.assertTrue(callback.called)
assert timestamp1 is not timestamp2
def __init__(self):
self.session = None
if config.get("submit_to_vt"):
self.vt = Virustotal(config.get("vt_key", optional=True))
else:
self.vt = None
self.cuckoo = Cuckoo(config)
self.do_ip_to_asn_resolution = False
self.ip2asn = config.get("ip_to_asn_resolution",
optional=True,
default=True)
if self.ip2asn == "offline":
self.do_ip_to_asn_resolution = True
self.fill_db_ipranges()
if self.ip2asn == "online":
self.do_ip_to_asn_resolution = True
def main():
np.random.seed(round(time.time()))
cuckoos = []
# Generisanje populacije
for p in range(params.get_population_size()):
cuckoos.append(Cuckoo())
cuckoos = sorted(cuckoos, key=lambda cuckoo: cuckoo.get_fitness())
best_position = cuckoos[0].get_position()
best_fitness = cuckoos[0].get_fitness()
for iteration in range(params.get_iterations()):
#generisanje novih resenja
for i in range(len(cuckoos)):
cuckoos[i].move_cuckoo()
cuckoos[i].set_fitness(fn.calculate(cuckoos[i].get_position()))
# nasumicno biramo resenje sa kojim uporedjujemo novo generisano
j = np.random.randint(0, params.get_population_size())
while j == i:
j = np.random.randint(0, params.get_population_size())
# za probleme minimizacije
if (cuckoos[i].get_fitness() < cuckoos[j].get_fitness()):
cuckoos[j].set_position(cuckoos[i].get_position())
cuckoos[j].set_fitness(cuckoos[i].get_fitness())
cuckoos = sorted(cuckoos, key=lambda cuckoo: cuckoo.get_fitness())
# zadrzavamo najbolje resenje
for a in range(1, len(cuckoos)):
r = np.random.rand()
if ( r < params.get_pa() ):
cuckoos[a].abandon()
cuckoos[a].set_fitness(fn.calculate(cuckoos[a].get_position()))
cuckoos = sorted(cuckoos, key=lambda cuckoo: cuckoo.get_fitness())
if ( cuckoos[0].get_fitness() < best_fitness ):
best_fitness = cuckoos[0].get_fitness()
best_position = cuckoos[0].get_position()
sys.stdout.write("\r Iteration:%7d, BestFitness:%.4f" % (iteration, best_fitness))
print()
print(best_position)
results.write(str(iteration) + "," + str(best_fitness) + "\n")
results.close()
input()
def test_msg_is_only_called_when_interval_reached(self):
callback = MagicMock()
message = 'a'
cuckoo = Cuckoo(0.1, callback)
cuckoo.reset()
cuckoo.msg(message)
self.assertFalse(callback.called)
sleep(0.11)
cuckoo.msg(message)
self.assertTrue(callback.called)
class ClientController:
def __init__(self, web):
self.web = web
self.session = None
if config.get("submit_to_vt"):
self.vt = Virustotal(config.get("vt_key", optional=True))
else:
self.vt = None
self.cuckoo = Cuckoo(config)
self.do_ip_to_asn_resolution = config.get("do_ip_to_asn_resolution",
optional=True,
default=True)
def get_asn(self, asn):
asn_obj = self.session.query(ASN).filter(ASN.asn == asn).first()
if asn_obj:
return asn_obj.json(depth=1)
else:
asn_info = get_asn_info(asn)
if asn_info:
asn_obj = ASN(asn=asn,
name=asn_info['name'],
reg=asn_info['reg'],
country=asn_info['country'])
self.session.add(asn_obj)
return asn_obj.json(depth=1)
@db_wrapper
def put_domain(self, domain):
if self.vt != None:
report = self.vt.query_domain_reports(domain)
if report != None:
domainReport = DomainReport(domain=domain,
report=json.dumps(report))
self.session.add(domainReport)
self.session.flush()
@db_wrapper
def put_session(self, session):
ipinfo = None
asn = None
block = None
country = None
if self.do_ip_to_asn_resolution:
ipinfo = get_ip_info(session["ip"])
if ipinfo:
asn_obj = self.get_asn(ipinfo["asn"])
asn = ipinfo["asn"]
block = ipinfo["ipblock"]
country = ipinfo["country"]
report = {}
if self.vt != None:
if self.web.get_ip_report(session["ip"]) == None:
report = self.vt.query_ip_reports(session["ip"])
ipReport = IpReport(ip=session["ip"],
report=json.dumps(report))
self.session.add(ipReport)
# Calculate "hash"
connhash = ""
for event in session["stream"]:
if event["in"]:
line = event["data"]
line = ''.join(char for char in line
if ord(char) < 128 and ord(char) > 32)
if line != "":
linehash = abs(hash(line)) % 0xFFFF
connhash += struct.pack("!H", linehash)
connhash = connhash.encode("hex")
backend_user = self.session.query(User).filter(
User.username == session["backend_username"]).first()
conn = Connection(ip=session["ip"],
user=session["user"],
date=session["date"],
password=session["pass"],
stream=json.dumps(session["stream"]),
asn_id=asn,
ipblock=block,
country=country,
connhash=connhash,
backend_user_id=backend_user.id)
self.session.add(conn)
self.session.flush()
req_urls = []
set_urls = set(session["urls"])
for url in set_urls:
db_url = self.db.get_url(url).fetchone()
url_id = 0
report = ''
parsed_uri = urlparse(url)
domain = '{uri.netloc}'.format(uri=parsed_uri)
if self.vt != None:
report = self.vt.query_domain_reports(domain)
domainReport = DomainReport(domain=domain,
report=json.dumps(report))
self.session.add(domainReport)
if db_url == None:
url_ip = None
url_asn = None
url_country = None
if self.do_ip_to_asn_resolution:
url_ip, url_info = get_url_info(url)
if url_info:
asn_obj_url = self.get_asn(url_info["asn"])
url_asn = url_info["asn"]
url_country = url_info["country"]
url_id = self.db.put_url(url, session["date"], url_ip, url_asn,
url_country)
req_urls.append(url)
elif db_url["sample"] == None:
req_urls.append(url)
url_id = db_url["id"]
else:
# Sample exists already
# TODO: Check url for oldness
url_id = db_url["id"]
self.db.link_conn_url(conn.id, url_id)
# Find previous connections
# A connection is associated when:
# - same honeypot/user
# - connection happened as long as 120s before
# - same client ip OR same username/password combo
assoc_timediff = 120
previous_conns = (self.session.query(Connection).filter(
Connection.date > (conn.date - assoc_timediff),
or_(
and_(Connection.user == conn.user,
Connection.password == conn.password),
Connection.ip == conn.ip),
Connection.backend_user_id == conn.backend_user_id,
Connection.id != conn.id).all())
for prev in previous_conns:
conn.conns_before.append(prev)
# Check connection against all tags
tags = self.session.query(Tag).all()
conn = self.session.query(Connection).filter(
Connection.id == conn.id).first()
for tag in tags:
json_obj = conn.json(depth=0)
json_obj["text_combined"] = filter_ascii(json_obj["text_combined"])
if simple_eval(tag.code, names=json_obj) == True:
self.db.link_conn_tag(conn.id, tag.id)
return req_urls
@db_wrapper
def put_sample_info(self, f):
url = f["url"]
url_id = self.db.get_url(url).fetchone()["id"]
result = None
try:
if self.vt != None:
vtobj = self.vt.query_hash_sha256(f["sha256"])
if vtobj:
result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]) + " " + self.vt.get_best_result(vtobj)
except:
pass
sample_id = self.db.put_sample(f["sha256"], f["name"], f["length"],
f["date"], f["info"], result)
self.db.link_url_sample(url_id, sample_id)
return f
@db_wrapper
def put_sample(self, data):
sha256 = hashlib.sha256(data).hexdigest()
self.db.put_sample_data(sha256, data)
if config.get("cuckoo_enabled"):
self.cuckoo.upload(os.path.join(config.get("sample_dir"), sha256),
sha256)
elif config.get("submit_to_vt"):
self.vt.upload_file(os.path.join(config.get("sample_dir"), sha256),
sha256)
@db_wrapper
def update_vt_result(self, sample_sha):
sample = self.session.query(Sample).filter(
Sample.sha256 == sample_sha).first()
if sample:
vtobj = self.vt.query_hash_sha256(sample_sha)
if vtobj:
sample.result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]) + " " + self.vt.get_best_result(vtobj)
return sample.json(depth=1)
return None
def __init__(self): self.vt = Virustotal(config["vt_key"]) self.db = None self.sess = None self.cuckoo = Cuckoo(config)
class ClientController: def __init__(self): self.vt = Virustotal(config["vt_key"]) self.db = None self.sess = None self.cuckoo = Cuckoo(config) def get_asn(self, asn): asn_obj = self.session.query(ASN).filter(ASN.asn == asn).first() if asn_obj: return asn_obj.json(depth=1) else: asn_info = get_asn_info(asn) if asn_info: asn_obj = ASN(asn=asn, name=asn_info['name'], reg=asn_info['reg'], country=asn_info['country']) self.session.add(asn_obj) return asn_obj.json(depth=1) @db_wrapper def put_session(self, session): ipinfo = get_ip_info(session["ip"]) asn = None block = None country = None if ipinfo: asn_obj = self.get_asn(ipinfo["asn"]) asn = ipinfo["asn"] block = ipinfo["ipblock"] country = ipinfo["country"] s_id = self.db.put_conn(session["ip"], session["user"], session["pass"], session["date"], session["text_combined"], asn, block, country) req_urls = [] for url in session["urls"]: db_url = self.db.get_url(url).fetchone() url_id = 0 if db_url == None: url_ip, url_info = get_url_info(url) url_asn = None url_country = None if url_info: asn_obj_url = self.get_asn(url_info["asn"]) url_asn = url_info["asn"] url_country = url_info["country"] url_id = self.db.put_url(url, session["date"], url_ip, url_asn, url_country) req_urls.append(url) elif db_url["sample"] == None: req_urls.append(url) url_id = db_url["id"] else: # Sample exists already # TODO: Check url for oldness url_id = db_url["id"] self.db.link_conn_url(s_id, url_id) return req_urls @db_wrapper def put_sample_info(self, f): url = f["url"] url_id = self.db.get_url(url).fetchone()["id"] result = None try: vtobj = self.vt.query_hash_sha256(f["sha256"]) if vtobj: result = str(vtobj["positives"]) + "/" + str(vtobj["total"]) + " " + self.vt.get_best_result(vtobj) except: pass sample_id = self.db.put_sample(f["sha256"], f["name"], f["length"], f["date"], f["info"], result) self.db.link_url_sample(url_id, sample_id) return f @db_wrapper def put_sample(self, data): sha256 = hashlib.sha256(data).hexdigest() self.db.put_sample_data(sha256, data) if config["cuckoo_enabled"]: self.cuckoo.upload(os.path.join(config["sample_dir"], sha256), sha256) elif config["submit_to_vt"]: self.vt.upload_file(os.path.join(config["sample_dir"], sha256), sha256)
if app.config["CACHEBUSTER"]:
from utils import cache_buster
app.after_request(cache_buster)
db_obj = DB(app.config["DBUSER"],
app.config["DBPASSWORD"],
app.config["DBHOST"],
app.config["DBPORT"],
app.config["DBNAME"],
debug=app.config["DEBUG"],
db_debug=app.config["DB_DEBUG"])
v = Virustotal(app.config["VIRUSTOTAL_API_KEY"], debug=debug)
c = Cuckoo(app.config["CUCKOO_API_URL"],
app.config["CUCKOO_API_USER"],
app.config["CUCKOO_API_PASS"],
debug=debug)
s = SSLSite(debug=debug)
try:
t = TopSite(debug=debug)
except Exception as e:
logging.info(
"error fetching top 1M site list, falling back to pickle method")
t = TopSite(autoload=False, debug=debug)
t._load_from_pickle()
scheduler.factory(refresh=app.config["TASK_REFRESH"], debug=debug)
scheduler_obj = scheduler.get_scheduler()
scheduler_obj.start()
analysis_manager.factory(db_obj, scheduler_obj, [v, c], [t, s])
class ClientController:
def __init__(self):
self.session = None
if config.get("submit_to_vt"):
self.vt = Virustotal(config.get("vt_key", optional=True))
else:
self.vt = None
self.cuckoo = Cuckoo(config)
self.do_ip_to_asn_resolution = config.get("do_ip_to_asn_resolution",
optional=True,
default=True)
def get_asn(self, asn):
asn_obj = self.session.query(ASN).filter(ASN.asn == asn).first()
if asn_obj:
return asn_obj.json(depth=1)
else:
asn_info = get_asn_info(asn)
if asn_info:
asn_obj = ASN(asn=asn,
name=asn_info['name'],
reg=asn_info['reg'],
country=asn_info['country'])
self.session.add(asn_obj)
return asn_obj.json(depth=1)
@db_wrapper
def put_session(self, session):
ipinfo = None
asn = None
block = None
country = None
network_id = None
if self.do_ip_to_asn_resolution:
ipinfo = get_ip_info(session["ip"])
if ipinfo:
asn_obj = self.get_asn(ipinfo["asn"])
asn = ipinfo["asn"]
block = ipinfo["ipblock"]
country = ipinfo["country"]
# Calculate "hash"
connhash = ""
for event in session["stream"]:
if event["in"]:
line = event["data"]
line = ''.join(char for char in line
if ord(char) < 128 and ord(char) > 32)
if line != "":
linehash = abs(hash(line)) % 0xFFFF
connhash += struct.pack("!H", linehash)
connhash = connhash.encode("hex")
backend_user = self.session.query(User).filter(
User.username == session["backend_username"]).first()
conn = Connection(ip=session["ip"],
user=session["user"],
date=session["date"],
password=session["pass"],
stream=json.dumps(session["stream"]),
asn_id=asn,
ipblock=block,
country=country,
connhash=connhash,
backend_user_id=backend_user.id)
self.session.add(conn)
self.session.flush() # to get id
samples = []
urls = []
for sample_json in session["samples"]:
sample, url = self.create_url_sample(sample_json)
if network_id == None and sample.network_id != None:
network_id = sample.network_id
if network_id == None and url.network_id != None:
network_id = url.network_id
conn.urls.append(url)
samples.append(sample)
urls.append(url)
# Find previous connections
# A connection is associated when:
# - same honeypot/user
# - connection happened as long as 120s before
# - same client ip OR same username/password combo
assoc_timediff = 120
previous_conns = (self.session.query(Connection).filter(
Connection.date > (conn.date - assoc_timediff),
or_(
and_(Connection.user == conn.user,
Connection.password == conn.password),
Connection.ip == conn.ip),
Connection.backend_user_id == conn.backend_user_id,
Connection.id != conn.id).all())
for prev in previous_conns:
if network_id == None and prev.network_id != None:
network_id = prev.network_id
conn.conns_before.append(prev)
# Check connection against all tags
tags = self.session.query(Tag).all()
for tag in tags:
json_obj = conn.json(depth=0)
json_obj["text_combined"] = filter_ascii(json_obj["text_combined"])
if simple_eval(tag.code, names=json_obj) == True:
self.db.link_conn_tag(conn.id, tag.id)
# Only create new networks for connections with urls or associtaed conns,
# to prevent the creation of thousands of networks
# NOTE: only conns with network == NULL will get their network updated
# later so whe should only create a network where we cannot easily
# change it later
if (len(conn.urls) > 0
or len(previous_conns) > 0) and network_id == None:
network_id = self.create_network().id
# Update network on self
conn.network_id = network_id
# Update network on all added Urls
for url in urls:
if url.network_id == None:
url.network_id = network_id
# Update network on all added Samples
for sample in samples:
if sample.network_id == None:
sample.network_id = network_id
# Update network on all previous connections withut one
if network_id != None:
for prev in previous_conns:
if prev.network_id == None:
prev.network_id = network_id
self.session.flush()
return []
@db_wrapper
def create_network(self):
net = Network()
self.session.add(net)
self.session.flush()
return net
def create_url_sample(self, f):
url = self.session.query(Url).filter(Url.url == f["url"]).first()
if url == None:
url_ip = None
url_asn = None
url_country = None
if self.do_ip_to_asn_resolution:
url_ip, url_info = get_url_info(f["url"])
if url_info:
asn_obj_url = self.get_asn(url_info["asn"])
url_asn = url_info["asn"]
url_country = url_info["country"]
url = Url(url=f["url"],
date=f["date"],
ip=url_ip,
asn=url_asn,
country=url_country)
self.session.add(url)
sample = self.session.query(Sample).filter(
Sample.sha256 == f["sha256"]).first()
if sample == None:
result = None
try:
if self.vt != None:
vtobj = self.vt.query_hash_sha256(f["sha256"])
if vtobj:
result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]) + " " + self.vt.get_best_result(
vtobj)
except:
pass
sample = Sample(sha256=f["sha256"],
name=f["name"],
length=f["length"],
date=f["date"],
info=f["info"],
result=result)
self.session.add(sample)
if sample.network_id != None and url.network_id == None:
url.network_id = sample.network_id
if sample.network_id == None and url.network_id != None:
sample.network_id = url.network_id
url.sample = sample
return sample, url
@db_wrapper
def put_sample(self, data):
sha256 = hashlib.sha256(data).hexdigest()
self.db.put_sample_data(sha256, data)
if config.get("cuckoo_enabled"):
self.cuckoo.upload(os.path.join(config.get("sample_dir"), sha256),
sha256)
elif config.get("submit_to_vt"):
self.vt.upload_file(os.path.join(config.get("sample_dir"), sha256),
sha256)
@db_wrapper
def update_vt_result(self, sample_sha):
sample = self.session.query(Sample).filter(
Sample.sha256 == sample_sha).first()
if sample:
vtobj = self.vt.query_hash_sha256(sample_sha)
if vtobj:
sample.result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]) + " " + self.vt.get_best_result(vtobj)
return sample.json(depth=1)
return None
def test_msg_calls_callback(self):
callback = MagicMock()
message = 'a'
cuckoo = Cuckoo(callback=callback)
cuckoo.msg(message)
callback.assert_called_with(message)
def test_interval_reached(self):
cuckoo = Cuckoo(0.1)
cuckoo.reset()
self.assertFalse(cuckoo._interval_reached())
sleep(0.11)
self.assertTrue(cuckoo._interval_reached())
def test_msg_gets_called_on_the_very_first_time(self):
callback = MagicMock()
message = 'a'
cuckoo = Cuckoo(1, callback)
cuckoo.msg(message)
self.assertTrue(callback.called)
def test_msg_sets_timestamp_on_first_call(self):
cuckoo = Cuckoo()
cuckoo.msg()
assert cuckoo.timestamp
class ClientController:
def __init__(self):
self.session = None
if config.get("submit_to_vt"):
self.vt = Virustotal(config.get("vt_key", optional=True))
else:
self.vt = None
self.cuckoo = Cuckoo(config)
self.do_ip_to_asn_resolution = False
self.ip2asn = config.get("ip_to_asn_resolution",
optional=True,
default=True)
if self.ip2asn == "offline":
self.do_ip_to_asn_resolution = True
self.fill_db_ipranges()
if self.ip2asn == "online":
self.do_ip_to_asn_resolution = True
@db_wrapper
def _get_asn(self, asn_id):
asn_obj = self.session.query(ASN).filter(ASN.asn == asn_id).first()
if asn_obj:
return asn_obj
else:
asn_info = additionalinfo.get_asn_info(asn_id)
if asn_info:
asn_obj = ASN(asn=asn_id,
name=asn_info['name'],
reg=asn_info['reg'],
country=asn_info['country'])
self.session.add(asn_obj)
return asn_obj
return None
def calc_connhash_similiarity(self, h1, h2):
l = min(len(h1), len(h2))
r = 0
for i in range(0, l):
r += int(h1[i] != h2[i])
if l == 0: return 0
return float(r) / float(l)
def calc_connhash(self, stream):
output = ""
for event in stream:
if event["in"]:
line = event["data"]
line = line.strip()
parts = line.split(" ")
for part in parts:
part_hash = chr(hash(part) % 0xFF)
output += part_hash
# Max db len is 256, half because of hex encoding
return output[:120]
@db_wrapper
def fill_db_ipranges(self):
if self.session.query(IPRange.ip_min).count() != 0:
return
print "Filling IPRange Tables"
asntable = ipdb.ipdb.get_asn()
progress = 0
for row in ipdb.ipdb.get_geo_iter():
progress += 1
if progress % 1000 == 0:
self.session.commit()
self.session.flush()
print str(100.0 * float(row[0]) / 4294967296.0) + "% / " + str(
100.0 * progress / 3315466) + "%"
ip = IPRange(ip_min=int(row[0]), ip_max=int(row[1]))
ip.country = row[2]
ip.region = row[4]
ip.city = row[5]
ip.zipcode = row[8]
ip.timezone = row[9]
ip.latitude = float(row[6])
ip.longitude = float(row[7])
asn_data = asntable.find_int(ip.ip_min)
if asn_data:
asn_id = int(asn_data[3])
asn_db = self.session.query(ASN).filter(
ASN.asn == asn_id).first()
if asn_db == None:
asn_db = ASN(asn=asn_id,
name=asn_data[4],
country=ip.country)
self.session.add(asn_db)
ip.asn = asn_db
# Dont add session if we cannot find an asn for it
self.session.add(ip)
print "IPranges loaded"
@db_wrapper
def get_ip_range_offline(self, ip):
ip_int = ipdb.ipdb.ipstr2int(ip)
range = self.session.query(IPRange).filter(
and_(IPRange.ip_min <= ip_int, ip_int <= IPRange.ip_max)).first()
return range
def get_ip_range_online(self, ip):
addinfo = additionalinfo.get_ip_info(ip)
if addinfo:
# TODO: Ugly hack
range = type('', (object, ), {})()
range.country = addinfo["country"]
range.city = "Unknown"
range.latitude = 0
range.longitude = 0
range.asn_id = int(addinfo["asn"])
range.asn = self._get_asn(range.asn_id)
range.cidr = addinfo["ipblock"]
return range
else:
return None
def get_ip_range(self, ip):
if self.ip2asn == "online":
return self.get_ip_range_online(ip)
else:
return self.get_ip_range_offline(ip)
def get_url_info(self, url):
parsed = urlparse.urlparse(url)
host = parsed.netloc.split(':')[0]
if host[0].isdigit():
ip = host
else:
try:
ip = socket.gethostbyname(host)
except:
return None
range = self.get_ip_range(ip)
return ip, range
@db_wrapper
def do_housekeeping(self):
for malware in self.session.query(Malware).all():
malware.name = random.choice(ANIMAL_NAMES)
# rebuild nb_firstconns
if False:
net_cache = {}
for conn in self.session.query(Connection).all():
if len(conn.conns_before) == 0:
if conn.network_id in net_cache:
net_cache[conn.network_id] += 1
else:
net_cache[conn.network_id] = 1
for network in self.session.query(Network).all():
if network.id in net_cache:
network.nb_firstconns = net_cache[network.id]
else:
network.nb_firstconns = 0
print "Net " + str(network.id) + ": " + str(
network.nb_firstconns)
@db_wrapper
def put_session(self, session):
connhash = self.calc_connhash(session["stream"]).encode("hex")
backend_user = self.session.query(User).filter(
User.username == session["backend_username"]).first()
conn = Connection(ip=session["ip"],
user=session["user"],
date=session["date"],
password=session["pass"],
stream=json.dumps(session["stream"]),
connhash=connhash,
backend_user_id=backend_user.id)
conn.user = filter_ascii(conn.user)
conn.password = filter_ascii(conn.password)
if self.do_ip_to_asn_resolution:
range = self.get_ip_range(conn.ip)
if range:
conn.country = range.country
conn.city = range.city
conn.lat = range.latitude
conn.lon = range.longitude
conn.asn = range.asn
self.session.add(conn)
self.session.flush() # to get id
network_id = None
samples = []
urls = []
for sample_json in session["samples"]:
# Ignore junk - may clean up the db a bit
if sample_json["length"] < 2000:
continue
sample, url = self.create_url_sample(sample_json)
if sample:
if network_id == None and sample.network_id != None:
network_id = sample.network_id
samples.append(sample)
if url:
if network_id == None and url.network_id != None:
network_id = url.network_id
conn.urls.append(url)
urls.append(url)
# Find previous connections
# A connection is associated when:
# - same honeypot/user
# - connection happened as long as 120s before
# - same client ip OR same username/password combo
assoc_timediff = 120
assoc_timediff_sameip = 3600
previous_conns = (self.session.query(Connection).filter(
or_(
and_(Connection.date > (conn.date - assoc_timediff),
Connection.user == conn.user,
Connection.password == conn.password),
and_(Connection.date > (conn.date - assoc_timediff_sameip),
Connection.ip == conn.ip)),
Connection.backend_user_id == conn.backend_user_id,
Connection.id != conn.id).all())
for prev in previous_conns:
if network_id == None and prev.network_id != None:
network_id = prev.network_id
conn.conns_before.append(prev)
# Check connection against all tags
tags = self.session.query(Tag).all()
for tag in tags:
json_obj = conn.json(depth=0)
json_obj["text_combined"] = filter_ascii(json_obj["text_combined"])
if simple_eval(tag.code, names=json_obj) == True:
self.db.link_conn_tag(conn.id, tag.id)
# Only create new networks for connections with urls or associtaed conns,
# to prevent the creation of thousands of networks
# NOTE: only conns with network == NULL will get their network updated
# later so whe should only create a network where we cannot easily
# change it later
haslogin = conn.user != None and conn.user != ""
if (len(conn.urls) > 0 or
len(previous_conns) > 0) and network_id == None and haslogin:
print(" --- create network --- ")
network_id = self.create_network().id
# Update network on self
conn.network_id = network_id
# Update network on all added Urls
for url in urls:
if url.network_id == None:
url.network_id = network_id
# Update network on all added Samples
for sample in samples:
if sample.network_id == None:
sample.network_id = network_id
# Update network on all previous connections withut one
if network_id != None:
for prev in previous_conns:
if prev.network_id == None:
prev.network_id = network_id
# Update number of first conns on network
if len(prev.conns_before) == 0:
conn.network.nb_firstconns += 1
self.session.flush()
# Check for Malware type
# only if our network exists AND has no malware associated
if conn.network != None and conn.network.malware == None:
# Find connections with similar connhash
similar_conns = (self.session.query(Connection).filter(
func.length(Connection.connhash) == len(connhash)).all())
min_sim = 2
min_conn = None
for similar in similar_conns:
if similar.network_id != None:
c1 = connhash.decode("hex")
c2 = similar.connhash.decode("hex")
sim = self.calc_connhash_similiarity(c1, c2)
if sim < min_sim and similar.network.malware != None:
min_sim = sim
min_conn = similar
# 0.9: 90% or more words in session are equal
# think this is probably the same kind of malware
# doesn't need to be the same botnet though!
if min_sim < 0.9:
conn.network.malware = min_conn.network.malware
else:
conn.network.malware = Malware()
conn.network.malware.name = random.choice(ANIMAL_NAMES)
self.session.add(conn.network.malware)
self.session.flush()
# Update network number of first connections
if len(previous_conns) == 0 and conn.network_id != None:
conn.network.nb_firstconns += 1
return conn.json(depth=1)
@db_wrapper
def create_network(self):
net = Network()
self.session.add(net)
self.session.flush()
return net
def create_url_sample(self, f):
url = self.session.query(Url).filter(Url.url == f["url"]).first()
if url == None:
url_ip = None
url_asn = None
url_country = None
if self.do_ip_to_asn_resolution:
url_ip, url_range = self.get_url_info(f["url"])
if url_range:
url_asn = url_range.asn_id
url_country = url_range.country
url = Url(url=f["url"],
date=f["date"],
ip=url_ip,
asn_id=url_asn,
country=url_country)
self.session.add(url)
if f["sha256"] != None:
sample = self.session.query(Sample).filter(
Sample.sha256 == f["sha256"]).first()
if sample == None:
result = None
try:
if self.vt != None:
vtobj = self.vt.query_hash_sha256(f["sha256"])
if vtobj:
result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]
) + " " + self.vt.get_best_result(vtobj)
except:
pass
sample = Sample(sha256=f["sha256"],
name=f["name"],
length=f["length"],
date=f["date"],
info=f["info"],
result=result)
self.session.add(sample)
if sample.network_id != None and url.network_id == None:
url.network_id = sample.network_id
if sample.network_id == None and url.network_id != None:
sample.network_id = url.network_id
else:
sample = None
url.sample = sample
return sample, url
@db_wrapper
def put_sample(self, data):
sha256 = hashlib.sha256(data).hexdigest()
self.db.put_sample_data(sha256, data)
if config.get("cuckoo_enabled"):
self.cuckoo.upload(os.path.join(config.get("sample_dir"), sha256),
sha256)
elif config.get("submit_to_vt"):
self.vt.upload_file(os.path.join(config.get("sample_dir"), sha256),
sha256)
@db_wrapper
def update_vt_result(self, sample_sha):
sample = self.session.query(Sample).filter(
Sample.sha256 == sample_sha).first()
if sample:
vtobj = self.vt.query_hash_sha256(sample_sha)
if vtobj:
sample.result = str(vtobj["positives"]) + "/" + str(
vtobj["total"]) + " " + self.vt.get_best_result(vtobj)
return sample.json(depth=1)
return None
def __init__(self):
self.session = None
self.vt = Virustotal(config.get("vt_key"))
self.cuckoo = Cuckoo(config)
def test_reset_timestamp(self):
cuckoo = Cuckoo()
cuckoo.reset()
now = datetime.now()
delta = datetime.now() - cuckoo.timestamp
self.assertGreater(delta.total_seconds(), 0)