def work(self):
while(True):
if(self.curJob == Job.IDLE and self.isInScanningEnviornment()):
# if no device found go for device search
if(len(getDatabase().get_basic_device_list())==0):
self.startDeviceSearch()
continue
# if lastScan more then 1 minutes ago scan
if(time.time() - self.lastScan > 60*1):
self.startNetworkScan()
continue
# if lastScan more then 5 minutes ago scan
if (time.time() - self.lastSearch > 60 * 5):
self.startDeviceSearch()
continue
# if devices exist that have not been fingerprinted fingerprint
tmp_cnt = False
for device in getDatabase().get_basic_device_list():
if not (device["last_time_deep_scan"] > 0):
self.startDeviceFingerprint(device)
tmp_cnt = True
break
if(tmp_cnt):
continue
# otherwise idle for a minute
self.startIdle()
else:
# idle if you cannot scan
logging.info("[i] not in home network")
self.startIdle()
def startDeviceFingerprint(self, device):
logging.info("[i] start device fingerpinting job")
self.curJob = Job.FINGERPRINT
device = getDeviceLibrary().fingerprint_device(device)
getDatabase().update_device_in_db(device, time.time())
logging.info("[i] finished device fingerpinting job")
self.curJob = Job.IDLE
def generate_places_json(self, device_list, scan_result):
"""
generate a plaaces json to visualize
:return:
"""
logging.info("[*] Lookup remote ips")
place_list = []
start = getDatabase().ip2loc("")
start["count"] = 0
start["src"] = ""
place_list.append(start)
for device in device_list:
test = scan_result[device["ip"]]
keys = scan_result[device["ip"]].items()
for key, value in test.items():
place = getDatabase().ip2loc(key)
place["count"] = value["count"]
place["threats"] = value["threats"]
place["src"] = device["ip"]
place_list.append(place)
final_json = "{\"type\": \"FeatureCollection\", \"features\": ["
final_json += ",".join(
map(
lambda x:
"{ \"type\": \"Feature\", \"properties\": { \"count\": " + str(
x["count"]) + ", \"device\":\"" + x["src"] +
"\"}, \"geometry\": { \"type\": \"Point\", \"coordinates\": ["
+ str(x["lat"]) + "," + str(x["lon"]) + "]}}", place_list))
final_json += "]}"
logging.info("[*] Write places")
text_file = open("scan_places.json", "w")
text_file.write(final_json)
text_file.close()
def get_internal_external():
timespan = 10000000
if ("timespan" in request.args):
timespan = int(request.args["timespan"])
conn, c = getDatabase().get_conn_cursor()
query = None
cidr = getDatabase().get_config("home_cidr")
first = IPNetwork(cidr).first
last = IPNetwork(cidr).first
if ("device" in request.args):
query = c.execute(
"SELECT Device.device_id, Device.device_name, Communication.internal, Device.mac, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY Device.device_id, Communication.internal",
(time.time() - timespan, int(request.args["device"])))
else:
query = c.execute(
"SELECT Device.device_id, Device.device_name, Communication.internal, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? GROUP BY Device.device_id, Communication.internal",
(time.time() - timespan, ))
entries = query.fetchall()
conn.close()
return json.dumps([dict(ix) for ix in entries])
def add_locations_to_device(self, device):
"""
adds location entry to scan_results
:param scan_result:
:return:
"""
device["location"] = getDatabase().ip2loc("")
if ("connections" in list(device.keys())):
for remote in device["connections"].keys():
device["connections"][remote]["location"] = getDatabase(
).ip2loc(remote)
def get_default_gateway(self):
import netifaces
gateways = netifaces.gateways()
default_gateway = gateways['default'][netifaces.AF_INET][0]
logging.info("[*] Default gateway identified: " + str(default_gateway))
if (getDatabase().get_config("home_mac") == None):
mac = self.get_mac(default_gateway)
getDatabase().insert_or_ignore_config("home_mac", mac)
getDatabase().insert_or_ignore_config("home_cidr",
default_gateway + "/24")
return default_gateway
def fingerprint_device(self, device):
"""
returns information concerning device and OS for a given IP
:param ip: to scan (schould be reachable)
:return: {"hostnames": [{"name":value}]}
"""
# check whether scanning device
own_ips = [
ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1]
if (device["ip"] in own_ips):
return {"hostnames": [{"name": platform.node()}]}
# otherwise perform scan
try:
logging.info("trying to perform deep scan and fingerprinting on " +
device["ip"])
nm = nmap.PortScanner()
# for name detection
#nm.scan(ip, arguments='-A')
# for OS detection
#nm.scan(device["ip"], arguments='-p 1-65535 -T4 -A -v ')
nm.scan(device["ip"], arguments='-T4 -A -v')
if not (nm[device["ip"]]["status"] == "down"):
logging.info("deep scan and fingerprinting successfull for " +
device["ip"])
sc_res = nm[device["ip"]]
if (len(sc_res['hostnames']) > 0):
device["name"] = sc_res['hostnames'][0]['name'].replace(
".lan", "")
if ('portsused' in sc_res):
device["ports"] = sc_res['portsused']
if ('tcp' in sc_res):
device["tcp"] = sc_res['tcp']
if ('osmatch' in sc_res):
device["osmatch"] = sc_res['osmatch']
if ('vendor' in sc_res):
device["vendor"] = sc_res['vendor']
device["fingerprint"] = 1
#self.save_device_list()
device["last_time_deep_scan"] = time.time()
getDatabase().update_device_in_db(device, time.time())
else:
logging.info(
"deep scan and fingerprinting failed (down) for " +
device["ip"])
return device
except KeyError:
print(KeyError)
return {}
def get_communication_history():
timespan = 10000000
if ("timespan" in request.args):
timespan = int(request.args["timespan"])
conn, c = getDatabase().get_conn_cursor()
query = c.execute(
"SELECT latitude as lat, longitude as lon, city, COUNTRY as CC FROM IPLocation WHERE ip = 0"
)
locations = query.fetchall()
if (len(locations) < 1):
return "[{},{}]"
json_location = json.dumps([dict(ix) for ix in locations][0])
query = None
if ("device" in request.args):
query = c.execute(
"SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip WHERE Communication.time > ? AND Device.device_id = ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC",
(time.time() - timespan, int(request.args["device"])))
else:
query = c.execute(
"SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports, mac FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip WHERE Communication.time > ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC",
(time.time() - timespan, ))
entries = query.fetchall()
conn.close()
return "[" + json_location + "," + json.dumps([dict(ix)
for ix in entries]) + "]"
def startNetworkScan(self):
logging.info("[i] start network scanning job")
self.curJob = Job.NETWORK_CAPTURE
getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 1)
getNetworkScanner().print_device_list(getDeviceLibrary().device_list)
packets = getNetworkScanner().sniff_devices(getDatabase().get_basic_device_list(), timeout=int(getDatabase().get_config("scan_duration")))
self.curJob = Job.NETWORK_ANALYZE
getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 0)
scan_result = getTrafficAnalyzer().analyze(packets, getDatabase().get_basic_device_list())
getDeviceLibrary().save_scan_result(scan_result)
getDatabase().save_scan_result(scan_result)
# remove packets from memory (can be huge!)
del packets
self.lastScan = time.time()
self.curJob = Job.IDLE
def arp_scan_network_for_devices(self):
""" Returns a list of devices which are currently active on the network """
devices = []
default_gateway = getNetworkScanner().get_default_gateway()
ip = getDatabase().get_config("home_cidr")
logging.info("[*] Scanning network (" + str(ip) +
") for devices (this takes up to 30 seconds)")
answered_list = srp(Ether(dst="ff:ff:ff:ff:ff:ff") / ARP(pdst=ip),
timeout=2,
retry=10)[0]
for element in answered_list:
if element[1].psrc != default_gateway:
device = {}
device["ip"] = element[1].psrc
device["mac"] = element[1].hwsrc.upper()
device["name"] = ""
device["scanning"] = 0
devices.append(device)
#self.save_device_list()
getDatabase().save_network_scan(devices)
return devices
def get_network_usage():
timespan = 10000000
if ("timespan" in request.args):
timespan = int(request.args["timespan"])
conn, c = getDatabase().get_conn_cursor()
query = None
if ("device" in request.args):
query = c.execute(
"SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY time, Communication.device_id ORDER BY time DESC",
(time.time() - timespan, int(request.args["device"])))
else:
query = c.execute(
"SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? GROUP BY time, Communication.device_id ORDER BY time DESC",
(time.time() - timespan, ))
entries = query.fetchall()
conn.close()
return json.dumps([dict(ix) for ix in entries])
async def scan(self, websocket):
cnt = 0
default_gateway = getNetworkScanner().get_default_gateway()
cidr = getDatabase().get_config("home_cidr")
last_update = 0
while (True):
if (last_update < getJobScheduler().getLastUpdate()):
last_update = getJobScheduler().getLastUpdate()
scan_result = {
"devices": getDeviceLibrary().device_list,
"cidr": cidr
}
await websocket.send(
"{\"type\":\"scan_result\", \"payload\":" +
json.dumps(scan_result) + "}")
await websocket.send("{\"type\":\"history\", \"payload\":" +
json.dumps(getDeviceLibrary().history) +
"}")
time.sleep(1)
def __init__(self):
self.thread = threading.Thread(target=self.work, name="JobScheduler")
self.thread.start()
getDatabase().initialize()
def analyze(self, packets_test, device_list):
#own_ip = get('https://api.ipify.org').text
scan_duration = int(getDatabase().get_config("scan_duration"))
self.ana_res = {}
for device in device_list:
if (device["ip"] not in self.ana_res):
self.ana_res[device["ip"]] = {}
default_gateway = getNetworkScanner().get_default_gateway()
cidr = getDatabase().get_config("home_cidr")
logging.info("[*] Start traffic analysis")
logging.info(" |--> IP traffic analysis")
logging.info("packets: " + str(len(packets_test)))
bytestotal = 0
# acclumerate IP packet information
self.tot_p_count += len(packets_test)
for pkt in packets_test:
"""if HTTP in pkt:
http_layer = pkt.getlayer(HTTP)
pkt.show()"""
if IP in pkt:
send = True
ip_src = pkt[IP].src
ip_dst = pkt[IP].dst
foreign_ip = None
local_ip = None
hport = None
pport = None
bytestotal += len(pkt)
if (ip_src in self.ana_res):
foreign_ip = ip_dst
local_ip = ip_src
if TCP in pkt:
hport = pkt[TCP].sport
pport = pkt[TCP].dport
send = True
if (ip_dst in self.ana_res):
foreign_ip = ip_src
local_ip = ip_dst
if TCP in pkt:
hport = pkt[TCP].dport
pport = pkt[TCP].sport
send = False
if (local_ip != None and foreign_ip != None):
if (foreign_ip not in self.ana_res[local_ip]):
if (send):
self.ana_res[local_ip][foreign_ip] = {
"snd": 1 / scan_duration,
"rcv": 0,
"bsnd": len(pkt),
"brcv": 0,
"domains": [],
"threats": [],
"hports": {hport},
"pports": {pport}
}
else:
self.ana_res[local_ip][foreign_ip] = {
"snd": 0,
"rcv": 1 / scan_duration,
"bsnd": len(pkt),
"brcv": 0,
"domains": [],
"threats": [],
"hports": {hport},
"pports": {pport}
}
else:
self.ana_res[local_ip][foreign_ip][
"hports"] = self.ana_res[local_ip][foreign_ip][
"hports"] | {hport}
self.ana_res[local_ip][foreign_ip][
"pports"] = self.ana_res[local_ip][foreign_ip][
"pports"] | {pport}
if (send):
self.ana_res[local_ip][foreign_ip][
"snd"] += 1 / scan_duration
self.ana_res[local_ip][foreign_ip]["bsnd"] += len(
pkt) / scan_duration
else:
self.ana_res[local_ip][foreign_ip][
"rcv"] += 1 / scan_duration
self.ana_res[local_ip][foreign_ip]["brcv"] += len(
pkt) / scan_duration
# perform IP check
#collect all ips
all_ips = set()
for key in self.ana_res.keys():
for ip in self.ana_res[key]:
all_ips.add(ip)
ip_threats = getIPThreatDatabase().check_IPs(all_ips)
for key in self.ana_res.keys():
for ip in self.ana_res[key]:
# Check for malicious ips
if (ip in ip_threats):
self.ana_res[key][ip]["threats"].append(ip_threats[ip])
# Perform reverse dns lookup
if (len(self.ana_res[key][ip]["domains"]) == 0):
try:
reversed_dns = socket.gethostbyaddr(ip)
if (len(reversed_dns) > 0):
getDatabase().submit_ip_domain(ip, reversed_dns[0])
self.ana_res[key][ip]["domains"].append(
reversed_dns[0])
except Exception:
logging.exception("no reverse dns for " + ip)
logging.info(" |--> DNS traffic analysis")
if (int(getDatabase().get_config("DNS_inspect")) > 0):
logging.info(" DNS traffic analysis enabled")
try:
for pkt in packets_test:
if pkt.haslayer(DNSRR):
# If the answer is a DNSRR, print the name it replied with.
if isinstance(pkt.an, DNSRR):
for key in self.ana_res.keys():
if (pkt.an.rdata in self.ana_res[key]):
getDatabase().submit_ip_domain(
pkt.an.rdata,
pkt.an.rrname.decode("utf-8"))
if (pkt.an.rrname.decode("utf-8")
in ';;;'.join(self.ana_res[key][
pkt.an.rdata]["domains"])):
self.ana_res[key][
pkt.an.rdata]["domains"].append(
pkt.an.rrname.decode("utf-8"))
# TODO: perform malicious dns lookup
except Exception:
logging.exception("Exception during DNS traffic analysis",
Exception)
else:
logging.info(" DNS traffic analysis disabled")
getDatabase().insert_or_replace_config(
"num_pack_scan",
int(getDatabase().get_config("num_pack_scan")) + len(packets_test))
getDatabase().insert_or_replace_config(
"num_bytes_scan",
int(getDatabase().get_config("num_bytes_scan")) + bytestotal)
logging.info("[*] Traffic analysis completed")
for device in device_list:
for rip in list(self.ana_res[device["ip"]].keys()):
self.ana_res[device["ip"]][rip]["hports"] = list(
self.ana_res[device["ip"]][rip]["hports"])
self.ana_res[device["ip"]][rip]["pports"] = list(
self.ana_res[device["ip"]][rip]["pports"])
device["connections"] = self.ana_res[device["ip"]]
getNetworkScanner().add_locations_to_device(device)
return {
"devices": copy.deepcopy(device_list),
"cidr": cidr,
"time": time.time()
}
#if __name__ == "__main__":
# test = getTrafficAnalyzer()
# packets = rdpcap('full_capture.pcap')
# print(test.analyze(packets, [{"ip":"192.168.8.113"},{"ip":"192.168.8.101"},{"ip":"192.168.8.1"}]))
def isInScanningEnviornment(self):
cur_def = getNetworkScanner().get_mac(getNetworkScanner().get_default_gateway())
stored_def = getDatabase().get_config("home_mac")
return cur_def == stored_def or cur_def == None
def get_devices():
devices = getDatabase().get_basic_device_list()
return json.dumps(devices)
def nmap_scan_and_fingerprint(self):
devices = []
default_gateway = getNetworkScanner().get_default_gateway()
cidr = getDatabase().get_config("home_cidr")
print("[*] Scanning network (" + str(cidr) +
") for devices (this takes up to 60 seconds)")
nm = nmap.PortScanner()
# for name detection
nm.scan(cidr, arguments='-sV -T4 -O -F --version-light')
for ip in nm._scan_result['scan']:
if ('mac' in nm[ip]['addresses']):
client_dict = {
"ip": ip,
"mac": (nm[ip]['addresses']['mac']).upper(),
"scanning": 0
}
already_found = False
for device in devices:
if (device["ip"] == ip):
client_dict = device
already_found = True
if (len(nm[ip]['hostnames']) > 0):
client_dict["name"] = nm[ip]['hostnames'][0][
'name'].replace(".lan", "")
if ('portsused' in nm[ip]):
client_dict["ports"] = nm[ip]['portsused']
if ('tcp' in nm[ip]):
client_dict["tcp"] = nm[ip]['tcp']
if ('osmatch' in nm[ip]):
client_dict["osmatch"] = nm[ip]['osmatch']
devices.append(client_dict)
else:
own_ips = [
ip
for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1]
#TODO make it work
if (ip in own_ips or True):
client_dict = {
"ip": ip,
"mac": str(get_mac()),
"scanning": 0
}
if (len(nm[ip]['hostnames']) > 0):
client_dict["name"] = nm[ip]['hostnames'][0][
'name'].replace(".lan", "")
if ('portsused' in nm[ip]):
client_dict["ports"] = nm[ip]['portsused']
if ('tcp' in nm[ip]):
client_dict["tcp"] = nm[ip]['tcp']
if ('osmatch' in nm[ip]):
client_dict["osmatch"] = nm[ip]['osmatch']
else:
print(
"WARNING: a device (" + ip +
") has been ignored because no MAC address was found")
del nm
#self.save_device_list()
getDatabase().save_network_scan(devices)
return devices
def get_config():
configs = getDatabase().get_all_configs()
return json.dumps(configs)
def update_config():
name = request.args.get('name')
value = request.args.get('value')
getDatabase().insert_or_replace_config(name, value)
return ""
