def work(self): while(True): if(self.curJob == Job.IDLE and self.isInScanningEnviornment()): # if no device found go for device search if(len(getDatabase().get_basic_device_list())==0): self.startDeviceSearch() continue # if lastScan more then 1 minutes ago scan if(time.time() - self.lastScan > 60*1): self.startNetworkScan() continue # if lastScan more then 5 minutes ago scan if (time.time() - self.lastSearch > 60 * 5): self.startDeviceSearch() continue # if devices exist that have not been fingerprinted fingerprint tmp_cnt = False for device in getDatabase().get_basic_device_list(): if not (device["last_time_deep_scan"] > 0): self.startDeviceFingerprint(device) tmp_cnt = True break if(tmp_cnt): continue # otherwise idle for a minute self.startIdle() else: # idle if you cannot scan logging.info("[i] not in home network") self.startIdle()
def startDeviceFingerprint(self, device): logging.info("[i] start device fingerpinting job") self.curJob = Job.FINGERPRINT device = getDeviceLibrary().fingerprint_device(device) getDatabase().update_device_in_db(device, time.time()) logging.info("[i] finished device fingerpinting job") self.curJob = Job.IDLE
def generate_places_json(self, device_list, scan_result): """ generate a plaaces json to visualize :return: """ logging.info("[*] Lookup remote ips") place_list = [] start = getDatabase().ip2loc("") start["count"] = 0 start["src"] = "" place_list.append(start) for device in device_list: test = scan_result[device["ip"]] keys = scan_result[device["ip"]].items() for key, value in test.items(): place = getDatabase().ip2loc(key) place["count"] = value["count"] place["threats"] = value["threats"] place["src"] = device["ip"] place_list.append(place) final_json = "{\"type\": \"FeatureCollection\", \"features\": [" final_json += ",".join( map( lambda x: "{ \"type\": \"Feature\", \"properties\": { \"count\": " + str( x["count"]) + ", \"device\":\"" + x["src"] + "\"}, \"geometry\": { \"type\": \"Point\", \"coordinates\": [" + str(x["lat"]) + "," + str(x["lon"]) + "]}}", place_list)) final_json += "]}" logging.info("[*] Write places") text_file = open("scan_places.json", "w") text_file.write(final_json) text_file.close()
def get_internal_external(): timespan = 10000000 if ("timespan" in request.args): timespan = int(request.args["timespan"]) conn, c = getDatabase().get_conn_cursor() query = None cidr = getDatabase().get_config("home_cidr") first = IPNetwork(cidr).first last = IPNetwork(cidr).first if ("device" in request.args): query = c.execute( "SELECT Device.device_id, Device.device_name, Communication.internal, Device.mac, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY Device.device_id, Communication.internal", (time.time() - timespan, int(request.args["device"]))) else: query = c.execute( "SELECT Device.device_id, Device.device_name, Communication.internal, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? GROUP BY Device.device_id, Communication.internal", (time.time() - timespan, )) entries = query.fetchall() conn.close() return json.dumps([dict(ix) for ix in entries])
def add_locations_to_device(self, device): """ adds location entry to scan_results :param scan_result: :return: """ device["location"] = getDatabase().ip2loc("") if ("connections" in list(device.keys())): for remote in device["connections"].keys(): device["connections"][remote]["location"] = getDatabase( ).ip2loc(remote)
def get_default_gateway(self): import netifaces gateways = netifaces.gateways() default_gateway = gateways['default'][netifaces.AF_INET][0] logging.info("[*] Default gateway identified: " + str(default_gateway)) if (getDatabase().get_config("home_mac") == None): mac = self.get_mac(default_gateway) getDatabase().insert_or_ignore_config("home_mac", mac) getDatabase().insert_or_ignore_config("home_cidr", default_gateway + "/24") return default_gateway
def fingerprint_device(self, device): """ returns information concerning device and OS for a given IP :param ip: to scan (schould be reachable) :return: {"hostnames": [{"name":value}]} """ # check whether scanning device own_ips = [ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1] if (device["ip"] in own_ips): return {"hostnames": [{"name": platform.node()}]} # otherwise perform scan try: logging.info("trying to perform deep scan and fingerprinting on " + device["ip"]) nm = nmap.PortScanner() # for name detection #nm.scan(ip, arguments='-A') # for OS detection #nm.scan(device["ip"], arguments='-p 1-65535 -T4 -A -v ') nm.scan(device["ip"], arguments='-T4 -A -v') if not (nm[device["ip"]]["status"] == "down"): logging.info("deep scan and fingerprinting successfull for " + device["ip"]) sc_res = nm[device["ip"]] if (len(sc_res['hostnames']) > 0): device["name"] = sc_res['hostnames'][0]['name'].replace( ".lan", "") if ('portsused' in sc_res): device["ports"] = sc_res['portsused'] if ('tcp' in sc_res): device["tcp"] = sc_res['tcp'] if ('osmatch' in sc_res): device["osmatch"] = sc_res['osmatch'] if ('vendor' in sc_res): device["vendor"] = sc_res['vendor'] device["fingerprint"] = 1 #self.save_device_list() device["last_time_deep_scan"] = time.time() getDatabase().update_device_in_db(device, time.time()) else: logging.info( "deep scan and fingerprinting failed (down) for " + device["ip"]) return device except KeyError: print(KeyError) return {}
def get_communication_history(): timespan = 10000000 if ("timespan" in request.args): timespan = int(request.args["timespan"]) conn, c = getDatabase().get_conn_cursor() query = c.execute( "SELECT latitude as lat, longitude as lon, city, COUNTRY as CC FROM IPLocation WHERE ip = 0" ) locations = query.fetchall() if (len(locations) < 1): return "[{},{}]" json_location = json.dumps([dict(ix) for ix in locations][0]) query = None if ("device" in request.args): query = c.execute( "SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip WHERE Communication.time > ? AND Device.device_id = ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC", (time.time() - timespan, int(request.args["device"]))) else: query = c.execute( "SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports, mac FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip WHERE Communication.time > ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC", (time.time() - timespan, )) entries = query.fetchall() conn.close() return "[" + json_location + "," + json.dumps([dict(ix) for ix in entries]) + "]"
def startNetworkScan(self): logging.info("[i] start network scanning job") self.curJob = Job.NETWORK_CAPTURE getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 1) getNetworkScanner().print_device_list(getDeviceLibrary().device_list) packets = getNetworkScanner().sniff_devices(getDatabase().get_basic_device_list(), timeout=int(getDatabase().get_config("scan_duration"))) self.curJob = Job.NETWORK_ANALYZE getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 0) scan_result = getTrafficAnalyzer().analyze(packets, getDatabase().get_basic_device_list()) getDeviceLibrary().save_scan_result(scan_result) getDatabase().save_scan_result(scan_result) # remove packets from memory (can be huge!) del packets self.lastScan = time.time() self.curJob = Job.IDLE
def arp_scan_network_for_devices(self): """ Returns a list of devices which are currently active on the network """ devices = [] default_gateway = getNetworkScanner().get_default_gateway() ip = getDatabase().get_config("home_cidr") logging.info("[*] Scanning network (" + str(ip) + ") for devices (this takes up to 30 seconds)") answered_list = srp(Ether(dst="ff:ff:ff:ff:ff:ff") / ARP(pdst=ip), timeout=2, retry=10)[0] for element in answered_list: if element[1].psrc != default_gateway: device = {} device["ip"] = element[1].psrc device["mac"] = element[1].hwsrc.upper() device["name"] = "" device["scanning"] = 0 devices.append(device) #self.save_device_list() getDatabase().save_network_scan(devices) return devices
def get_network_usage(): timespan = 10000000 if ("timespan" in request.args): timespan = int(request.args["timespan"]) conn, c = getDatabase().get_conn_cursor() query = None if ("device" in request.args): query = c.execute( "SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY time, Communication.device_id ORDER BY time DESC", (time.time() - timespan, int(request.args["device"]))) else: query = c.execute( "SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? GROUP BY time, Communication.device_id ORDER BY time DESC", (time.time() - timespan, )) entries = query.fetchall() conn.close() return json.dumps([dict(ix) for ix in entries])
async def scan(self, websocket): cnt = 0 default_gateway = getNetworkScanner().get_default_gateway() cidr = getDatabase().get_config("home_cidr") last_update = 0 while (True): if (last_update < getJobScheduler().getLastUpdate()): last_update = getJobScheduler().getLastUpdate() scan_result = { "devices": getDeviceLibrary().device_list, "cidr": cidr } await websocket.send( "{\"type\":\"scan_result\", \"payload\":" + json.dumps(scan_result) + "}") await websocket.send("{\"type\":\"history\", \"payload\":" + json.dumps(getDeviceLibrary().history) + "}") time.sleep(1)
def __init__(self): self.thread = threading.Thread(target=self.work, name="JobScheduler") self.thread.start() getDatabase().initialize()
def analyze(self, packets_test, device_list): #own_ip = get('https://api.ipify.org').text scan_duration = int(getDatabase().get_config("scan_duration")) self.ana_res = {} for device in device_list: if (device["ip"] not in self.ana_res): self.ana_res[device["ip"]] = {} default_gateway = getNetworkScanner().get_default_gateway() cidr = getDatabase().get_config("home_cidr") logging.info("[*] Start traffic analysis") logging.info(" |--> IP traffic analysis") logging.info("packets: " + str(len(packets_test))) bytestotal = 0 # acclumerate IP packet information self.tot_p_count += len(packets_test) for pkt in packets_test: """if HTTP in pkt: http_layer = pkt.getlayer(HTTP) pkt.show()""" if IP in pkt: send = True ip_src = pkt[IP].src ip_dst = pkt[IP].dst foreign_ip = None local_ip = None hport = None pport = None bytestotal += len(pkt) if (ip_src in self.ana_res): foreign_ip = ip_dst local_ip = ip_src if TCP in pkt: hport = pkt[TCP].sport pport = pkt[TCP].dport send = True if (ip_dst in self.ana_res): foreign_ip = ip_src local_ip = ip_dst if TCP in pkt: hport = pkt[TCP].dport pport = pkt[TCP].sport send = False if (local_ip != None and foreign_ip != None): if (foreign_ip not in self.ana_res[local_ip]): if (send): self.ana_res[local_ip][foreign_ip] = { "snd": 1 / scan_duration, "rcv": 0, "bsnd": len(pkt), "brcv": 0, "domains": [], "threats": [], "hports": {hport}, "pports": {pport} } else: self.ana_res[local_ip][foreign_ip] = { "snd": 0, "rcv": 1 / scan_duration, "bsnd": len(pkt), "brcv": 0, "domains": [], "threats": [], "hports": {hport}, "pports": {pport} } else: self.ana_res[local_ip][foreign_ip][ "hports"] = self.ana_res[local_ip][foreign_ip][ "hports"] | {hport} self.ana_res[local_ip][foreign_ip][ "pports"] = self.ana_res[local_ip][foreign_ip][ "pports"] | {pport} if (send): self.ana_res[local_ip][foreign_ip][ "snd"] += 1 / scan_duration self.ana_res[local_ip][foreign_ip]["bsnd"] += len( pkt) / scan_duration else: self.ana_res[local_ip][foreign_ip][ "rcv"] += 1 / scan_duration self.ana_res[local_ip][foreign_ip]["brcv"] += len( pkt) / scan_duration # perform IP check #collect all ips all_ips = set() for key in self.ana_res.keys(): for ip in self.ana_res[key]: all_ips.add(ip) ip_threats = getIPThreatDatabase().check_IPs(all_ips) for key in self.ana_res.keys(): for ip in self.ana_res[key]: # Check for malicious ips if (ip in ip_threats): self.ana_res[key][ip]["threats"].append(ip_threats[ip]) # Perform reverse dns lookup if (len(self.ana_res[key][ip]["domains"]) == 0): try: reversed_dns = socket.gethostbyaddr(ip) if (len(reversed_dns) > 0): getDatabase().submit_ip_domain(ip, reversed_dns[0]) self.ana_res[key][ip]["domains"].append( reversed_dns[0]) except Exception: logging.exception("no reverse dns for " + ip) logging.info(" |--> DNS traffic analysis") if (int(getDatabase().get_config("DNS_inspect")) > 0): logging.info(" DNS traffic analysis enabled") try: for pkt in packets_test: if pkt.haslayer(DNSRR): # If the answer is a DNSRR, print the name it replied with. if isinstance(pkt.an, DNSRR): for key in self.ana_res.keys(): if (pkt.an.rdata in self.ana_res[key]): getDatabase().submit_ip_domain( pkt.an.rdata, pkt.an.rrname.decode("utf-8")) if (pkt.an.rrname.decode("utf-8") in ';;;'.join(self.ana_res[key][ pkt.an.rdata]["domains"])): self.ana_res[key][ pkt.an.rdata]["domains"].append( pkt.an.rrname.decode("utf-8")) # TODO: perform malicious dns lookup except Exception: logging.exception("Exception during DNS traffic analysis", Exception) else: logging.info(" DNS traffic analysis disabled") getDatabase().insert_or_replace_config( "num_pack_scan", int(getDatabase().get_config("num_pack_scan")) + len(packets_test)) getDatabase().insert_or_replace_config( "num_bytes_scan", int(getDatabase().get_config("num_bytes_scan")) + bytestotal) logging.info("[*] Traffic analysis completed") for device in device_list: for rip in list(self.ana_res[device["ip"]].keys()): self.ana_res[device["ip"]][rip]["hports"] = list( self.ana_res[device["ip"]][rip]["hports"]) self.ana_res[device["ip"]][rip]["pports"] = list( self.ana_res[device["ip"]][rip]["pports"]) device["connections"] = self.ana_res[device["ip"]] getNetworkScanner().add_locations_to_device(device) return { "devices": copy.deepcopy(device_list), "cidr": cidr, "time": time.time() } #if __name__ == "__main__": # test = getTrafficAnalyzer() # packets = rdpcap('full_capture.pcap') # print(test.analyze(packets, [{"ip":"192.168.8.113"},{"ip":"192.168.8.101"},{"ip":"192.168.8.1"}]))
def isInScanningEnviornment(self): cur_def = getNetworkScanner().get_mac(getNetworkScanner().get_default_gateway()) stored_def = getDatabase().get_config("home_mac") return cur_def == stored_def or cur_def == None
def get_devices(): devices = getDatabase().get_basic_device_list() return json.dumps(devices)
def nmap_scan_and_fingerprint(self): devices = [] default_gateway = getNetworkScanner().get_default_gateway() cidr = getDatabase().get_config("home_cidr") print("[*] Scanning network (" + str(cidr) + ") for devices (this takes up to 60 seconds)") nm = nmap.PortScanner() # for name detection nm.scan(cidr, arguments='-sV -T4 -O -F --version-light') for ip in nm._scan_result['scan']: if ('mac' in nm[ip]['addresses']): client_dict = { "ip": ip, "mac": (nm[ip]['addresses']['mac']).upper(), "scanning": 0 } already_found = False for device in devices: if (device["ip"] == ip): client_dict = device already_found = True if (len(nm[ip]['hostnames']) > 0): client_dict["name"] = nm[ip]['hostnames'][0][ 'name'].replace(".lan", "") if ('portsused' in nm[ip]): client_dict["ports"] = nm[ip]['portsused'] if ('tcp' in nm[ip]): client_dict["tcp"] = nm[ip]['tcp'] if ('osmatch' in nm[ip]): client_dict["osmatch"] = nm[ip]['osmatch'] devices.append(client_dict) else: own_ips = [ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1] #TODO make it work if (ip in own_ips or True): client_dict = { "ip": ip, "mac": str(get_mac()), "scanning": 0 } if (len(nm[ip]['hostnames']) > 0): client_dict["name"] = nm[ip]['hostnames'][0][ 'name'].replace(".lan", "") if ('portsused' in nm[ip]): client_dict["ports"] = nm[ip]['portsused'] if ('tcp' in nm[ip]): client_dict["tcp"] = nm[ip]['tcp'] if ('osmatch' in nm[ip]): client_dict["osmatch"] = nm[ip]['osmatch'] else: print( "WARNING: a device (" + ip + ") has been ignored because no MAC address was found") del nm #self.save_device_list() getDatabase().save_network_scan(devices) return devices
def get_config(): configs = getDatabase().get_all_configs() return json.dumps(configs)
def update_config(): name = request.args.get('name') value = request.args.get('value') getDatabase().insert_or_replace_config(name, value) return ""