Ejemplo n.º 1
0
    def work(self):
        while(True):
            if(self.curJob == Job.IDLE and  self.isInScanningEnviornment()):
                # if no device found go for device search
                if(len(getDatabase().get_basic_device_list())==0):
                    self.startDeviceSearch()
                    continue

                # if lastScan more then 1 minutes ago scan
                if(time.time() - self.lastScan > 60*1):
                    self.startNetworkScan()
                    continue

                # if lastScan more then 5 minutes ago scan
                if (time.time() - self.lastSearch > 60 * 5):
                    self.startDeviceSearch()
                    continue

                # if devices exist that have not been fingerprinted fingerprint
                tmp_cnt = False
                for device in getDatabase().get_basic_device_list():
                    if not (device["last_time_deep_scan"] > 0):
                        self.startDeviceFingerprint(device)
                        tmp_cnt = True
                        break
                if(tmp_cnt):
                    continue

                # otherwise idle for a minute
                self.startIdle()
            else:
                # idle if you cannot scan
                logging.info("[i] not in home network")
                self.startIdle()
Ejemplo n.º 2
0
 def startDeviceFingerprint(self, device):
     logging.info("[i] start device fingerpinting job")
     self.curJob = Job.FINGERPRINT
     device = getDeviceLibrary().fingerprint_device(device)
     getDatabase().update_device_in_db(device, time.time())
     logging.info("[i] finished device fingerpinting job")
     self.curJob = Job.IDLE
Ejemplo n.º 3
0
    def generate_places_json(self, device_list, scan_result):
        """
        generate a plaaces json to visualize
        :return:
        """
        logging.info("[*] Lookup remote ips")
        place_list = []
        start = getDatabase().ip2loc("")
        start["count"] = 0
        start["src"] = ""
        place_list.append(start)
        for device in device_list:
            test = scan_result[device["ip"]]
            keys = scan_result[device["ip"]].items()
            for key, value in test.items():
                place = getDatabase().ip2loc(key)
                place["count"] = value["count"]
                place["threats"] = value["threats"]
                place["src"] = device["ip"]
                place_list.append(place)

        final_json = "{\"type\": \"FeatureCollection\", \"features\": ["
        final_json += ",".join(
            map(
                lambda x:
                "{ \"type\": \"Feature\", \"properties\": { \"count\": " + str(
                    x["count"]) + ", \"device\":\"" + x["src"] +
                "\"}, \"geometry\": { \"type\": \"Point\", \"coordinates\": ["
                + str(x["lat"]) + "," + str(x["lon"]) + "]}}", place_list))
        final_json += "]}"
        logging.info("[*] Write places")
        text_file = open("scan_places.json", "w")
        text_file.write(final_json)
        text_file.close()
Ejemplo n.º 4
0
def get_internal_external():
    timespan = 10000000
    if ("timespan" in request.args):
        timespan = int(request.args["timespan"])
    conn, c = getDatabase().get_conn_cursor()

    query = None

    cidr = getDatabase().get_config("home_cidr")
    first = IPNetwork(cidr).first
    last = IPNetwork(cidr).first

    if ("device" in request.args):
        query = c.execute(
            "SELECT Device.device_id, Device.device_name, Communication.internal, Device.mac, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY Device.device_id, Communication.internal",
            (time.time() - timespan, int(request.args["device"])))
    else:
        query = c.execute(
            "SELECT Device.device_id, Device.device_name, Communication.internal,  Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ?  GROUP BY Device.device_id, Communication.internal",
            (time.time() - timespan, ))

    entries = query.fetchall()

    conn.close()

    return json.dumps([dict(ix) for ix in entries])
Ejemplo n.º 5
0
 def add_locations_to_device(self, device):
     """
     adds location entry to scan_results
     :param scan_result:
     :return:
     """
     device["location"] = getDatabase().ip2loc("")
     if ("connections" in list(device.keys())):
         for remote in device["connections"].keys():
             device["connections"][remote]["location"] = getDatabase(
             ).ip2loc(remote)
Ejemplo n.º 6
0
    def get_default_gateway(self):
        import netifaces

        gateways = netifaces.gateways()
        default_gateway = gateways['default'][netifaces.AF_INET][0]
        logging.info("[*] Default gateway identified: " + str(default_gateway))
        if (getDatabase().get_config("home_mac") == None):
            mac = self.get_mac(default_gateway)
            getDatabase().insert_or_ignore_config("home_mac", mac)
            getDatabase().insert_or_ignore_config("home_cidr",
                                                  default_gateway + "/24")
        return default_gateway
Ejemplo n.º 7
0
    def fingerprint_device(self, device):
        """
        returns information concerning device and OS for a given IP
        :param ip: to scan (schould be reachable)
        :return: {"hostnames": [{"name":value}]}
        """

        # check whether scanning device
        own_ips = [
            ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
            if not ip.startswith("127.")
        ][:1]
        if (device["ip"] in own_ips):
            return {"hostnames": [{"name": platform.node()}]}

        # otherwise perform scan
        try:
            logging.info("trying to perform deep scan and fingerprinting on " +
                         device["ip"])
            nm = nmap.PortScanner()
            # for name detection
            #nm.scan(ip, arguments='-A')
            # for OS detection
            #nm.scan(device["ip"], arguments='-p 1-65535 -T4 -A -v ')
            nm.scan(device["ip"], arguments='-T4 -A -v')
            if not (nm[device["ip"]]["status"] == "down"):
                logging.info("deep scan and fingerprinting successfull for " +
                             device["ip"])
                sc_res = nm[device["ip"]]
                if (len(sc_res['hostnames']) > 0):
                    device["name"] = sc_res['hostnames'][0]['name'].replace(
                        ".lan", "")
                if ('portsused' in sc_res):
                    device["ports"] = sc_res['portsused']
                if ('tcp' in sc_res):
                    device["tcp"] = sc_res['tcp']
                if ('osmatch' in sc_res):
                    device["osmatch"] = sc_res['osmatch']
                if ('vendor' in sc_res):
                    device["vendor"] = sc_res['vendor']
                device["fingerprint"] = 1
                #self.save_device_list()
                device["last_time_deep_scan"] = time.time()
                getDatabase().update_device_in_db(device, time.time())
            else:
                logging.info(
                    "deep scan and fingerprinting failed (down) for " +
                    device["ip"])
            return device
        except KeyError:
            print(KeyError)
            return {}
Ejemplo n.º 8
0
def get_communication_history():
    timespan = 10000000
    if ("timespan" in request.args):
        timespan = int(request.args["timespan"])
    conn, c = getDatabase().get_conn_cursor()

    query = c.execute(
        "SELECT latitude as lat, longitude as lon, city, COUNTRY as CC FROM IPLocation WHERE ip = 0"
    )
    locations = query.fetchall()
    if (len(locations) < 1):
        return "[{},{}]"

    json_location = json.dumps([dict(ix) for ix in locations][0])

    query = None

    if ("device" in request.args):
        query = c.execute(
            "SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip WHERE Communication.time > ? AND Device.device_id = ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC",
            (time.time() - timespan, int(request.args["device"])))
    else:
        query = c.execute(
            "SELECT Device.device_id, GROUP_CONCAT(DISTINCT IPDNS.name) as domains, Device.device_name, external_ip, SUM(bsnd)/COUNT(external_ip) as bsnd_avg, SUM(brcv)/COUNT(external_ip) as brcv_avg, latitude as lat, longitude as lon, city, COUNTRY as CC , hports, pports, mac FROM Communication INNER JOIN Device ON Communication.device_id = Device.device_id INNER JOIN IPLocation ON IPLocation.ip = Communication.external_ip  INNER JOIN IPDNS ON IPDNS.ip = Communication.external_ip  WHERE Communication.time > ? GROUP BY external_ip ORDER BY bsnd_avg+brcv_avg DESC",
            (time.time() - timespan, ))

    entries = query.fetchall()

    conn.close()

    return "[" + json_location + "," + json.dumps([dict(ix)
                                                   for ix in entries]) + "]"
Ejemplo n.º 9
0
    def startNetworkScan(self):
        logging.info("[i] start network scanning job")
        self.curJob = Job.NETWORK_CAPTURE
        getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 1)
        getNetworkScanner().print_device_list(getDeviceLibrary().device_list)

        packets = getNetworkScanner().sniff_devices(getDatabase().get_basic_device_list(), timeout=int(getDatabase().get_config("scan_duration")))
        self.curJob = Job.NETWORK_ANALYZE
        getNetworkScanner().set_devices_scanning(getDeviceLibrary().device_list, 0)
        scan_result = getTrafficAnalyzer().analyze(packets, getDatabase().get_basic_device_list())

        getDeviceLibrary().save_scan_result(scan_result)
        getDatabase().save_scan_result(scan_result)

        # remove packets from memory (can be huge!)
        del packets


        self.lastScan = time.time()
        self.curJob = Job.IDLE
Ejemplo n.º 10
0
    def arp_scan_network_for_devices(self):
        """ Returns a list of devices which are currently active on the network """
        devices = []
        default_gateway = getNetworkScanner().get_default_gateway()
        ip = getDatabase().get_config("home_cidr")
        logging.info("[*] Scanning network (" + str(ip) +
                     ") for devices (this takes up to 30 seconds)")

        answered_list = srp(Ether(dst="ff:ff:ff:ff:ff:ff") / ARP(pdst=ip),
                            timeout=2,
                            retry=10)[0]

        for element in answered_list:
            if element[1].psrc != default_gateway:
                device = {}
                device["ip"] = element[1].psrc
                device["mac"] = element[1].hwsrc.upper()
                device["name"] = ""
                device["scanning"] = 0
                devices.append(device)

        #self.save_device_list()
        getDatabase().save_network_scan(devices)
        return devices
Ejemplo n.º 11
0
def get_network_usage():
    timespan = 10000000
    if ("timespan" in request.args):
        timespan = int(request.args["timespan"])
    conn, c = getDatabase().get_conn_cursor()

    query = None

    if ("device" in request.args):
        query = c.execute(
            "SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? AND Device.device_id = ? GROUP BY time, Communication.device_id  ORDER BY time DESC",
            (time.time() - timespan, int(request.args["device"])))
    else:
        query = c.execute(
            "SELECT Device.device_id, Device.device_name, Device.mac, time, SUM(brcv) as brcv, SUM(bsnd) as bsnd, SUM(prcv) as prcv, SUM(psnd) as psnd FROM Communication INNER JOIN Device ON Device.device_id=Communication.device_id WHERE time > ? GROUP BY time, Communication.device_id  ORDER BY time DESC",
            (time.time() - timespan, ))

    entries = query.fetchall()

    conn.close()

    return json.dumps([dict(ix) for ix in entries])
Ejemplo n.º 12
0
    async def scan(self, websocket):
        cnt = 0
        default_gateway = getNetworkScanner().get_default_gateway()
        cidr = getDatabase().get_config("home_cidr")
        last_update = 0
        while (True):

            if (last_update < getJobScheduler().getLastUpdate()):
                last_update = getJobScheduler().getLastUpdate()
                scan_result = {
                    "devices": getDeviceLibrary().device_list,
                    "cidr": cidr
                }

                await websocket.send(
                    "{\"type\":\"scan_result\", \"payload\":" +
                    json.dumps(scan_result) + "}")
                await websocket.send("{\"type\":\"history\", \"payload\":" +
                                     json.dumps(getDeviceLibrary().history) +
                                     "}")

            time.sleep(1)
Ejemplo n.º 13
0
 def __init__(self):
     self.thread = threading.Thread(target=self.work, name="JobScheduler")
     self.thread.start()
     getDatabase().initialize()
Ejemplo n.º 14
0
    def analyze(self, packets_test, device_list):
        #own_ip = get('https://api.ipify.org').text

        scan_duration = int(getDatabase().get_config("scan_duration"))

        self.ana_res = {}

        for device in device_list:
            if (device["ip"] not in self.ana_res):
                self.ana_res[device["ip"]] = {}

        default_gateway = getNetworkScanner().get_default_gateway()
        cidr = getDatabase().get_config("home_cidr")

        logging.info("[*] Start traffic analysis")
        logging.info(" |--> IP traffic analysis")
        logging.info("packets: " + str(len(packets_test)))

        bytestotal = 0

        # acclumerate IP packet information
        self.tot_p_count += len(packets_test)
        for pkt in packets_test:
            """if HTTP in pkt:
                http_layer = pkt.getlayer(HTTP)
                pkt.show()"""

            if IP in pkt:
                send = True
                ip_src = pkt[IP].src
                ip_dst = pkt[IP].dst
                foreign_ip = None
                local_ip = None
                hport = None
                pport = None

                bytestotal += len(pkt)

                if (ip_src in self.ana_res):
                    foreign_ip = ip_dst
                    local_ip = ip_src
                    if TCP in pkt:
                        hport = pkt[TCP].sport
                        pport = pkt[TCP].dport
                    send = True

                if (ip_dst in self.ana_res):
                    foreign_ip = ip_src
                    local_ip = ip_dst
                    if TCP in pkt:
                        hport = pkt[TCP].dport
                        pport = pkt[TCP].sport
                    send = False

                if (local_ip != None and foreign_ip != None):
                    if (foreign_ip not in self.ana_res[local_ip]):
                        if (send):
                            self.ana_res[local_ip][foreign_ip] = {
                                "snd": 1 / scan_duration,
                                "rcv": 0,
                                "bsnd": len(pkt),
                                "brcv": 0,
                                "domains": [],
                                "threats": [],
                                "hports": {hport},
                                "pports": {pport}
                            }
                        else:
                            self.ana_res[local_ip][foreign_ip] = {
                                "snd": 0,
                                "rcv": 1 / scan_duration,
                                "bsnd": len(pkt),
                                "brcv": 0,
                                "domains": [],
                                "threats": [],
                                "hports": {hport},
                                "pports": {pport}
                            }
                    else:
                        self.ana_res[local_ip][foreign_ip][
                            "hports"] = self.ana_res[local_ip][foreign_ip][
                                "hports"] | {hport}
                        self.ana_res[local_ip][foreign_ip][
                            "pports"] = self.ana_res[local_ip][foreign_ip][
                                "pports"] | {pport}
                        if (send):
                            self.ana_res[local_ip][foreign_ip][
                                "snd"] += 1 / scan_duration
                            self.ana_res[local_ip][foreign_ip]["bsnd"] += len(
                                pkt) / scan_duration
                        else:
                            self.ana_res[local_ip][foreign_ip][
                                "rcv"] += 1 / scan_duration
                            self.ana_res[local_ip][foreign_ip]["brcv"] += len(
                                pkt) / scan_duration

        # perform IP check
        #collect all ips
        all_ips = set()

        for key in self.ana_res.keys():
            for ip in self.ana_res[key]:
                all_ips.add(ip)

        ip_threats = getIPThreatDatabase().check_IPs(all_ips)

        for key in self.ana_res.keys():
            for ip in self.ana_res[key]:
                # Check for malicious ips
                if (ip in ip_threats):
                    self.ana_res[key][ip]["threats"].append(ip_threats[ip])

                # Perform reverse dns lookup
                if (len(self.ana_res[key][ip]["domains"]) == 0):
                    try:
                        reversed_dns = socket.gethostbyaddr(ip)
                        if (len(reversed_dns) > 0):
                            getDatabase().submit_ip_domain(ip, reversed_dns[0])
                            self.ana_res[key][ip]["domains"].append(
                                reversed_dns[0])
                    except Exception:
                        logging.exception("no reverse dns for " + ip)

        logging.info(" |--> DNS traffic analysis")
        if (int(getDatabase().get_config("DNS_inspect")) > 0):
            logging.info("      DNS traffic analysis enabled")
            try:
                for pkt in packets_test:
                    if pkt.haslayer(DNSRR):
                        # If the answer is a DNSRR, print the name it replied with.
                        if isinstance(pkt.an, DNSRR):
                            for key in self.ana_res.keys():
                                if (pkt.an.rdata in self.ana_res[key]):
                                    getDatabase().submit_ip_domain(
                                        pkt.an.rdata,
                                        pkt.an.rrname.decode("utf-8"))
                                    if (pkt.an.rrname.decode("utf-8")
                                            in ';;;'.join(self.ana_res[key][
                                                pkt.an.rdata]["domains"])):
                                        self.ana_res[key][
                                            pkt.an.rdata]["domains"].append(
                                                pkt.an.rrname.decode("utf-8"))
                                        # TODO: perform malicious dns lookup
            except Exception:
                logging.exception("Exception during DNS traffic analysis",
                                  Exception)
        else:
            logging.info("      DNS traffic analysis disabled")

        getDatabase().insert_or_replace_config(
            "num_pack_scan",
            int(getDatabase().get_config("num_pack_scan")) + len(packets_test))
        getDatabase().insert_or_replace_config(
            "num_bytes_scan",
            int(getDatabase().get_config("num_bytes_scan")) + bytestotal)

        logging.info("[*] Traffic analysis completed")

        for device in device_list:
            for rip in list(self.ana_res[device["ip"]].keys()):
                self.ana_res[device["ip"]][rip]["hports"] = list(
                    self.ana_res[device["ip"]][rip]["hports"])
                self.ana_res[device["ip"]][rip]["pports"] = list(
                    self.ana_res[device["ip"]][rip]["pports"])
            device["connections"] = self.ana_res[device["ip"]]
            getNetworkScanner().add_locations_to_device(device)
        return {
            "devices": copy.deepcopy(device_list),
            "cidr": cidr,
            "time": time.time()
        }


#if __name__ == "__main__":
#    test = getTrafficAnalyzer()
#    packets = rdpcap('full_capture.pcap')
#    print(test.analyze(packets, [{"ip":"192.168.8.113"},{"ip":"192.168.8.101"},{"ip":"192.168.8.1"}]))
Ejemplo n.º 15
0
 def isInScanningEnviornment(self):
     cur_def = getNetworkScanner().get_mac(getNetworkScanner().get_default_gateway())
     stored_def = getDatabase().get_config("home_mac")
     return cur_def == stored_def or cur_def == None
Ejemplo n.º 16
0
def get_devices():
    devices = getDatabase().get_basic_device_list()
    return json.dumps(devices)
Ejemplo n.º 17
0
    def nmap_scan_and_fingerprint(self):
        devices = []
        default_gateway = getNetworkScanner().get_default_gateway()
        cidr = getDatabase().get_config("home_cidr")
        print("[*] Scanning network (" + str(cidr) +
              ") for devices (this takes up to 60 seconds)")
        nm = nmap.PortScanner()
        # for name detection
        nm.scan(cidr, arguments='-sV -T4 -O -F --version-light')
        for ip in nm._scan_result['scan']:
            if ('mac' in nm[ip]['addresses']):
                client_dict = {
                    "ip": ip,
                    "mac": (nm[ip]['addresses']['mac']).upper(),
                    "scanning": 0
                }
                already_found = False
                for device in devices:
                    if (device["ip"] == ip):
                        client_dict = device
                        already_found = True
                if (len(nm[ip]['hostnames']) > 0):
                    client_dict["name"] = nm[ip]['hostnames'][0][
                        'name'].replace(".lan", "")
                if ('portsused' in nm[ip]):
                    client_dict["ports"] = nm[ip]['portsused']
                if ('tcp' in nm[ip]):
                    client_dict["tcp"] = nm[ip]['tcp']
                if ('osmatch' in nm[ip]):
                    client_dict["osmatch"] = nm[ip]['osmatch']

                devices.append(client_dict)
            else:
                own_ips = [
                    ip
                    for ip in socket.gethostbyname_ex(socket.gethostname())[2]
                    if not ip.startswith("127.")
                ][:1]
                #TODO make it work
                if (ip in own_ips or True):
                    client_dict = {
                        "ip": ip,
                        "mac": str(get_mac()),
                        "scanning": 0
                    }
                    if (len(nm[ip]['hostnames']) > 0):
                        client_dict["name"] = nm[ip]['hostnames'][0][
                            'name'].replace(".lan", "")
                    if ('portsused' in nm[ip]):
                        client_dict["ports"] = nm[ip]['portsused']
                    if ('tcp' in nm[ip]):
                        client_dict["tcp"] = nm[ip]['tcp']
                    if ('osmatch' in nm[ip]):
                        client_dict["osmatch"] = nm[ip]['osmatch']
                else:
                    print(
                        "WARNING: a device (" + ip +
                        ") has been ignored because no MAC address was found")

        del nm
        #self.save_device_list()
        getDatabase().save_network_scan(devices)
        return devices
Ejemplo n.º 18
0
def get_config():
    configs = getDatabase().get_all_configs()
    return json.dumps(configs)
Ejemplo n.º 19
0
def update_config():
    name = request.args.get('name')
    value = request.args.get('value')
    getDatabase().insert_or_replace_config(name, value)
    return ""