def _setup_db(self):
        """
        Setup the probe specific database.
        :return:
        """
        # Create a database
        self._database = ProbeDb(
            self._configs['name']
        )

        # Setup the known database
        if self.behaviour == PROBE_MONITORING:

            # We need to check the already registered ips
            self._database.setup_db(
                self._tables
            )

            # Get the table
            table = self._database.get_table('KNOWN_IP')

            # Add the entries to the internal db
            for ip in self._configs['known']:

                # We need to convert it into a supported IP
                # This IP can be a subnet IP that needs to be put into a list
                address = self.__get_address(
                    ip
                )

                if type(address) is list():
                    for ip in address:

                        # Add the Ips in the known table
                        table.insert(
                            {
                                'type'          : 'IP',
                                'address'       : ip,
                            }
                        )
                else:
                    # Add the Ips in the known table
                    table.insert(
                        {
                            'type'          : 'IP',
                            'address'       : ip,
                        }
                    )
        elif self.behaviour == PROBE_OBSERVING:

            # Create a new table
            tables = [
                'IP'
            ]

            # We need to check the already registered ips
            self._database.setup_db(
                tables
            )
        return
    def _setup_db(self):
        """
        Setup the probe specific database.
        :return:
        """
        # Create a database
        self._database = ProbeDb(
            self._configs['name']
        )

        # Setup the known database
        if self.behaviour == PROBE_MONITORING:

            # We need to check the already registered ips
            self._database.setup_db(
                self._tables
            )

            # Get the table
            table = self._database.get_table('KNOWN_MAC')

            # Add the entries to the internal db
            for mac in self._configs['known']:

                # Add the Ips in the known table
                table.insert(
                    {
                        'type'          : 'MAC',
                        'address'       : mac,
                    }
                )
        elif self.behaviour == PROBE_OBSERVING:

            # Create a new table
            tables = [
                'MAC'
            ]

            # We need to check the already registered ips
            self._database.setup_db(
                tables
            )
        return
class MacProbe(PassiveNetworkProbe):
    """
    This is the MAC probe that will be used to poll
    the network interface and sniff out the MAC addresse.
    Then is reports them in the data record sent to the
    message engine.

    extends: NetworkProbe
    """

    # The class name
    name            = "MacProbe"

    # The probe type
    type            = "MAC"

    # Description
    description     = \
    "This is the probe that will monitor the mac addresses " \
    "on the network and will correlate them to a db."

    # Groups
    groups          = [
        "mac",
        "network",
        "reconnaissance"
    ]

    # Layer filter
    layers          = [Ether, IP]

    # ====================
    # Protected
    # ====================

    # Local database tables
    _tables         = [
        'KNOWN_MAC',
        'UNKNOWN_MAC'
    ]

    # ====================
    # Private
    # ====================

    # App configs
    _configs        = None

    # Time
    _date           = None

    def __init__(self, queue, configs):
        """
        This is the default constructor for the class.
        We supply the iface and the queue.

        :param type:        The type of probe
        :param queue:       The application queue
        :param configs:     The app configs (i.e. known, iface, save)
        :return:
        """

        # Get the configs
        self._configs = configs
        self._configs['name'] = self.name

        # Register the probe type as a passive probe
        PassiveNetworkProbe.__init__(
            self,
            self.type,
            queue,
            **{
                'iface' : configs['iface']
            }
        )

        # Set the behaviour
        self.logger.info(
            "Created a new Probe of type: %s" %self.type
        )
        return

    def _setup_db(self):
        """
        Setup the probe specific database.
        :return:
        """
        # Create a database
        self._database = ProbeDb(
            self._configs['name']
        )

        # Setup the known database
        if self.behaviour == PROBE_MONITORING:

            # We need to check the already registered ips
            self._database.setup_db(
                self._tables
            )

            # Get the table
            table = self._database.get_table('KNOWN_MAC')

            # Add the entries to the internal db
            for mac in self._configs['known']:

                # Add the Ips in the known table
                table.insert(
                    {
                        'type'          : 'MAC',
                        'address'       : mac,
                    }
                )
        elif self.behaviour == PROBE_OBSERVING:

            # Create a new table
            tables = [
                'MAC'
            ]

            # We need to check the already registered ips
            self._database.setup_db(
                tables
            )
        return

    def _correlate(self, pkt):
        """
        This is the correlation algorithm that will look at the databases and
        check either the registry or the black list.

        :param pkt:             The read packet
        :return:
        """

        # Get the ip layer
        dest_mac        = pkt[Ether].dst
        src_mac         = pkt[Ether].src
        type_mac        = pkt[Ether].type
        dest_ip         = pkt[IP].dst
        src_ip          = pkt[IP].src
        ip_len          = pkt[IP].len
        ip_chksum       = pkt[IP].chksum
        ip_version      = pkt[IP].version
        ip_id           = pkt[IP].id
        ip_ttl          = pkt[IP].ttl

        dest_data = {
            'type'          : 'IP|MAC',
            'seq'           : self._packet_count,
            'time'          : time.asctime(
                time.localtime(
                    time.time()
                )
            ),
            'mac'           : dest_mac,
            'mactype'       : type_mac,
            'length'        : ip_len,
            'checksum'      : ip_chksum,
            'ttl'           : ip_ttl,
            'id'            : ip_id,
            'ip'            : dest_ip,
            'ver'           : ip_version
        }

        src_data = {
            'type'          : 'IP|MAC',
            'seq'           : self._packet_count,
            'time'          : time.asctime(
                time.localtime(
                    time.time()
                )
            ),
            'mac'           : src_mac,
            'mactype'       : type_mac,
            'length'        : ip_len,
            'checksum'      : ip_chksum,
            'ttl'           : ip_ttl,
            'id'            : ip_id,
            'ip'            : src_ip,
            'ver'           : ip_version

        }

        # We check the dehaviour
        if self.behaviour == PROBE_MONITORING:

            # We need to check the validity of the ip
            # Get the table
            unknown_table = self._database.get_table(
                'UNKNOWN_MAC'
            )
            known_table = self._database.get_table(
                'KNOWN_MAC'
            )
            src_pkt = known_table.search(
                Query().address == src_mac
            )
            dest_pkt = known_table.search(
                Query().address == dest_mac
            )
            if dest_pkt is None:

                # We have an unknown destination ip
                unknown_table.insert(
                    dest_data
                )
            if src_pkt is None:

                # We have an unknown destination ip
                unknown_table.insert(
                    src_data
                )

        # Just register the IP for logging
        elif self.behaviour == PROBE_OBSERVING:
            table = self._database.get_table(
                'MAC'
            )
            table.insert(
                src_data
            )
            table.insert(
                dest_data
            )
        return
class DnsProbe(IpProbe):
    """
    This is the class object that defines the probing agent.
    In this class we define the running method and what the
    local behaviour of the probe is.

    extends: IpProbe
    """

    # The class name
    name            = "DnsProbe"

    # The probe type
    type            = "DNS"

    # Description
    description     = \
    "This is the probe that will monitor the dns addresses " \
    "on the network and will correlate them to a db."

    # Groups
    groups          = [
        "dns",
        "network",
        "reconnaissance"
    ]

    # Layer filter
    layers          = [DNS, IP]

    # ====================
    # Protected
    # ====================

    # Local database tables
    _tables         = [
        'KNOWN_DNS',
        'UNKNOWN_DNS',
    ]

    # ====================
    # Private
    # ====================

    # App configs
    _configs        = None

    # Time
    _date           = None

    def _setup_db(self):
        """
        Setup the probe specific database.
        :return:
        """
        # Create a database
        self._database = ProbeDb(
            self._configs['name']
        )

        # Setup the known database
        if self.behaviour == PROBE_MONITORING:

            # We need to check the already registered ips
            self._database.setup_db(
                self._tables
            )

            # Get the table
            table = self._database.get_table('KNOWN_DNS')

            # Add the entries to the internal db
            for dns in self._configs['known']:

                # Add the Ips in the known table
                table.insert(
                    {
                        'type'          : 'DNS',
                        'address'       : dns,
                    }
                )
        elif self.behaviour == PROBE_OBSERVING:

            # Create a new table
            tables = [
                'DNS'
            ]

            # We need to check the already registered ips
            self._database.setup_db(
                tables
            )
        return

    def _correlate(self, pkt):
        """
        This is the correlation algorithm that will look at the databases and
        check either the registry or the black list.

        :param pkt:             The read packet
        :return:
        """

        # Get the ip layer
        dst             = pkt[IP].dst
        src             = pkt[IP].src
        ip_len          = pkt[IP].len
        ip_chksum       = pkt[IP].chksum
        ip_version      = pkt[IP].version
        ip_id           = pkt[IP].id
        ip_ttl          = pkt[IP].ttl

        # Get the DNS name
        dns_name        = pkt[DNS]

        dns_data = {
            'type'          : 'DNS|IP',
            'seq'           : self._packet_count,
            'time'          : time.asctime(
                time.localtime(
                    time.time()
                )
            ),
            'dns'           : dns_name.__dict__,
            'src'           : src,
            'dst'           : dst,
            'length'        : ip_len,
            'checksum'      : ip_chksum,
            'ttl'           : ip_ttl,
            'id'            : ip_id,
            'version'       : ip_version
        }

        # We check the dehaviour
        if self.behaviour == PROBE_MONITORING:

            # We need to check the validity of the ip
            # Get the table
            unknown_table = self._database.get_table(
                'UNKNOWN_DNS'
            )
            known_table = self._database.get_table(
                'KNOWN_DNS'
            )
            pkt = known_table.search(
                Query().address == dns_name
            )
            if pkt is None:

                # We have an unknown destination ip
                unknown_table.insert(
                    dns_data
                )
        # Just register the IP for logging
        elif self.behaviour == PROBE_OBSERVING:
            table = self._database.get_table(
                'DNS'
            )
            table.insert(
                dns_data
            )
        return
class IpProbe(PassiveNetworkProbe):
    """
    This is the ip probe that is a mutable probe. It sniffs the
    ip addresses of all IP packets and correlates the IPs to an
    internal JSON database containing the previous registered IPs.
    The use can provide a previously defined database, that will then
    serve as a filter. In which case the added IPs are reported as foreign.
    If the database is not provided, the IPs are registered and reported
    as alive / dead.

    extends: PassiveNetworkProbe
    """

    # The class name
    name            = "IpProbe"

    # The probe type
    type            = "IP"

    # Description
    description     = \
    "This is the probe that will monitor the ip addresses " \
    "on the network and will correlate them to a db."

    # Groups
    groups          = [
        "ip",
        "network",
        "reconnaissance"
    ]

    # Layer filter
    layers          = [Ether, IP]

    # ====================
    # Protected
    # ====================

    # Local database tables
    _tables         = [
        'KNOWN_IP',
        'UNKNOWN_IP'
    ]

    # ====================
    # Private
    # ====================

    # App configs
    _configs       = None

    # Time
    _date          = None

    def __init__(self, queue, configs):
        """
        This is the default constructor for the class.
        We supply the iface and the queue.

        :param type:        The type of probe
        :param queue:       The application queue
        :param configs:     The app configs (i.e. known, iface, save)
        :return:
        """

        # Get the configs
        self._configs = configs
        self._configs['name'] = self.name

        # Register the probe type as a passive probe
        PassiveNetworkProbe.__init__(
            self,
            self.type,
            queue,
            **{
                'iface' : configs['iface']
            }
        )

        self.logger.info(
            "Created a new Probe of type: %s" %self.type
        )
        return

    def _setup_db(self):
        """
        Setup the probe specific database.
        :return:
        """
        # Create a database
        self._database = ProbeDb(
            self._configs['name']
        )

        # Setup the known database
        if self.behaviour == PROBE_MONITORING:

            # We need to check the already registered ips
            self._database.setup_db(
                self._tables
            )

            # Get the table
            table = self._database.get_table('KNOWN_IP')

            # Add the entries to the internal db
            for ip in self._configs['known']:

                # We need to convert it into a supported IP
                # This IP can be a subnet IP that needs to be put into a list
                address = self.__get_address(
                    ip
                )

                if type(address) is list():
                    for ip in address:

                        # Add the Ips in the known table
                        table.insert(
                            {
                                'type'          : 'IP',
                                'address'       : ip,
                            }
                        )
                else:
                    # Add the Ips in the known table
                    table.insert(
                        {
                            'type'          : 'IP',
                            'address'       : ip,
                        }
                    )
        elif self.behaviour == PROBE_OBSERVING:

            # Create a new table
            tables = [
                'IP'
            ]

            # We need to check the already registered ips
            self._database.setup_db(
                tables
            )
        return

    def _correlate(self, pkt):
        """
        This is the correlation algorithm that will look at the databases and
        check either the registry or the black list.

        :param pkt:             The read packet
        :return:
        """

        # Get the ip layer
        dest_mac        = pkt[Ether].dst
        src_mac         = pkt[Ether].src
        type_mac        = pkt[Ether].type
        dest_ip         = pkt[IP].dst
        src_ip          = pkt[IP].src
        ip_len          = pkt[IP].len
        ip_chksum       = pkt[IP].chksum
        ip_version      = pkt[IP].version
        ip_id           = pkt[IP].id
        ip_ttl          = pkt[IP].ttl

        dest_data = {
            'type'          : 'IP|MAC',
            'seq'           : self._packet_count,
            'time'          : time.asctime(
                time.localtime(
                    time.time()
                )
            ),
            'mac'           : dest_mac,
            'iptype'        : type_mac,
            'length'        : ip_len,
            'checksum'      : ip_chksum,
            'ttl'           : ip_ttl,
            'id'            : ip_id,
            'ip'            : dest_ip,
            'version'       : ip_version
        }

        src_data = {
            'type'          : 'IP|MAC',
            'seq'           : self._packet_count,
            'time'          : time.asctime(
                time.localtime(
                    time.time()
                )
            ),
            'mac'           : src_mac,
            'iptype'        : type_mac,
            'length'        : ip_len,
            'checksum'      : ip_chksum,
            'ttl'           : ip_ttl,
            'id'            : ip_id,
            'ip'            : src_ip,
            'version'       : ip_version
        }

        # We check the dehaviour
        if self.behaviour == PROBE_MONITORING:

            # We need to check the validity of the ip
            # Get the table
            unknown_table = self._database.get_table(
                'UNKNOWN_IP'
            )
            known_table = self._database.get_table(
                'KNOWN_IP'
            )
            src_pkt = known_table.search(
                Query().address == src_ip
            )
            dest_pkt = known_table.search(
                Query().address == dest_ip
            )
            if dest_pkt is None:

                # We have an unknown destination ip
                unknown_table.insert(
                    dest_data
                )
            if src_pkt is None:

                # We have an unknown destination ip
                unknown_table.insert(
                    src_data
                )

        # Just register the IP for logging
        elif self.behaviour == PROBE_OBSERVING:
            table = self._database.get_table(
                'IP'
            )
            table.insert(
                src_data
            )
            table.insert(
                dest_data
            )
        return

    def __get_address(self, ip):
        """
        This is the method that is used to get the ip address
        or network that is needed in the probing.

        :param ip:              The ip address or the ip network
        :return:
        """

        # Container
        addresses = []

        # Generator
        generate_ip = lambda address : addresses.append(
                {
                    'v4'    :   str(
                        address.ipv4()
                    ),
                    'v6'    :   str(
                        address.ipv6()
                    )
                }
        )

        # We have a network to generate
        if '/' in ip:
            network = list(IPNetwork(ip))
            for address in network:
                generate_ip(address)
            return addresses
        else:
            return ip