""" Anomaly detection with spectral clustering algorithm.
    First training set only, to see what would happen with only known classes
    Next with test set, to see what would happen with only unknown classes
    """
    import time
    start = time.time()

    headers, attacks = preprocessing.get_header_data()
    headers.remove('protocol_type')
    headers.remove('attack')
    headers.remove('difficulty')

    df_training_20, df_training_full, gmms_20, gmms_full = preprocessing.get_preprocessed_training_data()
    df_test_20, df_test_full, gmms_test_20, gmms_test_full = preprocessing.get_preprocessed_test_data()

    logger.set_file(today + "/log_main.txt")

    # with training-set
    gmms = gmms_20
    df1 = df_training_20[0:500]

    title = "training20_only"
    logger.debug("#################################################")
    logger.debug(title)
    test_clustering(df1, gmms, title=title, save_to_file=True)

    # with test-set
    dataset_description = "training20_test20"
    for attack_type_index, attack_type in enumerate(model.attack_types) :
        if attack_type_index == model.attack_normal : # why <= instead of ==
            continue
Exemple #2
0
    def __init__(self):
        '''
            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
            In general, external programs and plugins should not use this class.
        '''

        try:
            os.chdir(sys.path[0])
        except FileNotFoundError:
            pass

        # Load global configuration data

        data_exists = os.path.exists('data/')

        if not data_exists:
            os.mkdir('data/')

        if os.path.exists('static-data/default_config.json'):
            config.set_config(
                json.loads(open('static-data/default_config.json').read())
            )  # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
        else:
            # the default config file doesn't exist, try hardcoded config
            config.set_config({
                'devmode': True,
                'log': {
                    'file': {
                        'output': True,
                        'path': 'data/output.log'
                    },
                    'console': {
                        'output': True,
                        'color': True
                    }
                }
            })
        if not data_exists:
            config.save()
        config.reload()  # this will read the configuration file into memory

        settings = 0b000
        if config.get('log', {'console': {'color': True}})['console']['color']:
            settings = settings | logger.USE_ANSI
        if config.get('log', {'console': {
                'output': True
        }})['console']['output']:
            settings = settings | logger.OUTPUT_TO_CONSOLE
        if config.get('log', {'file': {'output': True}})['file']['output']:
            settings = settings | logger.OUTPUT_TO_FILE
            logger.set_file(
                config.get('log', {'file': {
                    'path': 'data/output.log'
                }})['file']['path'])
        logger.set_settings(settings)

        if str(config.get('devmode', True)).lower() == 'true':
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.onionrCore = core.Core()
        self.onionrUtils = OnionrUtils(self.onionrCore)

        # Handle commands

        self.debug = False  # Whole application debugging

        if os.path.exists('data-encrypted.dat'):
            while True:
                print('Enter password to decrypt:')
                password = getpass.getpass()
                result = self.onionrCore.dataDirDecrypt(password)
                if os.path.exists('data/'):
                    break
                else:
                    logger.error('Failed to decrypt: ' + result[1],
                                 timestamp=False)
        else:
            # If data folder does not exist
            if not data_exists:
                if not os.path.exists('data/blocks/'):
                    os.mkdir('data/blocks/')

            # Copy default plugins into plugins folder
            if not os.path.exists(plugins.get_plugins_folder()):
                if os.path.exists('static-data/default-plugins/'):
                    names = [
                        f for f in os.listdir("static-data/default-plugins/")
                        if not os.path.isfile(f)
                    ]
                    shutil.copytree('static-data/default-plugins/',
                                    plugins.get_plugins_folder())

                    # Enable plugins
                    for name in names:
                        if not name in plugins.get_enabled_plugins():
                            plugins.enable(name, self)

        for name in plugins.get_enabled_plugins():
            if not os.path.exists(plugins.get_plugin_data_folder(name)):
                try:
                    os.mkdir(plugins.get_plugin_data_folder(name))
                except:
                    plugins.disable(name, onionr=self, stop_event=False)

        if not os.path.exists(self.onionrCore.peerDB):
            self.onionrCore.createPeerDB()
            pass
        if not os.path.exists(self.onionrCore.addressDB):
            self.onionrCore.createAddressDB()

        # Get configuration

        if not data_exists:
            # Generate default config
            # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
            if self.debug:
                randomPort = 8080
            else:
                while True:
                    randomPort = random.randint(1024, 65535)
                    if self.onionrUtils.checkPort(randomPort):
                        break
            config.set(
                'client', {
                    'participate': 'true',
                    'client_hmac': base64.b16encode(
                        os.urandom(32)).decode('utf-8'),
                    'port': randomPort,
                    'api_version': API_VERSION
                }, True)

        self.cmds = {
            '': self.showHelpSuggestion,
            'help': self.showHelp,
            'version': self.version,
            'config': self.configure,
            'start': self.start,
            'stop': self.killDaemon,
            'status': self.showStats,
            'statistics': self.showStats,
            'stats': self.showStats,
            'enable-plugin': self.enablePlugin,
            'enplugin': self.enablePlugin,
            'enableplugin': self.enablePlugin,
            'enmod': self.enablePlugin,
            'disable-plugin': self.disablePlugin,
            'displugin': self.disablePlugin,
            'disableplugin': self.disablePlugin,
            'dismod': self.disablePlugin,
            'reload-plugin': self.reloadPlugin,
            'reloadplugin': self.reloadPlugin,
            'reload-plugins': self.reloadPlugin,
            'reloadplugins': self.reloadPlugin,
            'create-plugin': self.createPlugin,
            'createplugin': self.createPlugin,
            'plugin-create': self.createPlugin,
            'listkeys': self.listKeys,
            'list-keys': self.listKeys,
            'addmsg': self.addMessage,
            'addmessage': self.addMessage,
            'add-msg': self.addMessage,
            'add-message': self.addMessage,
            'pm': self.sendEncrypt,
            'getpms': self.getPMs,
            'get-pms': self.getPMs,
            'addpeer': self.addPeer,
            'add-peer': self.addPeer,
            'add-address': self.addAddress,
            'add-addr': self.addAddress,
            'addaddr': self.addAddress,
            'addaddress': self.addAddress,
            'addfile': self.addFile,
            'importblocks': self.onionrUtils.importNewBlocks,
            'introduce': self.onionrCore.introduceNode,
            'connect': self.addAddress
        }

        self.cmdhelp = {
            'help': 'Displays this Onionr help menu',
            'version': 'Displays the Onionr version',
            'config': 'Configures something and adds it to the file',
            'start': 'Starts the Onionr daemon',
            'stop': 'Stops the Onionr daemon',
            'stats': 'Displays node statistics',
            'enable-plugin': 'Enables and starts a plugin',
            'disable-plugin': 'Disables and stops a plugin',
            'reload-plugin': 'Reloads a plugin',
            'create-plugin': 'Creates directory structure for a plugin',
            'add-peer': 'Adds a peer to database',
            'list-peers': 'Displays a list of peers',
            'add-msg': 'Broadcasts a message to the Onionr network',
            'pm': 'Adds a private message to block',
            'get-pms': 'Shows private messages sent to you',
            'addfile': 'Create an Onionr block from a file',
            'importblocks':
            'import blocks from the disk (Onionr is transport-agnostic!)',
            'introduce': 'Introduce your node to the public Onionr network',
        }

        # initialize plugins
        events.event('init', onionr=self, threaded=False)

        command = ''
        try:
            command = sys.argv[1].lower()
        except IndexError:
            command = ''
        finally:
            self.execute(command)

        if not self._developmentMode:
            encryptionPassword = self.onionrUtils.getPassword(
                'Enter password to encrypt directory: ')
            self.onionrCore.dataDirEncrypt(encryptionPassword)
            shutil.rmtree('data/')

        return
            Z = mlab.bivariate_normal(X, Y, np.std(x), np.std(y), np.mean(x), np.mean(y))
            plt.contour(X,Y,Z)

#    for i, r in df.iterrows() :
#        if r['attack']
#    for i, p in enumerate(cproj):
#        if res[i] == 8 :
#            ax1.scatter(p[0], p[1], c='g')

#    plt.xticks(())
#    plt.yticks(())

    plt.show()
    plt.close()

if __name__ == '__main__':
    """ Anomaly detection with spectral clustering algorithm.
    First training set only, to see what would happen with only known classes
    Next with test set, to see what would happen with only unknown classes
    """
    import time
    start = time.time()

    logger.set_file(today + "/log_plots.txt")
    gen_plots()
    # gen_one_plot()
    # test()

    elapsed = (time.time() - start)
    print "done in %s seconds" % (elapsed)
    Next with test set, to see what would happen with only unknown classes
    """
    import time
    start = time.time()

    headers, attacks = preprocessing.get_header_data()
    headers.remove('protocol_type')
    headers.remove('attack')
    headers.remove('difficulty')

    df_training_20, df_training_full, gmms_20, gmms_full = preprocessing.get_preprocessed_training_data(
    )
    df_test_20, df_test_full, gmms_test_20, gmms_test_full = preprocessing.get_preprocessed_test_data(
    )

    logger.set_file(today + "/log_main.txt")

    # with training-set
    gmms = gmms_20
    df1 = df_training_20[0:500]

    title = "training20_only"
    logger.debug("#################################################")
    logger.debug(title)
    test_clustering(df1, gmms, title=title, save_to_file=True)

    # with test-set
    dataset_description = "training20_test20"
    for attack_type_index, attack_type in enumerate(model.attack_types):
        if attack_type_index == model.attack_normal:  # why <= instead of ==
            continue
Exemple #5
0
import logger

logger.info("Hello, welcome to Simon's logger!")
logger.debug("You can write to a default file, which is stamped with when the module was imported.")
logger.set_file("new_file.txt")
logger.debug("Or you can write to a file of your choosing!")
logger.config(user='******')
logger.info("You can set attributes using the .config method")
Exemple #6
0
    def __init__(self):
        '''
            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
            In general, external programs and plugins should not use this class.
        '''
        self.userRunDir = os.getcwd() # Directory user runs the program from
        self.killed = False

        if sys.argv[0] == os.path.basename(__file__):
            try:
                os.chdir(sys.path[0])
            except FileNotFoundError:
                pass

        # set data dir
        self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
        if not self.dataDir.endswith('/'):
            self.dataDir += '/'

        # set log file
        logger.set_file(os.environ.get('LOG_DIR', 'data') + '/onionr.log')

        # Load global configuration data
        data_exists = Onionr.setupConfig(self.dataDir, self)

        if netcontroller.torBinary() is None:
            logger.error('Tor is not installed')
            sys.exit(1)

        # If block data folder does not exist
        if not os.path.exists(self.dataDir + 'blocks/'):
            os.mkdir(self.dataDir + 'blocks/')

        # Copy default plugins into plugins folder
        if not os.path.exists(plugins.get_plugins_folder()):
            if os.path.exists('static-data/default-plugins/'):
                names = [f for f in os.listdir("static-data/default-plugins/")]
                shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())

                # Enable plugins
                for name in names:
                    if not name in plugins.get_enabled_plugins():
                        plugins.enable(name, self)

        for name in plugins.get_enabled_plugins():
            if not os.path.exists(plugins.get_plugin_data_folder(name)):
                try:
                    os.mkdir(plugins.get_plugin_data_folder(name))
                except:
                    plugins.disable(name, onionr = self, stop_event = False)

        self.communicatorInst = None
        self.onionrCore = core.Core()
        self.onionrCore.onionrInst = self
        #self.deleteRunFiles()
        self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)

        self.clientAPIInst = '' # Client http api instance
        self.publicAPIInst = '' # Public http api instance

        signal.signal(signal.SIGTERM, self.exitSigterm)

        # Handle commands

        self.debug = False # Whole application debugging

        # Get configuration
        if type(config.get('client.webpassword')) is type(None):
            config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
        if type(config.get('client.client.port')) is type(None):
            randomPort = netcontroller.getOpenPort()
            config.set('client.client.port', randomPort, savefile=True)
        if type(config.get('client.public.port')) is type(None):
            randomPort = netcontroller.getOpenPort()
            config.set('client.public.port', randomPort, savefile=True)
        if type(config.get('client.api_version')) is type(None):
            config.set('client.api_version', API_VERSION, savefile=True)

        self.cmds = commands.get_commands(self)
        self.cmdhelp = commands.cmd_help

        # initialize plugins
        events.event('init', onionr = self, threaded = False)

        command = ''
        try:
            command = sys.argv[1].lower()
        except IndexError:
            command = ''
        finally:
            self.execute(command)

        return
Exemple #7
0
    def __init__(self):
        '''
            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
            In general, external programs and plugins should not use this class.
        '''

        try:
            os.chdir(sys.path[0])
        except FileNotFoundError:
            pass

        # Load global configuration data

        exists = os.path.exists(config.get_config_file())
        config.set_config(
            {
                'devmode': True,
                'log': {
                    'file': {
                        'output': True,
                        'path': 'data/output.log'
                    },
                    'console': {
                        'output': True,
                        'color': True
                    }
                }
            }
        )  # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
        config.reload()  # this will read the configuration file into memory

        settings = 0b000
        if config.get('log', {'console': {'color': True}})['console']['color']:
            settings = settings | logger.USE_ANSI
        if config.get('log', {'console': {
                'output': True
        }})['console']['output']:
            settings = settings | logger.OUTPUT_TO_CONSOLE
        if config.get('log', {'file': {'output': True}})['file']['output']:
            settings = settings | logger.OUTPUT_TO_FILE
            logger.set_file(
                config.get('log', {'file': {
                    'path': 'data/output.log'
                }})['file']['path'])
        logger.set_settings(settings)

        if config.get('devmode', True):
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.onionrCore = core.Core()
        self.onionrUtils = OnionrUtils(self.onionrCore)

        # Handle commands

        self.debug = False  # Whole application debugging

        if os.path.exists('data-encrypted.dat'):
            while True:
                print('Enter password to decrypt:')
                password = getpass.getpass()
                result = self.onionrCore.dataDirDecrypt(password)
                if os.path.exists('data/'):
                    break
                else:
                    logger.error('Failed to decrypt: ' + result[1])
        else:
            if not os.path.exists('data/'):
                os.mkdir('data/')
                os.mkdir('data/blocks/')

        if not os.path.exists(self.onionrCore.peerDB):
            self.onionrCore.createPeerDB()
            pass
        if not os.path.exists(self.onionrCore.addressDB):
            self.onionrCore.createAddressDB()

        # Get configuration

        if not exists:
            # Generate default config
            # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
            if self.debug:
                randomPort = 8080
            else:
                while True:
                    randomPort = random.randint(1024, 65535)
                    if self.onionrUtils.checkPort(randomPort):
                        break
            config.set(
                'client', {
                    'participate': 'true',
                    'client_hmac': base64.b64encode(
                        os.urandom(32)).decode('utf-8'),
                    'port': randomPort,
                    'api_version': API_VERSION
                }, True)

        self.cmds = {
            '': self.showHelpSuggestion,
            'help': self.showHelp,
            'version': self.version,
            'config': self.configure,
            'start': self.start,
            'stop': self.killDaemon,
            'stats': self.showStats,
            'enable-plugin': self.enablePlugin,
            'enplugin': self.enablePlugin,
            'enableplugin': self.enablePlugin,
            'enmod': self.enablePlugin,
            'disable-plugin': self.disablePlugin,
            'displugin': self.disablePlugin,
            'disableplugin': self.disablePlugin,
            'dismod': self.disablePlugin,
            'reload-plugin': self.reloadPlugin,
            'reloadplugin': self.reloadPlugin,
            'reload-plugins': self.reloadPlugin,
            'reloadplugins': self.reloadPlugin,
            'listpeers': self.listPeers,
            'list-peers': self.listPeers,
            'addmsg': self.addMessage,
            'addmessage': self.addMessage,
            'add-msg': self.addMessage,
            'add-message': self.addMessage,
            'pm': self.sendEncrypt,
            'gui': self.openGUI,
            'addpeer': self.addPeer,
            'add-peer': self.addPeer,
            'add-address': self.addAddress,
            'addaddress': self.addAddress,
            'connect': self.addAddress
        }

        self.cmdhelp = {
            'help': 'Displays this Onionr help menu',
            'version': 'Displays the Onionr version',
            'config': 'Configures something and adds it to the file',
            'start': 'Starts the Onionr daemon',
            'stop': 'Stops the Onionr daemon',
            'stats': 'Displays node statistics',
            'enable-plugin': 'Enables and starts a plugin',
            'disable-plugin': 'Disables and stops a plugin',
            'reload-plugin': 'Reloads a plugin',
            'list-peers': 'Displays a list of peers',
            'add-peer': 'Adds a peer (?)',
            'add-msg': 'Broadcasts a message to the Onionr network',
            'pm': 'Adds a private message to block',
            'gui': 'Opens a graphical interface for Onionr'
        }

        command = ''
        try:
            command = sys.argv[1].lower()
        except IndexError:
            command = ''
        finally:
            self.execute(command)

        if not self._developmentMode:
            encryptionPassword = self.onionrUtils.getPassword(
                'Enter password to encrypt directory: ')
            self.onionrCore.dataDirEncrypt(encryptionPassword)
            shutil.rmtree('data/')

        return
            plt.contour(X, Y, Z)


#    for i, r in df.iterrows() :
#        if r['attack']
#    for i, p in enumerate(cproj):
#        if res[i] == 8 :
#            ax1.scatter(p[0], p[1], c='g')

#    plt.xticks(())
#    plt.yticks(())

    plt.show()
    plt.close()

if __name__ == '__main__':
    """ Anomaly detection with spectral clustering algorithm.
    First training set only, to see what would happen with only known classes
    Next with test set, to see what would happen with only unknown classes
    """
    import time
    start = time.time()

    logger.set_file(today + "/log_plots.txt")
    gen_plots()
    # gen_one_plot()
    # test()

    elapsed = (time.time() - start)
    print "done in %s seconds" % (elapsed)