""" Anomaly detection with spectral clustering algorithm.
First training set only, to see what would happen with only known classes
Next with test set, to see what would happen with only unknown classes
"""
import time
start = time.time()
headers, attacks = preprocessing.get_header_data()
headers.remove('protocol_type')
headers.remove('attack')
headers.remove('difficulty')
df_training_20, df_training_full, gmms_20, gmms_full = preprocessing.get_preprocessed_training_data()
df_test_20, df_test_full, gmms_test_20, gmms_test_full = preprocessing.get_preprocessed_test_data()
logger.set_file(today + "/log_main.txt")
# with training-set
gmms = gmms_20
df1 = df_training_20[0:500]
title = "training20_only"
logger.debug("#################################################")
logger.debug(title)
test_clustering(df1, gmms, title=title, save_to_file=True)
# with test-set
dataset_description = "training20_test20"
for attack_type_index, attack_type in enumerate(model.attack_types) :
if attack_type_index == model.attack_normal : # why <= instead of ==
continue
def __init__(self):
'''
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass
# Load global configuration data
data_exists = os.path.exists('data/')
if not data_exists:
os.mkdir('data/')
if os.path.exists('static-data/default_config.json'):
config.set_config(
json.loads(open('static-data/default_config.json').read())
) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else:
# the default config file doesn't exist, try hardcoded config
config.set_config({
'devmode': True,
'log': {
'file': {
'output': True,
'path': 'data/output.log'
},
'console': {
'output': True,
'color': True
}
}
})
if not data_exists:
config.save()
config.reload() # this will read the configuration file into memory
settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']:
settings = settings | logger.USE_ANSI
if config.get('log', {'console': {
'output': True
}})['console']['output']:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']:
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(
config.get('log', {'file': {
'path': 'data/output.log'
}})['file']['path'])
logger.set_settings(settings)
if str(config.get('devmode', True)).lower() == 'true':
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.onionrCore = core.Core()
self.onionrUtils = OnionrUtils(self.onionrCore)
# Handle commands
self.debug = False # Whole application debugging
if os.path.exists('data-encrypted.dat'):
while True:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
break
else:
logger.error('Failed to decrypt: ' + result[1],
timestamp=False)
else:
# If data folder does not exist
if not data_exists:
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'):
names = [
f for f in os.listdir("static-data/default-plugins/")
if not os.path.isfile(f)
]
shutil.copytree('static-data/default-plugins/',
plugins.get_plugins_folder())
# Enable plugins
for name in names:
if not name in plugins.get_enabled_plugins():
plugins.enable(name, self)
for name in plugins.get_enabled_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
try:
os.mkdir(plugins.get_plugin_data_folder(name))
except:
plugins.disable(name, onionr=self, stop_event=False)
if not os.path.exists(self.onionrCore.peerDB):
self.onionrCore.createPeerDB()
pass
if not os.path.exists(self.onionrCore.addressDB):
self.onionrCore.createAddressDB()
# Get configuration
if not data_exists:
# Generate default config
# Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
if self.debug:
randomPort = 8080
else:
while True:
randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort):
break
config.set(
'client', {
'participate': 'true',
'client_hmac': base64.b16encode(
os.urandom(32)).decode('utf-8'),
'port': randomPort,
'api_version': API_VERSION
}, True)
self.cmds = {
'': self.showHelpSuggestion,
'help': self.showHelp,
'version': self.version,
'config': self.configure,
'start': self.start,
'stop': self.killDaemon,
'status': self.showStats,
'statistics': self.showStats,
'stats': self.showStats,
'enable-plugin': self.enablePlugin,
'enplugin': self.enablePlugin,
'enableplugin': self.enablePlugin,
'enmod': self.enablePlugin,
'disable-plugin': self.disablePlugin,
'displugin': self.disablePlugin,
'disableplugin': self.disablePlugin,
'dismod': self.disablePlugin,
'reload-plugin': self.reloadPlugin,
'reloadplugin': self.reloadPlugin,
'reload-plugins': self.reloadPlugin,
'reloadplugins': self.reloadPlugin,
'create-plugin': self.createPlugin,
'createplugin': self.createPlugin,
'plugin-create': self.createPlugin,
'listkeys': self.listKeys,
'list-keys': self.listKeys,
'addmsg': self.addMessage,
'addmessage': self.addMessage,
'add-msg': self.addMessage,
'add-message': self.addMessage,
'pm': self.sendEncrypt,
'getpms': self.getPMs,
'get-pms': self.getPMs,
'addpeer': self.addPeer,
'add-peer': self.addPeer,
'add-address': self.addAddress,
'add-addr': self.addAddress,
'addaddr': self.addAddress,
'addaddress': self.addAddress,
'addfile': self.addFile,
'importblocks': self.onionrUtils.importNewBlocks,
'introduce': self.onionrCore.introduceNode,
'connect': self.addAddress
}
self.cmdhelp = {
'help': 'Displays this Onionr help menu',
'version': 'Displays the Onionr version',
'config': 'Configures something and adds it to the file',
'start': 'Starts the Onionr daemon',
'stop': 'Stops the Onionr daemon',
'stats': 'Displays node statistics',
'enable-plugin': 'Enables and starts a plugin',
'disable-plugin': 'Disables and stops a plugin',
'reload-plugin': 'Reloads a plugin',
'create-plugin': 'Creates directory structure for a plugin',
'add-peer': 'Adds a peer to database',
'list-peers': 'Displays a list of peers',
'add-msg': 'Broadcasts a message to the Onionr network',
'pm': 'Adds a private message to block',
'get-pms': 'Shows private messages sent to you',
'addfile': 'Create an Onionr block from a file',
'importblocks':
'import blocks from the disk (Onionr is transport-agnostic!)',
'introduce': 'Introduce your node to the public Onionr network',
}
# initialize plugins
events.event('init', onionr=self, threaded=False)
command = ''
try:
command = sys.argv[1].lower()
except IndexError:
command = ''
finally:
self.execute(command)
if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword(
'Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')
return
Z = mlab.bivariate_normal(X, Y, np.std(x), np.std(y), np.mean(x), np.mean(y))
plt.contour(X,Y,Z)
# for i, r in df.iterrows() :
# if r['attack']
# for i, p in enumerate(cproj):
# if res[i] == 8 :
# ax1.scatter(p[0], p[1], c='g')
# plt.xticks(())
# plt.yticks(())
plt.show()
plt.close()
if __name__ == '__main__':
""" Anomaly detection with spectral clustering algorithm.
First training set only, to see what would happen with only known classes
Next with test set, to see what would happen with only unknown classes
"""
import time
start = time.time()
logger.set_file(today + "/log_plots.txt")
gen_plots()
# gen_one_plot()
# test()
elapsed = (time.time() - start)
print "done in %s seconds" % (elapsed)
Next with test set, to see what would happen with only unknown classes
"""
import time
start = time.time()
headers, attacks = preprocessing.get_header_data()
headers.remove('protocol_type')
headers.remove('attack')
headers.remove('difficulty')
df_training_20, df_training_full, gmms_20, gmms_full = preprocessing.get_preprocessed_training_data(
)
df_test_20, df_test_full, gmms_test_20, gmms_test_full = preprocessing.get_preprocessed_test_data(
)
logger.set_file(today + "/log_main.txt")
# with training-set
gmms = gmms_20
df1 = df_training_20[0:500]
title = "training20_only"
logger.debug("#################################################")
logger.debug(title)
test_clustering(df1, gmms, title=title, save_to_file=True)
# with test-set
dataset_description = "training20_test20"
for attack_type_index, attack_type in enumerate(model.attack_types):
if attack_type_index == model.attack_normal: # why <= instead of ==
continue
import logger
logger.info("Hello, welcome to Simon's logger!")
logger.debug("You can write to a default file, which is stamped with when the module was imported.")
logger.set_file("new_file.txt")
logger.debug("Or you can write to a file of your choosing!")
logger.config(user='******')
logger.info("You can set attributes using the .config method")
def __init__(self):
'''
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
self.userRunDir = os.getcwd() # Directory user runs the program from
self.killed = False
if sys.argv[0] == os.path.basename(__file__):
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass
# set data dir
self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
if not self.dataDir.endswith('/'):
self.dataDir += '/'
# set log file
logger.set_file(os.environ.get('LOG_DIR', 'data') + '/onionr.log')
# Load global configuration data
data_exists = Onionr.setupConfig(self.dataDir, self)
if netcontroller.torBinary() is None:
logger.error('Tor is not installed')
sys.exit(1)
# If block data folder does not exist
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'):
names = [f for f in os.listdir("static-data/default-plugins/")]
shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
# Enable plugins
for name in names:
if not name in plugins.get_enabled_plugins():
plugins.enable(name, self)
for name in plugins.get_enabled_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
try:
os.mkdir(plugins.get_plugin_data_folder(name))
except:
plugins.disable(name, onionr = self, stop_event = False)
self.communicatorInst = None
self.onionrCore = core.Core()
self.onionrCore.onionrInst = self
#self.deleteRunFiles()
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
self.clientAPIInst = '' # Client http api instance
self.publicAPIInst = '' # Public http api instance
signal.signal(signal.SIGTERM, self.exitSigterm)
# Handle commands
self.debug = False # Whole application debugging
# Get configuration
if type(config.get('client.webpassword')) is type(None):
config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
if type(config.get('client.client.port')) is type(None):
randomPort = netcontroller.getOpenPort()
config.set('client.client.port', randomPort, savefile=True)
if type(config.get('client.public.port')) is type(None):
randomPort = netcontroller.getOpenPort()
config.set('client.public.port', randomPort, savefile=True)
if type(config.get('client.api_version')) is type(None):
config.set('client.api_version', API_VERSION, savefile=True)
self.cmds = commands.get_commands(self)
self.cmdhelp = commands.cmd_help
# initialize plugins
events.event('init', onionr = self, threaded = False)
command = ''
try:
command = sys.argv[1].lower()
except IndexError:
command = ''
finally:
self.execute(command)
return
def __init__(self):
'''
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass
# Load global configuration data
exists = os.path.exists(config.get_config_file())
config.set_config(
{
'devmode': True,
'log': {
'file': {
'output': True,
'path': 'data/output.log'
},
'console': {
'output': True,
'color': True
}
}
}
) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
config.reload() # this will read the configuration file into memory
settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']:
settings = settings | logger.USE_ANSI
if config.get('log', {'console': {
'output': True
}})['console']['output']:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']:
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(
config.get('log', {'file': {
'path': 'data/output.log'
}})['file']['path'])
logger.set_settings(settings)
if config.get('devmode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.onionrCore = core.Core()
self.onionrUtils = OnionrUtils(self.onionrCore)
# Handle commands
self.debug = False # Whole application debugging
if os.path.exists('data-encrypted.dat'):
while True:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
break
else:
logger.error('Failed to decrypt: ' + result[1])
else:
if not os.path.exists('data/'):
os.mkdir('data/')
os.mkdir('data/blocks/')
if not os.path.exists(self.onionrCore.peerDB):
self.onionrCore.createPeerDB()
pass
if not os.path.exists(self.onionrCore.addressDB):
self.onionrCore.createAddressDB()
# Get configuration
if not exists:
# Generate default config
# Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
if self.debug:
randomPort = 8080
else:
while True:
randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort):
break
config.set(
'client', {
'participate': 'true',
'client_hmac': base64.b64encode(
os.urandom(32)).decode('utf-8'),
'port': randomPort,
'api_version': API_VERSION
}, True)
self.cmds = {
'': self.showHelpSuggestion,
'help': self.showHelp,
'version': self.version,
'config': self.configure,
'start': self.start,
'stop': self.killDaemon,
'stats': self.showStats,
'enable-plugin': self.enablePlugin,
'enplugin': self.enablePlugin,
'enableplugin': self.enablePlugin,
'enmod': self.enablePlugin,
'disable-plugin': self.disablePlugin,
'displugin': self.disablePlugin,
'disableplugin': self.disablePlugin,
'dismod': self.disablePlugin,
'reload-plugin': self.reloadPlugin,
'reloadplugin': self.reloadPlugin,
'reload-plugins': self.reloadPlugin,
'reloadplugins': self.reloadPlugin,
'listpeers': self.listPeers,
'list-peers': self.listPeers,
'addmsg': self.addMessage,
'addmessage': self.addMessage,
'add-msg': self.addMessage,
'add-message': self.addMessage,
'pm': self.sendEncrypt,
'gui': self.openGUI,
'addpeer': self.addPeer,
'add-peer': self.addPeer,
'add-address': self.addAddress,
'addaddress': self.addAddress,
'connect': self.addAddress
}
self.cmdhelp = {
'help': 'Displays this Onionr help menu',
'version': 'Displays the Onionr version',
'config': 'Configures something and adds it to the file',
'start': 'Starts the Onionr daemon',
'stop': 'Stops the Onionr daemon',
'stats': 'Displays node statistics',
'enable-plugin': 'Enables and starts a plugin',
'disable-plugin': 'Disables and stops a plugin',
'reload-plugin': 'Reloads a plugin',
'list-peers': 'Displays a list of peers',
'add-peer': 'Adds a peer (?)',
'add-msg': 'Broadcasts a message to the Onionr network',
'pm': 'Adds a private message to block',
'gui': 'Opens a graphical interface for Onionr'
}
command = ''
try:
command = sys.argv[1].lower()
except IndexError:
command = ''
finally:
self.execute(command)
if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword(
'Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')
return
plt.contour(X, Y, Z)
# for i, r in df.iterrows() :
# if r['attack']
# for i, p in enumerate(cproj):
# if res[i] == 8 :
# ax1.scatter(p[0], p[1], c='g')
# plt.xticks(())
# plt.yticks(())
plt.show()
plt.close()
if __name__ == '__main__':
""" Anomaly detection with spectral clustering algorithm.
First training set only, to see what would happen with only known classes
Next with test set, to see what would happen with only unknown classes
"""
import time
start = time.time()
logger.set_file(today + "/log_plots.txt")
gen_plots()
# gen_one_plot()
# test()
elapsed = (time.time() - start)
print "done in %s seconds" % (elapsed)