def __init__(self):
# Required line for anything that inherits from QObject.
QObject.__init__(self)
self._ids = TwoStageIDS()
# Enumerate all available Two Stage IDS models.
self._available_models = []
for file_or_dirname in os.listdir(dnnmodels_dir):
if file_or_dirname.endswith('.params'):
self._available_models.append(file_or_dirname.replace('.params', ''))
self._model_name = ''
def new_model(self, model_name, parameters):
self._ids = TwoStageIDS()
parameters = parameters.toVariant()
if model_name in self._available_models:
raise ValueError()
self._ids.change_ids_parameters('dnn_dir_path', dnnmodels_dir + '/' + model_name)
self._ids.change_ids_parameters('rules_profile', model_name)
self._ids.change_dnn_parameters('hidden_units', literal_eval(parameters['Hidden Units']))
print(parameters['Optimizer Properties'])
self._ids.change_dnn_parameters('optimizer', construct_optimizer(
parameters['Optimizer'],
parameters['Optimizer Properties']
))
self._ids.change_dnn_parameters('activation_fn', activation_fn_convert_reverse[parameters['Activation Function']])
self._ids.change_dnn_parameters('loss_reduction', loss_reduction_convert_reverse[parameters['Loss Reduction']])
self._ids.init_ids()
self._model_name = model_name
self.get_parameters.emit()
self._available_models.append(model_name)
self.get_availableModels.emit()
def test_training(prepared_ids: TwoStageIDS, feature_lists_labels,
canlist_good):
# Check that retraining rules works properly.
prepared_ids.retrain_rules(canlist_good)
assert prepared_ids.rules_trained
# Check that training the DNN works properly.
features, labels = feature_lists_labels
prepared_ids.train_dnn(dnn_input_function(features, labels, shuffle=True),
2000)
assert prepared_ids.dnn_trained
# Check that trying to train the DNN without an initialized DNN Based IDS
# causes an error.
not_initialized_ids = TwoStageIDS()
with pytest.raises(RuntimeError):
not_initialized_ids.train_dnn(
dnn_input_function(features, labels, shuffle=True), 2000)
class TwoStageIDSManager(QObject):
def __init__(self):
# Required line for anything that inherits from QObject.
QObject.__init__(self)
self._ids = TwoStageIDS()
# Enumerate all available Two Stage IDS models.
self._available_models = []
for file_or_dirname in os.listdir(dnnmodels_dir):
if file_or_dirname.endswith('.params'):
self._available_models.append(file_or_dirname.replace('.params', ''))
self._model_name = ''
# List of all Two Stage IDS models.
get_availableModels = pyqtSignal()
@pyqtProperty(QVariant, notify=get_availableModels)
def availableModels(self):
return self._available_models
# Transform internal properties of the IDS into a form the GUI can
# understand.
get_parameters = pyqtSignal()
@pyqtProperty(QVariant, notify=get_parameters)
def parameters(self):
if self._model_name:
return {
'Model Name': self._model_name,
'Rules Trained': self._ids.rules_trained,
'DNN Trained': self._ids.dnn_trained,
'Hidden Units': self._ids.dnn._params['hidden_units'],
'Activation Function': activation_fn_convert[self._ids.dnn._params['activation_fn']],
'Optimizer': optimizer_convert[type(self._ids.dnn._params['optimizer'])],
'Loss Reduction': loss_reduction_convert[self._ids.dnn._params['loss_reduction']]
}
else:
return {
'Model Name': 'No Model',
'Rules Trained': False,
'DNN Trained': False,
'Hidden Units': '',
'Activation Function': '',
'Optimizer': '',
'Loss Reduction': ''
}
@pyqtSlot(str, QJSValue)
def new_model(self, model_name, parameters):
self._ids = TwoStageIDS()
parameters = parameters.toVariant()
if model_name in self._available_models:
raise ValueError()
self._ids.change_ids_parameters('dnn_dir_path', dnnmodels_dir + '/' + model_name)
self._ids.change_ids_parameters('rules_profile', model_name)
self._ids.change_dnn_parameters('hidden_units', literal_eval(parameters['Hidden Units']))
print(parameters['Optimizer Properties'])
self._ids.change_dnn_parameters('optimizer', construct_optimizer(
parameters['Optimizer'],
parameters['Optimizer Properties']
))
self._ids.change_dnn_parameters('activation_fn', activation_fn_convert_reverse[parameters['Activation Function']])
self._ids.change_dnn_parameters('loss_reduction', loss_reduction_convert_reverse[parameters['Loss Reduction']])
self._ids.init_ids()
self._model_name = model_name
self.get_parameters.emit()
self._available_models.append(model_name)
self.get_availableModels.emit()
@pyqtSlot(str)
def load_model(self, model_name):
if model_name not in self._available_models:
raise ValueError()
self._ids.change_ids_parameters('dnn_dir_path', dnnmodels_dir + '/' + model_name)
self._ids.change_ids_parameters('rules_profile', model_name)
self._ids.init_ids()
self._model_name = model_name
self.get_parameters.emit()
@pyqtSlot(str)
def delete_model(self, model_name):
if model_name not in self._available_models:
raise ValueError()
if model_name == self._model_name:
raise RuntimeError()
params_file = dnnmodels_dir + '/' + model_name + '.params'
if os.path.exists(params_file):
os.remove(params_file)
model_dir = dnnmodels_dir + '/' + model_name
rules_profile_dir = ruleprofiles_dir + '/' + model_name
for dir in [model_dir, rules_profile_dir]:
if os.path.exists(dir):
shutil.rmtree(dir)
self._available_models.remove(model_name)
self.get_availableModels.emit()
@pyqtSlot(str)
def train_rules(self, dataset_name):
canlist = dp.load_canlist(datasets_dir + '/' + dataset_name + '/good_canlist.json')
self._ids.retrain_rules(canlist)
self.get_parameters.emit()
@pyqtSlot(str, int)
def train_dnn(self, dataset_name, num_steps):
features, labels = dp.load_feature_lists(datasets_dir + '/' + dataset_name + '/features_labels.json')
self._ids.train_dnn(dnn_input_function(features, labels, shuffle=True), num_steps)
self.get_parameters.emit()
# These are distinctly not a slot, because it will be called by the Simulation
# Manager, which will take this class in its constructor.
def start_simulation(self):
self._ids.start_simulation()
def judge_single_frame(self, can_frame):
result = self._ids.judge_single_frame(can_frame)
return result
def stop_simulation(self):
self._ids.stop_simulation()
def test_change_parameters(prepared_ids: TwoStageIDS):
# Check that invalid keys raise a ValueError for change_ids_parameters.
with pytest.raises(ValueError):
prepared_ids.change_ids_parameters('invalid_key', 'invalid_value')
# Check that changing parameters works correctly.
ids = prepared_ids
ids.change_ids_parameters('dnn_dir_path', 'test')
ids.change_ids_parameters('rules_profile', 'test')
ids.change_ids_parameters('idprobs_path', 'test')
assert ids.params == {
'dnn_dir_path': 'test',
'rules_profile': 'test',
'idprobs_path': 'test'
}
# Check that initialized DNN can't have parameters changed.
with pytest.raises(RuntimeError):
ids.change_dnn_parameters('hidden_units', 'test')
# Check that invalid keys raise a ValueError for change_dnn_parameters.
ids = TwoStageIDS()
with pytest.raises(ValueError):
ids.change_dnn_parameters('invalid_key', 'invalid_value')
# Check that changing parameters works correctly.
ids.change_dnn_parameters('hidden_units', 'test')
ids.change_dnn_parameters('optimizer', 'test')
ids.change_dnn_parameters('activation_fn', 'test')
ids.change_dnn_parameters('loss_reduction', 'test')
assert ids.dnn._params == {
'hidden_units': 'test',
'optimizer': 'test',
'activation_fn': 'test',
'loss_reduction': 'test'
}
def prepared_ids(canlist_good, feature_lists_labels):
features, labels = feature_lists_labels
ids = TwoStageIDS()
ids.change_ids_parameters('dnn_dir_path',
test_dir + '/../savedata/test_dnn')
ids.change_ids_parameters('rules_profile', 'test_rules')
ids.change_ids_parameters(
'idprobs_path',
test_dir + '/sample_data/two_stage_ids_test/idprobs.json')
ids.init_ids()
if not ids.rules_trained:
ids.retrain_rules(canlist_good)
if not ids.dnn_trained:
ids.train_dnn(dnn_input_function(features, labels, shuffle=True), 2000)
return ids
def test_judge_dataset(prepared_ids: TwoStageIDS, feature_lists_labels,
bad_canlist):
features, labels = feature_lists_labels
ids = TwoStageIDS()
# Test trying to judge dataset when DNN isn't trained.
with pytest.raises(RuntimeError, match='The DNN must be trained'):
ids.judge_dataset(bad_canlist, dnn_input_function(features, labels))
ids.dnn_trained = True
# Test trying to judge dataset when Rules IDS isn't trained.
with pytest.raises(RuntimeError,
match='The Rules Based IDS must be prepared'):
ids.judge_dataset(bad_canlist, dnn_input_function(features, labels))
ids = prepared_ids
ids.start_simulation()
# Test trying to judge dataset while in simulation.
with pytest.raises(RuntimeError,
match='The TwoStageIDS must not be in a simulation'):
ids.judge_dataset(bad_canlist, dnn_input_function(features, labels))
ids.stop_simulation()
# Verify that judging dataset produces a good output.
results = ids.judge_dataset(bad_canlist,
dnn_input_function(features, labels))
for result in results:
assert isinstance(result, tuple) and isinstance(
result[0], bool) and (isinstance(result[1], float)
or isinstance(result[1], str))