def test_training_infer(config): """Test anomaly detection training on public dataset.""" model_adapter = SomModelAdapter( SomStorageAdapter(config=config, feedback_strategy=None)) tc_train = SomTrainCommand(node_map=2, model_adapter=model_adapter, recreate_model=True) result, dist = tc_train.execute() assert result == 0 model_adapter = SomModelAdapter( SomStorageAdapter(config=config, feedback_strategy=None)) tc_infer = SomInferCommand(model_adapter=model_adapter, sleep=False) result = tc_infer.execute() assert result == 0
def test_model_shape(config): """Test that the trained model size is expected based on given parameters.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() assert model_adapter.model.model.shape[0:2] == (2, 2)
def test_output_length(config): """Test that correct number of outputs are generated with Hadoop_2k.json.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() assert len(dist) == 2000
def test_output_values(config): """Test that all distance values in training set are less than or equal to 1 on Hadoop_2k.json.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() assert sum(dist) <= 2000
def test_log_similarity(cnf_hadoop2k_w2v_params): """Check that two words have consistent similar logs after training.""" storage_adapter = SomStorageAdapter(config=cnf_hadoop2k_w2v_params, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() log_1 = 'INFOmainorgapachehadoopmapreducevappMRAppMasterExecutingwithtokens' answer_1 = 'INFOmainorgapachehadoopmapreducevappMRAppMasterCreatedMRAppMasterforapplicationappattempt' match_1 = [ model_adapter.w2v_model.model["message"].wv.most_similar(log_1)[i][0] for i in range(3) ] assert answer_1 in match_1 log_2 = 'ERRORRMCommunicatorAllocatororgapachehadoopmapreducevapprmRMContainerAllocatorERRORINCONTACTINGRM' answer_2 = 'WARNLeaseRenewermsrabimsrasaorgapachehadoophdfsLeaseRenewerFailedtorenewleaseforDFSClient' \ 'NONMAPREDUCEforsecondsWillretryshortly' match_2 = [ model_adapter.w2v_model.model["message"].wv.most_similar(log_2)[i][0] for i in range(3) ] logging.info(match_2[0]) assert answer_2 in match_2
def test_end2endtraining(config): """Test anomaly detection training on public dataset.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainCommand(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() assert result == 0
def __init__(self, config, feedback_strategy=None): """Abstraction around model adapter run method.""" if feedback_strategy is None: feedback_strategy = FeedbackStrategy(config=config) storage_adapter = SomStorageAdapter(config, feedback_strategy) self.__model_adapter = SomModelAdapter(storage_adapter) self.mgr = TaskQueue()
def test_vocab_length(config): """Check length of processed vocab on on Hadoop_2k.json.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() assert len(model_adapter.w2v_model.model["message"].wv.vocab) == 141
def test_loss_value(config): """Check the loss value is not greater then during testing.""" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) result, dist = tc.execute() print(model_adapter.w2v_model.model["message"].get_latest_training_loss()) tl = model_adapter.w2v_model.model["message"].get_latest_training_loss() assert tl < 320000.0
def get_score(config, node_map, feedback): """Simple utility function for injecting custom mock function into Detector.""" feedback_strategy = FeedbackStrategy(config, fn=feedback) storage_adapter = SomStorageAdapter(config=config, feedback_strategy=feedback_strategy) model_adapter = SomModelAdapter(storage_adapter=storage_adapter) tc = SomTrainCommand(node_map=node_map, model_adapter=model_adapter) success, dist = tc.execute() freq_one = dist[-1] return freq_one
def test_train_command(cnf_hadoop_2k, pipeline): """Test case for validating that when we train a model and add it to task queue that it will run.""" storage_adapter = SomStorageAdapter(config=cnf_hadoop_2k, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter) train_job = SomTrainJob(node_map=2, model_adapter=model_adapter) pipeline.add_steps(train_job) assert len(pipeline) == TASKS_IN_QUEUE assert pipeline.count != TASKS_IN_QUEUE pipeline.execute_steps() assert pipeline.count == TASKS_IN_QUEUE
def __init__(self, config, feedback_strategy=None, tracing_enabled=False): """Set up required properties to run training or prediction. :param config: configuration provided via yaml or environment variables :param feedback_strategy: a function that runs to improve the feedback of system """ if feedback_strategy is None: feedback_strategy = FeedbackStrategy(config=config) storage_adapter = SomStorageAdapter(config, feedback_strategy) self.__model_adapter = SomModelAdapter(storage_adapter) self.tasks = TaskQueue() self.tracing_enabled = tracing_enabled
def test_train_command(self): """Test case for validating that when we train a model and add it to task queue that it will run.""" mgr = TaskQueue() config = Configuration(config_yaml="config_files/.env_config.yaml") storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter) tc = SomTrainCommand(node_map=2, model_adapter=model_adapter) mgr.add_steps(tc) self.assertEqual(len(mgr), TASKS_IN_QUEUE) self.assertNotEqual(mgr.count, TASKS_IN_QUEUE) mgr.execute_steps() self.assertEqual(mgr.count, TASKS_IN_QUEUE) mgr.clear()
def test_train_command(self): """Test case for validating that when we train a model and add it to task queue that it will run.""" mgr = DetectorPipeline() config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) mgr.add_steps(tc) self.assertEqual(len(mgr), TASKS_IN_QUEUE) self.assertNotEqual(mgr.count, TASKS_IN_QUEUE) mgr.execute_steps() self.assertEqual(mgr.count, TASKS_IN_QUEUE) mgr.clear()
def __init__(self, config): """Abstraction around model adapter run method.""" storage_adapter = SomStorageAdapter(config) self.__model_adapter = SomModelAdapter(storage_adapter)