def test_end2endtraining(config):
"""Test anomaly detection training on public dataset."""
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter=storage_adapter)
tc = SomTrainJob(node_map=2, model_adapter=model_adapter)
result, dist = tc.execute()
assert result == 0
def get_score(config, node_map, feedback):
"""Simple utility function for injecting custom mock function into Detector."""
feedback_strategy = FeedbackStrategy(config, func=feedback)
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=feedback_strategy)
model_adapter = SomModelAdapter(storage_adapter=storage_adapter)
tc = SomTrainJob(node_map=node_map, model_adapter=model_adapter)
success, dist = tc.execute()
freq_one = dist[-1]
return freq_one
def test_training_infer(config):
"""Test anomaly detection training on public dataset."""
model_adapter = SomModelAdapter(SomStorageAdapter(config=config, feedback_strategy=None))
tc_train = SomTrainJob(node_map=2, model_adapter=model_adapter, recreate_model=True)
result, dist = tc_train.execute()
assert result == 0
model_adapter = SomModelAdapter(SomStorageAdapter(config=config, feedback_strategy=None))
tc_infer = SomInferenceJob(model_adapter=model_adapter, sleep=False)
result = tc_infer.execute()
assert result == 0
def _sompy_train_job(cls, config, feedback_strategy):
"""Perform Training and inference of SOMPY Model."""
pipeline = DetectorPipeline()
model_adapter = cls.create_sompy_modeladapter(config,
feedback_strategy)
train = SomTrainJob(node_map=config.SOMPY_NODE_MAP,
model_adapter=model_adapter)
pipeline.add_steps(train)
return pipeline
def test_train_command(cnf_hadoop_2k, pipeline):
"""Test case for validating that when we train a model and add it to task queue that it will run."""
storage_adapter = SomStorageAdapter(config=cnf_hadoop_2k,
feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter)
train_job = SomTrainJob(node_map=2, model_adapter=model_adapter)
pipeline.add_steps(train_job)
assert len(pipeline) == TASKS_IN_QUEUE
assert pipeline.count != TASKS_IN_QUEUE
pipeline.execute_steps()
assert pipeline.count == TASKS_IN_QUEUE
def test_train_command(self):
"""Test case for validating that when we train a model and add it to task queue that it will run."""
mgr = DetectorPipeline()
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter)
tc = SomTrainJob(node_map=2, model_adapter=model_adapter)
mgr.add_steps(tc)
self.assertEqual(len(mgr), TASKS_IN_QUEUE)
self.assertNotEqual(mgr.count, TASKS_IN_QUEUE)
mgr.execute_steps()
self.assertEqual(mgr.count, TASKS_IN_QUEUE)
mgr.clear()