def config():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
return config
def cnf_localdir():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "localdir"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/test_sample_input"
return config
def cnf_100K_events():
"""Provide default configurations to load yaml instead of env var."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/log_anomaly_detector-100000-events.json"
return config
def cnf_local_500():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "local"
config.LS_INPUT_PATH = "validation_data/orders-500.log"
config.LS_OUTPUT_PATH = "validation_data/results-oct4.1.txt"
return config
def config():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
config.W2V_MIN_COUNT = 1
config.W2V_ITER = 500
config.W2V_COMPUTE_LOSS = "True"
config.W2V_SEED = 50
config.W2V_WORKERS = 1
return config
def test_train_command(self):
"""Test case for validating that when we train a model and add it to task queue that it will run."""
mgr = DetectorPipeline()
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter)
tc = SomTrainJob(node_map=2, model_adapter=model_adapter)
mgr.add_steps(tc)
self.assertEqual(len(mgr), TASKS_IN_QUEUE)
self.assertNotEqual(mgr.count, TASKS_IN_QUEUE)
mgr.execute_steps()
self.assertEqual(mgr.count, TASKS_IN_QUEUE)
mgr.clear()