def config():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
return config
def config():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_BACKEND = "local"
config.LS_INPUT_PATH = "validation_data/orders-500.log"
config.LS_OUTPUT_PATH = "validation_data/results-oct4.1.txt"
return config
def cnf_localdir():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "localdir"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/test_sample_input"
return config
def cnf_100K_events():
"""Provide default configurations to load yaml instead of env var."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/log_anomaly_detector-100000-events.json"
return config
def config():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
config.W2V_MIN_COUNT = 1
config.W2V_ITER = 500
config.W2V_COMPUTE_LOSS = "True"
config.W2V_SEED = 50
config.W2V_WORKERS = 1
return config
def detector():
"""Initialize configurations before testing."""
# prefix=None, config_yaml=None):
config = Configuration(prefix=CONFIGURATION_PREFIX,
config_yaml=".env_config.yaml")
anomaly_detector = AnomalyDetector(config)
return anomaly_detector
def run(job_type: str, config_yaml: str, single_run: bool,
tracing_enabled: bool):
"""Perform machine learning model generation with input log data.
:param job_type: provide user the ability to run one training or inference or both.
:param config_yaml: provides path to the config file to load into application.
:param single_run: for running the system a single time.
:param tracing_enabled: enabling open tracing to see the performance.
:return: None
"""
click.echo("Starting...")
config = Configuration(prefix=CONFIGURATION_PREFIX,
config_yaml=config_yaml)
anomaly_detector = Facade(config=config, tracing_enabled=tracing_enabled)
click.echo("Created jobtype {}".format(job_type))
if job_type == "train":
click.echo("Performing training...")
anomaly_detector.train()
elif job_type == "inference":
click.echo("Perform inference...")
anomaly_detector.infer()
elif job_type == "all":
click.echo("Perform training and inference in loop...")
anomaly_detector.run(single_run=single_run)
def test_train_command(self):
"""Test case for validating that when we train a model and add it to task queue that it will run."""
mgr = DetectorPipeline()
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "stdout"
config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json"
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter)
tc = SomTrainJob(node_map=2, model_adapter=model_adapter)
mgr.add_steps(tc)
self.assertEqual(len(mgr), TASKS_IN_QUEUE)
self.assertNotEqual(mgr.count, TASKS_IN_QUEUE)
mgr.execute_steps()
self.assertEqual(mgr.count, TASKS_IN_QUEUE)
mgr.clear()
def cnf_local_500():
"""Initialize configurations before testing."""
config = Configuration()
config.STORAGE_DATASOURCE = "local"
config.STORAGE_DATASINK = "local"
config.LS_INPUT_PATH = "validation_data/orders-500.log"
config.LS_OUTPUT_PATH = "validation_data/results-oct4.1.txt"
return config
def test_train_command(self):
"""Test case for validating that when we train a model and add it to task queue that it will run."""
mgr = TaskQueue()
config = Configuration(config_yaml="config_files/.env_config.yaml")
storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None)
model_adapter = SomModelAdapter(storage_adapter)
tc = SomTrainCommand(node_map=2, model_adapter=model_adapter)
mgr.add_steps(tc)
self.assertEqual(len(mgr), TASKS_IN_QUEUE)
self.assertNotEqual(mgr.count, TASKS_IN_QUEUE)
mgr.execute_steps()
self.assertEqual(mgr.count, TASKS_IN_QUEUE)
mgr.clear()
def run(job_type, config_yaml, single_run):
"""Perform machine learning model generation with input log data."""
click.echo("Starting...")
config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=config_yaml)
anomaly_detector = AnomalyDetectorFacade(config)
click.echo("Created jobtype {}".format(job_type))
if job_type == "train":
click.echo("Performing training...")
anomaly_detector.train()
elif job_type == "inference":
click.echo("Perform inference...")
anomaly_detector.infer()
elif job_type == "all":
click.echo("Perform training and inference in loop...")
anomaly_detector.run(single_run=single_run)
def config():
"""Initialize configurations before testing."""
config = Configuration(prefix=CONFIGURATION_PREFIX,
config_yaml="config_files/.env_local_dir.yaml")
return config
def config():
"""Provide default configurations to load yaml instead of env var."""
config = Configuration(config_yaml=".test_env_config.yaml")
return config
def detector():
"""Provide default configurations to load yaml instead of env var."""
config = Configuration(prefix=CONFIGURATION_PREFIX,
config_yaml=".test_env_config.yaml")
anomaly_detector = AnomalyDetectorFacade(config)
return anomaly_detector
def _main():
_LOGGER.info("Starting...")
config = Configuration(CONFIGURATION_PREFIX)
anomaly_detector = AnomalyDetector(config)
anomaly_detector.run()