def config(): """Initialize configurations before testing.""" config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json" return config
def config(): """Initialize configurations before testing.""" config = Configuration() config.STORAGE_BACKEND = "local" config.LS_INPUT_PATH = "validation_data/orders-500.log" config.LS_OUTPUT_PATH = "validation_data/results-oct4.1.txt" return config
def cnf_localdir(): """Initialize configurations before testing.""" config = Configuration() config.STORAGE_DATASOURCE = "localdir" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/test_sample_input" return config
def detector(): """Initialize configurations before testing.""" # prefix=None, config_yaml=None): config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=".env_config.yaml") anomaly_detector = AnomalyDetector(config) return anomaly_detector
def cnf_100K_events(): """Provide default configurations to load yaml instead of env var.""" config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/log_anomaly_detector-100000-events.json" return config
def run(job_type: str, config_yaml: str, single_run: bool, tracing_enabled: bool): """Perform machine learning model generation with input log data. :param job_type: provide user the ability to run one training or inference or both. :param config_yaml: provides path to the config file to load into application. :param single_run: for running the system a single time. :param tracing_enabled: enabling open tracing to see the performance. :return: None """ click.echo("Starting...") config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=config_yaml) anomaly_detector = Facade(config=config, tracing_enabled=tracing_enabled) click.echo("Created jobtype {}".format(job_type)) if job_type == "train": click.echo("Performing training...") anomaly_detector.train() elif job_type == "inference": click.echo("Perform inference...") anomaly_detector.infer() elif job_type == "all": click.echo("Perform training and inference in loop...") anomaly_detector.run(single_run=single_run)
def cnf_local_500(): """Initialize configurations before testing.""" config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "local" config.LS_INPUT_PATH = "validation_data/orders-500.log" config.LS_OUTPUT_PATH = "validation_data/results-oct4.1.txt" return config
def config(): """Initialize configurations before testing.""" config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json" config.W2V_MIN_COUNT = 1 config.W2V_ITER = 500 config.W2V_COMPUTE_LOSS = "True" config.W2V_SEED = 50 config.W2V_WORKERS = 1 return config
def test_train_command(self): """Test case for validating that when we train a model and add it to task queue that it will run.""" mgr = TaskQueue() config = Configuration(config_yaml="config_files/.env_config.yaml") storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter) tc = SomTrainCommand(node_map=2, model_adapter=model_adapter) mgr.add_steps(tc) self.assertEqual(len(mgr), TASKS_IN_QUEUE) self.assertNotEqual(mgr.count, TASKS_IN_QUEUE) mgr.execute_steps() self.assertEqual(mgr.count, TASKS_IN_QUEUE) mgr.clear()
def run(job_type, config_yaml, single_run): """Perform machine learning model generation with input log data.""" click.echo("Starting...") config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=config_yaml) anomaly_detector = AnomalyDetectorFacade(config) click.echo("Created jobtype {}".format(job_type)) if job_type == "train": click.echo("Performing training...") anomaly_detector.train() elif job_type == "inference": click.echo("Perform inference...") anomaly_detector.infer() elif job_type == "all": click.echo("Perform training and inference in loop...") anomaly_detector.run(single_run=single_run)
def test_train_command(self): """Test case for validating that when we train a model and add it to task queue that it will run.""" mgr = DetectorPipeline() config = Configuration() config.STORAGE_DATASOURCE = "local" config.STORAGE_DATASINK = "stdout" config.LS_INPUT_PATH = "validation_data/Hadoop_2k.json" storage_adapter = SomStorageAdapter(config=config, feedback_strategy=None) model_adapter = SomModelAdapter(storage_adapter) tc = SomTrainJob(node_map=2, model_adapter=model_adapter) mgr.add_steps(tc) self.assertEqual(len(mgr), TASKS_IN_QUEUE) self.assertNotEqual(mgr.count, TASKS_IN_QUEUE) mgr.execute_steps() self.assertEqual(mgr.count, TASKS_IN_QUEUE) mgr.clear()
def config(): """Initialize configurations before testing.""" config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml="config_files/.env_local_dir.yaml") return config
def config(): """Provide default configurations to load yaml instead of env var.""" config = Configuration(config_yaml=".test_env_config.yaml") return config
def detector(): """Provide default configurations to load yaml instead of env var.""" config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=".test_env_config.yaml") anomaly_detector = AnomalyDetectorFacade(config) return anomaly_detector
def _main(): _LOGGER.info("Starting...") config = Configuration(CONFIGURATION_PREFIX) anomaly_detector = AnomalyDetector(config) anomaly_detector.run()