def test_transform_fit( self, resources, framework, problem, weights, tmp_path, ): language = PYTHON custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language=framework, ) input_dataset = resources.datasets(framework, problem) weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd( weights, input_dataset, r_fit=language == R_FIT ) target_type = TRANSFORM cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format( ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir, input_dataset ) if problem != ANOMALY: cmd += " --target {}".format(resources.targets(problem)) if problem in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=target_type ) cmd += weights_cmd _exec_shell_cmd( cmd, "Failed in {} command line! {}".format(ArgumentsOptions.MAIN_COMMAND, cmd) )
def test_fit_hyperparameters( self, resources, framework, problem, docker, parameters, weights, tmp_path, ): if framework == RDS_HYPERPARAMETERS: language = R_FIT else: language = PYTHON custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, is_training=True, ) input_dataset = resources.datasets(framework, problem) parameter_file = resources.datasets(framework, parameters) weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd( weights, input_dataset, r_fit=language == R_FIT) target_type = resources.target_types( problem) if "transform" not in framework else TRANSFORM cmd = "{} fit --target-type {} --code-dir {} --input {} --parameter-file {} --verbose ".format( ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir, input_dataset, parameter_file, ) if problem != ANOMALY: cmd += " --target {}".format(resources.targets(problem)) if problem in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels(cmd, resources.class_labels( framework, problem), target_type=target_type) if docker: cmd += " --docker {} ".format(docker) cmd += weights_cmd _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd))
def test_fit_for_use_output_and_nested( self, resources, framework, problem, docker, weights, use_output, tmp_path, nested, ): if docker and framework != SKLEARN: return if framework == RDS: language = R_FIT else: language = PYTHON custom_model_dir = _create_custom_model_dir(resources, tmp_path, framework, problem, language, is_training=True, nested=nested) input_dataset = resources.datasets(framework, problem) weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd( weights, input_dataset, r_fit=language == R_FIT) output = tmp_path / "output" output.mkdir() cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format( ArgumentsOptions.MAIN_COMMAND, problem, custom_model_dir, input_dataset) if problem != ANOMALY: cmd += " --target {}".format(resources.targets(problem)) if use_output: cmd += " --output {}".format(output) if problem == BINARY: cmd = _cmd_add_class_labels(cmd, resources.class_labels( framework, problem), target_type=problem) if docker: cmd += " --docker {} ".format(docker) cmd += weights_cmd _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd))
def test_docker_image_with_wrong_dep_install( self, resources, framework, problem, code_dir, env_dir, tmp_path, ): custom_model_dir = os.path.join(MODEL_TEMPLATES_PATH, code_dir) tmp_dir = tmp_path / "tmp_code_dir" custom_model_dir = shutil.copytree(custom_model_dir, tmp_dir) with open(os.path.join(custom_model_dir, "requirements.txt"), mode="w") as f: f.write("\nnon_existing_dep") docker_env = os.path.join(PUBLIC_DROPIN_ENVS_PATH, env_dir) input_dataset = resources.datasets(framework, problem) output = tmp_path / "output" cmd = '{} score --code-dir {} --input "{}" --output {} --target-type {}'.format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, output, resources.target_types(problem), ) cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), multiclass_label_file=None, ) cmd += " --docker {} --verbose ".format(docker_env) _, _, stde = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) assert re.search( r"ERROR drum: Failed to build a docker image", stde, ) assert re.search( r"Could not find a version that satisfies the requirement non_existing_dep", stde, )
def test_fit_variety(framework, variety_resources, resources, variety_data_names, tmp_path): # get data info from fixtures df = variety_data_names df_path = variety_resources.dataset(df) problem = variety_resources.problem(df) target = variety_resources.target(df) if problem == BINARY: class_labels = variety_resources.class_labels(df) if framework == RDS: # there's one annoying dataset where R needs 0 and 1 and python wants 1.0 and 0.0 class_labels = [ int(x) if type(x) is float else x for x in class_labels ] # figure out language if framework == RDS: language = R_FIT else: language = PYTHON custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, is_training=True, nested=False, ) output = tmp_path / "output" output.mkdir() cmd = "{} fit --code-dir {} --input {} --verbose ".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, df_path) if problem == ANOMALY: cmd += " --unsupervised" else: cmd += " --target {}".format(target) if problem == BINARY: cmd = _cmd_add_class_labels(cmd, class_labels, target_type=problem) p, _, err = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format(ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) if p.returncode != 0: raise AssertionError(err)
def test_custom_models_perf_test( self, resources, framework, problem, language, docker, timeout, tmp_path, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) input_dataset = resources.datasets(framework, problem) cmd = "{} perf-test -i 200 -s 1000 --code-dir {} --input {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, resources.target_types(problem), ) if timeout is not None: cmd += " --timeout {}".format(timeout) if resources.target_types(problem) in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), ) if docker: cmd += " --docker {}".format(docker) _, stdo, _ = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd)) if timeout is not None: expected_str = "timed out ({}s)".format(timeout) assert expected_str in stdo assert "NA" in stdo else: assert "NA" not in stdo
def test_custom_models_with_drum( self, resources, framework, problem, language, docker, tmp_path, use_labels_file, temp_file, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) input_dataset = resources.datasets(framework, problem) output = tmp_path / "output" cmd = '{} score --code-dir {} --input "{}" --output {} --target-type {}'.format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, output, resources.target_types(problem), ) if resources.target_types(problem) in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), multiclass_label_file=temp_file if use_labels_file else None, ) if docker: cmd += " --docker {} --verbose ".format(docker) _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd)) in_data = pd.read_csv(input_dataset) out_data = pd.read_csv(output) assert in_data.shape[0] == out_data.shape[0]
def test_custom_models_validation_test( self, resources, framework, problem, language, docker, tmp_path, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) input_dataset = resources.datasets(framework, problem) cmd = "{} validation --code-dir {} --input {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, resources.target_types(problem), ) if problem == BINARY: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), ) if docker: cmd += " --docker {}".format(docker) _, stdo, _ = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) if language == NO_CUSTOM: assert re.search(r"Null value imputation\s+FAILED", stdo) else: assert re.search(r"Null value imputation\s+PASSED", stdo)
def test_prediction_consistency(self, resources, tmp_path, framework, problem): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, SPARSE, language=PYTHON, is_training=True, ) input_dataset = resources.datasets(framework, problem) if problem in [BINARY_TEXT, BINARY_BOOL]: target_type = BINARY else: target_type = problem cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format( ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir, input_dataset) cmd += " --target {}".format(resources.targets(problem)) if target_type in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type) _, stdout, stderr = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=True, ) # we should throw a warning, not an error assert "Your predictions were different when we tried to predict twice." in stderr # but don't error out assert ( "Your model can be fit to your data, and predictions can be made on the fit model!" in stdout) # clean up sample_dir = stderr.split(":")[-1] if sample_dir.endswith("\n"): sample_dir = sample_dir[:-1] os.remove(sample_dir.strip())
def _drum_with_monitoring(resources, framework, problem, language, docker, tmp_path): """ We expect the run of drum to be ok, since mlops is assumed to be installed. """ custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) mlops_spool_dir = tmp_path / "mlops_spool" os.mkdir(str(mlops_spool_dir)) input_dataset = resources.datasets(framework, problem) output = tmp_path / "output" cmd = "{} score --code-dir {} --input {} --output {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, output, resources.target_types(problem), ) monitor_settings = ( "spooler_type=filesystem;directory={};max_files=1;file_max_size=1024000" .format(mlops_spool_dir)) cmd += ' --monitor --model-id 555 --deployment-id 777 --monitor-settings="{}"'.format( monitor_settings) if problem == BINARY: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), ) if docker: cmd += " --docker {} --verbose ".format(docker) return cmd, input_dataset, output, mlops_spool_dir
def __init__( self, target_type, labels, custom_model_dir, docker=None, with_error_server=False, show_stacktrace=True, nginx=False, memory=None, fail_on_shutdown_error=True, ): port = CMRunnerUtils.find_free_port() self.server_address = "localhost:{}".format(port) url_host = os.environ.get("TEST_URL_HOST", "localhost") if docker: self.url_server_address = "http://{}:{}".format(url_host, port) else: self.url_server_address = "http://localhost:{}".format(port) cmd = "{} server --code-dir {} --target-type {} --address {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, target_type, self.server_address) if labels: cmd = _cmd_add_class_labels(cmd, labels, target_type=target_type) if docker: cmd += " --docker {}".format(docker) if memory: cmd += " --memory {}".format(memory) if with_error_server: cmd += " --with-error-server" if show_stacktrace: cmd += " --show-stacktrace" if nginx: cmd += " --production" self._cmd = cmd self._process_object_holder = DrumServerProcess() self._server_thread = None self._with_nginx = nginx self._fail_on_shutdown_error = fail_on_shutdown_error
def test_classification_validation_fails_on_add_up_to_one( self, resources, framework, problem, language, tmp_path, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) input_dataset = resources.datasets(framework, problem) cmd = "{} validation --code-dir {} --input {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, resources.target_types(problem), ) if problem == BINARY: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), ) _, stdo, _ = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) assert re.search(r"Basic batch prediction\s+FAILED", stdo) assert re.search(r"Null value imputation\s+FAILED", stdo)
def test_custom_models_perf_test( self, resources, framework, problem, language, docker, tmp_path, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, problem, language, ) input_dataset = resources.datasets(framework, problem) cmd = "{} perf-test -i 200 -s 1000 --code-dir {} --input {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, resources.target_types(problem), ) if resources.target_types(problem) in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), ) if docker: cmd += " --docker {}".format(docker) _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd))
def test_classification_validation_fails( self, resources, framework, language, target_type, tmp_path, ): custom_model_dir = _create_custom_model_dir( resources, tmp_path, framework, None, language, ) input_dataset = resources.datasets(framework, BINARY) cmd = "{} score --code-dir {} --input {} --target-type {}".format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, target_type) if resources.target_types(target_type) in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, target_type), target_type=resources.target_types(target_type), ) _, _, stde = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) assert "Your prediction probabilities do not add up to 1." in str(stde)
def __init__( self, target_type, labels, custom_model_dir, docker=None, with_error_server=False, show_stacktrace=True, nginx=False, memory=None, fail_on_shutdown_error=True, pass_args_as_env_vars=False, verbose=True, append_cmd=None, ): port = CMRunnerUtils.find_free_port() self.server_address = "localhost:{}".format(port) url_host = os.environ.get("TEST_URL_HOST", "localhost") if docker: self.url_server_address = "http://{}:{}".format(url_host, port) else: self.url_server_address = "http://localhost:{}".format(port) cmd = "{} server".format(ArgumentsOptions.MAIN_COMMAND) if pass_args_as_env_vars: os.environ[ArgumentOptionsEnvVars.CODE_DIR] = str(custom_model_dir) os.environ[ArgumentOptionsEnvVars.TARGET_TYPE] = target_type os.environ[ArgumentOptionsEnvVars.ADDRESS] = self.server_address else: cmd += " --code-dir {} --target-type {} --address {}".format( custom_model_dir, target_type, self.server_address ) if labels: cmd = _cmd_add_class_labels( cmd, labels, target_type=target_type, pass_args_as_env_vars=pass_args_as_env_vars ) if docker: cmd += " --docker {}".format(docker) if memory: cmd += " --memory {}".format(memory) if with_error_server: if pass_args_as_env_vars: os.environ[ArgumentOptionsEnvVars.WITH_ERROR_SERVER] = "1" else: cmd += " --with-error-server" if show_stacktrace: if pass_args_as_env_vars: os.environ[ArgumentOptionsEnvVars.SHOW_STACKTRACE] = "1" else: cmd += " --show-stacktrace" if nginx: if pass_args_as_env_vars: os.environ[ArgumentOptionsEnvVars.PRODUCTION] = "1" else: cmd += " --production" if append_cmd is not None: cmd += " " + append_cmd self._cmd = cmd self._process_object_holder = DrumServerProcess() self._server_thread = None self._with_nginx = nginx self._fail_on_shutdown_error = fail_on_shutdown_error self._verbose = verbose
def test_docker_image_with_deps_install( self, resources, framework, problem, code_dir, env_dir, skip_deps_install, tmp_path, ): custom_model_dir = os.path.join(MODEL_TEMPLATES_PATH, code_dir) if framework == CODEGEN and not skip_deps_install: tmp_dir = tmp_path / "tmp_code_dir" custom_model_dir = shutil.copytree(custom_model_dir, tmp_dir) with open(os.path.join(custom_model_dir, "requirements.txt"), mode="w") as f: f.write("deps_are_not_supported_in_java") docker_env = os.path.join(PUBLIC_DROPIN_ENVS_PATH, env_dir) input_dataset = resources.datasets(framework, problem) output = tmp_path / "output" cmd = '{} score --code-dir {} --input "{}" --output {} --target-type {}'.format( ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, output, resources.target_types(problem), ) if resources.target_types(problem) in [BINARY, MULTICLASS]: cmd = _cmd_add_class_labels( cmd, resources.class_labels(framework, problem), target_type=resources.target_types(problem), multiclass_label_file=None, ) cmd += " --docker {} --verbose ".format(docker_env) if skip_deps_install: cmd += " --skip-deps-install" _, stdo, stde = _exec_shell_cmd( cmd, "Failed in {} command line! {}".format( ArgumentsOptions.MAIN_COMMAND, cmd), assert_if_fail=False, ) if skip_deps_install: # requirements.txt is not supported for java models, so test should pass if framework == CODEGEN: in_data = pd.read_csv(input_dataset) out_data = pd.read_csv(output) assert in_data.shape[0] == out_data.shape[0] else: assert re.search( r"ERROR drum: Error from docker process:", stde, ) else: if framework is None and problem == UNSTRUCTURED: with open(output) as f: out_data = f.read() assert "10" in out_data elif framework == PYTORCH and problem == MULTICLASS: in_data = pd.read_csv(input_dataset) out_data = pd.read_csv(output) assert in_data.shape[0] == out_data.shape[0] elif framework == CODEGEN and problem == REGRESSION: assert re.search( r"WARNING drum: Dependencies management is not supported for the 'java' language and will not be installed into an image", stde, ) else: assert False