def test_duplicate_target_name(self, resources, tmp_path):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
SKLEARN_REGRESSION,
SPARSE,
language=PYTHON,
is_training=True,
)
input_dataset = resources.datasets(SKLEARN_REGRESSION,
TARGET_NAME_DUPLICATED_X)
target_dataset = resources.datasets(SKLEARN_REGRESSION,
TARGET_NAME_DUPLICATED_Y)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --code-dir {} --input {} --target-type {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset,
REGRESSION)
cmd += " --target-csv " + target_dataset
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_predictors_supported_payload_formats(
self,
resources,
framework,
problem,
language,
supported_payload_formats,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
resources.target_types(problem),
resources.class_labels(framework, problem),
custom_model_dir,
) as run:
response = requests.get(run.url_server_address + "/capabilities/")
assert response.ok
assert response.json() == {
"supported_payload_formats": supported_payload_formats
}
def test_perf_test_drum_server_kill(
self, resources, framework, problem, language, docker, tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources, tmp_path, framework, problem, language,
)
input_dataset = resources.datasets(framework, problem)
cmd = "{} perf-test -i 10 -s 10 --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
# wait for drum perf-test server from prev test case to be stopped
time.sleep(0.5)
assert _find_drum_perf_test_server_process() is None
p = subprocess.Popen(cmd, shell=True, env=os.environ, universal_newlines=True,)
time.sleep(2)
# kill drum perf-test process, child server should be running
p.kill()
pid = _find_drum_perf_test_server_process()
assert pid is not None
_kill_drum_perf_test_server_process(pid)
assert _find_drum_perf_test_server_process() is None
def test_custom_model_with_all_hooks(
self,
resources,
framework,
language,
hooks_list,
target_type,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
None,
language,
)
input_dataset = resources.datasets(framework, REGRESSION)
output = tmp_path / "output"
cmd = "{} score --code-dir {} --input {} --output {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset,
output, target_type)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
if hooks_list == CustomHooks.ALL_PREDICT_STRUCTURED:
preds = pd.read_csv(output)
assert all(val for val in (
preds["Predictions"] == len(hooks_list)).values), preds
elif hooks_list == CustomHooks.ALL_PREDICT_UNSTRUCTURED:
with open(output) as f:
all_data = f.read()
assert str(len(hooks_list)) in all_data
def test_fit_schema_validation(self, resources, tmp_path):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
SKLEARN_BINARY_SCHEMA_VALIDATION,
BINARY,
PYTHON,
is_training=True,
include_metadata=True,
)
input_dataset = resources.datasets(SKLEARN, BINARY)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --target-type {} --code-dir {} --target {} --input {} --verbose".format(
ArgumentsOptions.MAIN_COMMAND,
BINARY,
custom_model_dir,
resources.targets(BINARY),
input_dataset,
)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_drum_push_training(
self,
resources,
framework,
problem,
language,
tmp_path,
get_target,
sklearn_drop_in_env,
):
custom_model_dir = _create_custom_model_dir(resources,
tmp_path,
framework,
problem,
language,
is_training=True)
env_id, _ = sklearn_drop_in_env
yaml_string = get_push_yaml(env_id,
resources.datasets(framework, problem),
problem, get_target(problem))
with open(os.path.join(custom_model_dir, "model-metadata.yaml"),
"w") as outfile:
yaml.dump(yaml.load(yaml_string), outfile)
cmd = "{} push --code-dir {} --verbose".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_transform_fit(
self, resources, framework, problem, weights, tmp_path,
):
language = PYTHON
custom_model_dir = _create_custom_model_dir(
resources, tmp_path, framework, problem, language=framework,
)
input_dataset = resources.datasets(framework, problem)
weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd(
weights, input_dataset, r_fit=language == R_FIT
)
target_type = TRANSFORM
cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir, input_dataset
)
if problem != ANOMALY:
cmd += " --target {}".format(resources.targets(problem))
if problem in [BINARY, MULTICLASS]:
cmd = _cmd_add_class_labels(
cmd, resources.class_labels(framework, problem), target_type=target_type
)
cmd += weights_cmd
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(ArgumentsOptions.MAIN_COMMAND, cmd)
)
def test_custom_models_with_drum_nginx_prediction_server(
self,
resources,
framework,
problem,
language,
docker,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
resources.target_types(problem),
resources.class_labels(framework, problem),
custom_model_dir,
docker,
nginx=True,
) as run:
input_dataset = resources.datasets(framework, problem)
# do predictions
for endpoint in ["/predict/", "/predictions/"]:
for post_args in [
{
"files": {
"X": open(input_dataset)
}
},
{
"data": open(input_dataset, "rb")
},
]:
response = requests.post(run.url_server_address + endpoint,
**post_args)
assert response.ok
actual_num_predictions = len(
json.loads(response.text)[RESPONSE_PREDICTIONS_KEY])
in_data = pd.read_csv(input_dataset)
assert in_data.shape[0] == actual_num_predictions
# test model info
response = requests.get(run.url_server_address + "/info/")
assert response.ok
response_dict = response.json()
for key in ModelInfoKeys.REQUIRED:
assert key in response_dict
assert response_dict[
ModelInfoKeys.TARGET_TYPE] == resources.target_types(problem)
assert response_dict[ModelInfoKeys.DRUM_SERVER] == "nginx + uwsgi"
assert response_dict[ModelInfoKeys.DRUM_VERSION] == drum_version
assert ModelInfoKeys.MODEL_METADATA in response_dict
def test_fit_sparse(self, resources, tmp_path, framework):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
SPARSE,
language=R_FIT if framework == RDS else PYTHON,
is_training=True,
)
input_dataset = resources.datasets(framework, SPARSE)
target_dataset = resources.datasets(framework, SPARSE_TARGET)
columns = resources.datasets(framework, SPARSE_COLUMNS)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --code-dir {} --input {} --target-type {} --verbose --sparse-column-file {}".format(
ArgumentsOptions.MAIN_COMMAND, custom_model_dir, input_dataset, REGRESSION, columns
)
cmd += " --target-csv " + target_dataset
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(ArgumentsOptions.MAIN_COMMAND, cmd)
)
def test_unstructured_mode_prediction_server_wrong_endpoint(
self,
resources,
framework,
problem,
language,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
"unstructured",
resources.class_labels(framework, problem),
custom_model_dir,
) as run:
for endpoint in ["/predict/", "/predictions/"]:
response = requests.post(url=run.url_server_address + endpoint)
assert response.status_code == HTTP_422_UNPROCESSABLE_ENTITY
expected_msg = "ERROR: This model has target type 'unstructured', use the /predictUnstructured/ or /predictionsUnstructured/ endpoint."
assert response.json()["message"] == expected_msg
def test_fit_schema_failure(self, resources, tmp_path):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
SKLEARN_BINARY_SCHEMA_VALIDATION,
BINARY,
PYTHON,
is_training=True,
include_metadata=True,
)
input_dataset = resources.datasets(SKLEARN, BINARY_TEXT)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --target-type {} --code-dir {} --target {} --input {} --verbose".format(
ArgumentsOptions.MAIN_COMMAND,
BINARY,
custom_model_dir,
resources.targets(BINARY_TEXT),
input_dataset,
)
with pytest.raises(AssertionError):
_, _, stderr = _exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
assert "DrumSchemaValidationException" in stderr
def test_fit_simple(
self,
resources,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
SIMPLE,
REGRESSION,
PYTHON,
is_training=True,
nested=True,
)
input_dataset = resources.datasets(SKLEARN, REGRESSION)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --target-type {} --code-dir {} --target {} --input {} --verbose".format(
ArgumentsOptions.MAIN_COMMAND,
REGRESSION,
custom_model_dir,
resources.targets(REGRESSION),
input_dataset,
)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_sparse_transform_fit(
self,
framework,
resources,
tmp_path,
):
input_dataset = resources.datasets(None, SPARSE)
target_dataset = resources.datasets(None, SPARSE_TARGET)
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
REGRESSION,
language=framework,
)
columns = resources.datasets(framework, SPARSE_COLUMNS)
cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose --target-csv {} --sparse-column-file {}".format(
ArgumentsOptions.MAIN_COMMAND,
TRANSFORM,
custom_model_dir,
input_dataset,
target_dataset,
columns,
)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_fit_sh(
self, resources, framework, problem, weights, tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources, tmp_path, framework, problem, PYTHON, is_training=True,
)
env = os.environ
fit_sh = os.path.join(
TESTS_ROOT_PATH,
"..",
"public_dropin_environments/{}_{}/fit.sh".format(
PYTHON,
framework
if framework
not in [SKLEARN_ANOMALY, SKLEARN_BINARY, SKLEARN_MULTICLASS, SKLEARN_SPARSE]
else SKLEARN,
),
)
input_dir = tmp_path / "input_dir"
self._create_fit_input_data_dir(
resources.targets,
resources.datasets,
input_dir,
problem,
weights,
is_sparse=framework == SKLEARN_SPARSE,
)
output = tmp_path / "output"
output.mkdir()
unset_drum_supported_env_vars()
env["CODEPATH"] = str(custom_model_dir)
env["INPUT_DIRECTORY"] = str(input_dir)
env["ARTIFACT_DIRECTORY"] = str(output)
env["TARGET_TYPE"] = problem if problem != BINARY_TEXT else BINARY
if framework == SKLEARN_SPARSE:
env["TRAINING_DATA_EXTENSION"] = ".mtx"
else:
env["TRAINING_DATA_EXTENSION"] = ".csv"
if problem in [BINARY, BINARY_TEXT]:
labels = resources.class_labels(framework, problem)
env["NEGATIVE_CLASS_LABEL"] = labels[0]
env["POSITIVE_CLASS_LABEL"] = labels[1]
elif problem == MULTICLASS:
labels = resources.class_labels(framework, problem)
with open(os.path.join(tmp_path, "class_labels.txt"), mode="w") as f:
f.write("\n".join(labels))
env["CLASS_LABELS_FILE"] = f.name
_exec_shell_cmd(fit_sh, "Failed cmd {}".format(fit_sh), env=env)
# clear env vars as it may affect next test cases
unset_drum_supported_env_vars()
def test_fit_hyperparameters(
self,
resources,
framework,
problem,
docker,
parameters,
weights,
tmp_path,
):
if framework == RDS_HYPERPARAMETERS:
language = R_FIT
else:
language = PYTHON
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
is_training=True,
)
input_dataset = resources.datasets(framework, problem)
parameter_file = resources.datasets(framework, parameters)
weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd(
weights, input_dataset, r_fit=language == R_FIT)
target_type = resources.target_types(
problem) if "transform" not in framework else TRANSFORM
cmd = "{} fit --target-type {} --code-dir {} --input {} --parameter-file {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND,
target_type,
custom_model_dir,
input_dataset,
parameter_file,
)
if problem != ANOMALY:
cmd += " --target {}".format(resources.targets(problem))
if problem in [BINARY, MULTICLASS]:
cmd = _cmd_add_class_labels(cmd,
resources.class_labels(
framework, problem),
target_type=target_type)
if docker:
cmd += " --docker {} ".format(docker)
cmd += weights_cmd
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_fit_for_use_output_and_nested(
self,
resources,
framework,
problem,
docker,
weights,
use_output,
tmp_path,
nested,
):
if docker and framework != SKLEARN:
return
if framework == RDS:
language = R_FIT
else:
language = PYTHON
custom_model_dir = _create_custom_model_dir(resources,
tmp_path,
framework,
problem,
language,
is_training=True,
nested=nested)
input_dataset = resources.datasets(framework, problem)
weights_cmd, input_dataset, __keep_this_around = self._add_weights_cmd(
weights, input_dataset, r_fit=language == R_FIT)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND, problem, custom_model_dir,
input_dataset)
if problem != ANOMALY:
cmd += " --target {}".format(resources.targets(problem))
if use_output:
cmd += " --output {}".format(output)
if problem == BINARY:
cmd = _cmd_add_class_labels(cmd,
resources.class_labels(
framework, problem),
target_type=problem)
if docker:
cmd += " --docker {} ".format(docker)
cmd += weights_cmd
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
def test_custom_transform_server(
self,
resources,
framework,
problem,
language,
docker,
tmp_path,
use_arrow,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
resources.target_types(problem),
resources.class_labels(framework, problem),
custom_model_dir,
docker,
) as run:
input_dataset = resources.datasets(framework, problem)
# do predictions
files = {"X": open(input_dataset)}
if use_arrow:
files["arrow_version"] = ".2"
response = requests.post(run.url_server_address + "/transform/",
files=files)
print(response.text)
assert response.ok
in_data = pd.read_csv(input_dataset)
if framework == SKLEARN_TRANSFORM_DENSE:
if use_arrow:
transformed_out = read_arrow_payload(eval(response.text))
assert eval(response.text)["out.format"] == "arrow"
else:
transformed_out = read_csv_payload(eval(response.text))
assert eval(response.text)["out.format"] == "csv"
actual_num_predictions = transformed_out.shape[0]
else:
transformed_out = read_mtx_payload(eval(response.text))
actual_num_predictions = transformed_out.shape[0]
assert eval(response.text)["out.format"] == "sparse"
validate_transformed_output(
transformed_out,
should_be_sparse=framework == SKLEARN_TRANSFORM)
assert in_data.shape[0] == actual_num_predictions
def test_fit_variety(framework, variety_resources, resources,
variety_data_names, tmp_path):
# get data info from fixtures
df = variety_data_names
df_path = variety_resources.dataset(df)
problem = variety_resources.problem(df)
target = variety_resources.target(df)
if problem == BINARY:
class_labels = variety_resources.class_labels(df)
if framework == RDS:
# there's one annoying dataset where R needs 0 and 1 and python wants 1.0 and 0.0
class_labels = [
int(x) if type(x) is float else x for x in class_labels
]
# figure out language
if framework == RDS:
language = R_FIT
else:
language = PYTHON
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
is_training=True,
nested=False,
)
output = tmp_path / "output"
output.mkdir()
cmd = "{} fit --code-dir {} --input {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND, custom_model_dir, df_path)
if problem == ANOMALY:
cmd += " --unsupervised"
else:
cmd += " --target {}".format(target)
if problem == BINARY:
cmd = _cmd_add_class_labels(cmd, class_labels, target_type=problem)
p, _, err = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(ArgumentsOptions.MAIN_COMMAND,
cmd),
assert_if_fail=False,
)
if p.returncode != 0:
raise AssertionError(err)
def test_predictions_r_mtx(
self,
resources,
framework,
problem,
language,
nginx,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
resources.target_types(problem),
resources.class_labels(framework, problem),
custom_model_dir,
nginx=nginx,
) as run:
input_dataset = resources.datasets(framework, SPARSE)
# do predictions
for endpoint in ["/predict/", "/predictions/"]:
for post_args in [
{
"files": {
"X": ("X.mtx", open(input_dataset))
}
},
{
"data": open(input_dataset),
"headers": {
"Content-Type":
"{};".format(PredictionServerMimetypes.TEXT_MTX)
},
},
]:
response = requests.post(run.url_server_address + endpoint,
**post_args)
assert response.ok
actual_num_predictions = len(
json.loads(response.text)[RESPONSE_PREDICTIONS_KEY])
in_data = StructuredInputReadUtils.read_structured_input_file_as_df(
input_dataset)
assert in_data.shape[0] == actual_num_predictions
def test_custom_models_perf_test(
self,
resources,
framework,
problem,
language,
docker,
timeout,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
cmd = "{} perf-test -i 200 -s 1000 --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
if timeout is not None:
cmd += " --timeout {}".format(timeout)
if resources.target_types(problem) in [BINARY, MULTICLASS]:
cmd = _cmd_add_class_labels(
cmd,
resources.class_labels(framework, problem),
target_type=resources.target_types(problem),
)
if docker:
cmd += " --docker {}".format(docker)
_, stdo, _ = _exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
if timeout is not None:
expected_str = "timed out ({}s)".format(timeout)
assert expected_str in stdo
assert "NA" in stdo
else:
assert "NA" not in stdo
def test_validation_check_with_bad_column_names(
self,
resources,
framework,
problem,
language,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
column_names = [
"column",
"col/unm",
"col\\unm",
'col"umn',
"col umn",
"col:umn",
'col""umn',
]
d = {col: [1.0] for col in column_names}
df = pd.DataFrame(data=d)
with NamedTemporaryFile(mode="w") as temp_f:
df.to_csv(temp_f.name)
input_dataset = temp_f.name
cmd = "{} validation --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
_, stdo, _ = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd),
assert_if_fail=False,
)
assert re.search(r"Null value imputation\s+PASSED", stdo)
def test_unstructured_models_batch(
self,
resources,
framework,
problem,
language,
docker,
mimetype,
ret_mode,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
output = tmp_path / "output"
content_type = "--content-type '{};'".format(
mimetype) if mimetype is not None else ""
cmd = "{} score --code-dir {} --input {} --output {} --target-type unstructured {} --query 'ret_mode={}'".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
output,
content_type,
ret_mode,
)
if docker:
cmd += " --docker {} --verbose ".format(docker)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
if ret_mode == "binary":
with open(output, "rb") as f:
out_data = f.read()
assert 10 == int.from_bytes(out_data, byteorder="big")
else:
with open(output) as f:
out_data = f.read()
assert "10" in out_data
def test_set_language(
self,
resources,
framework,
problem,
language,
set_language,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
cmd = "{} score --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
if set_language:
cmd += " --language {}".format(set_language)
if problem == BINARY:
cmd += " --positive-class-label yes --negative-class-label no"
p, stdo, stde = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd),
assert_if_fail=False,
)
if not set_language:
stdo_stde = str(stdo) + str(stde)
cases_4_5_6_7 = (str(stdo_stde).find(
"Can not detect language by artifacts and/or custom.py/R files"
) != -1)
assert cases_4_5_6_7
if framework == CODEGEN_AND_SKLEARN and set_language == "r":
stdo_stde = str(stdo) + str(stde)
case = (str(stdo_stde).find(
"Could not find a serialized model artifact with .rds extension, supported by default R predictor. "
"If your artifact is not supported by default predictor, implement custom.load_model hook."
) != -1)
assert case
def test_custom_models_with_drum_nginx_prediction_server(
self,
resources,
framework,
problem,
language,
docker,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
resources.target_types(problem),
resources.class_labels(framework, problem),
custom_model_dir,
docker,
nginx=True,
) as run:
input_dataset = resources.datasets(framework, problem)
# do predictions
for endpoint in ["/predict/", "/predictions/"]:
for post_args in [
{
"files": {
"X": open(input_dataset)
}
},
{
"data": open(input_dataset, "rb")
},
]:
response = requests.post(run.url_server_address + endpoint,
**post_args)
assert response.ok
actual_num_predictions = len(
json.loads(response.text)[RESPONSE_PREDICTIONS_KEY])
in_data = pd.read_csv(input_dataset)
assert in_data.shape[0] == actual_num_predictions
def test_custom_models_with_drum(
self,
resources,
framework,
problem,
language,
docker,
tmp_path,
use_labels_file,
temp_file,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
output = tmp_path / "output"
cmd = '{} score --code-dir {} --input "{}" --output {} --target-type {}'.format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
output,
resources.target_types(problem),
)
if resources.target_types(problem) in [BINARY, MULTICLASS]:
cmd = _cmd_add_class_labels(
cmd,
resources.class_labels(framework, problem),
target_type=resources.target_types(problem),
multiclass_label_file=temp_file if use_labels_file else None,
)
if docker:
cmd += " --docker {} --verbose ".format(docker)
_exec_shell_cmd(
cmd, "Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd))
in_data = pd.read_csv(input_dataset)
out_data = pd.read_csv(output)
assert in_data.shape[0] == out_data.shape[0]
def test_prediction_consistency(self, resources, tmp_path, framework,
problem):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
SPARSE,
language=PYTHON,
is_training=True,
)
input_dataset = resources.datasets(framework, problem)
if problem in [BINARY_TEXT, BINARY_BOOL]:
target_type = BINARY
else:
target_type = problem
cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format(
ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir,
input_dataset)
cmd += " --target {}".format(resources.targets(problem))
if target_type in [BINARY, MULTICLASS]:
cmd = _cmd_add_class_labels(
cmd, resources.class_labels(framework, problem), target_type)
_, stdout, stderr = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd),
assert_if_fail=True,
)
# we should throw a warning, not an error
assert "Your predictions were different when we tried to predict twice." in stderr
# but don't error out
assert (
"Your model can be fit to your data, and predictions can be made on the fit model!"
in stdout)
# clean up
sample_dir = stderr.split(":")[-1]
if sample_dir.endswith("\n"):
sample_dir = sample_dir[:-1]
os.remove(sample_dir.strip())
def test_custom_models_validation_test(
self,
resources,
framework,
problem,
language,
docker,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
cmd = "{} validation --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
if problem == BINARY:
cmd = _cmd_add_class_labels(
cmd,
resources.class_labels(framework, problem),
target_type=resources.target_types(problem),
)
if docker:
cmd += " --docker {}".format(docker)
_, stdo, _ = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd),
assert_if_fail=False,
)
if language == NO_CUSTOM:
assert re.search(r"Null value imputation\s+FAILED", stdo)
else:
assert re.search(r"Null value imputation\s+PASSED", stdo)
def test_bin_models_with_wrong_labels(
self,
resources,
framework,
problem,
language,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
input_dataset = resources.datasets(framework, problem)
cmd = "{} score --code-dir {} --input {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
resources.target_types(problem),
)
if problem == BINARY:
cmd = cmd + " --positive-class-label yes --negative-class-label no"
p, stdo, stde = _exec_shell_cmd(
cmd,
"Failed in {} command line! {}".format(
ArgumentsOptions.MAIN_COMMAND, cmd),
assert_if_fail=False,
)
stdo_stde = str(stdo) + str(stde)
if framework == SKLEARN:
assert (str(stdo_stde).find(
"Wrong class labels ['no', 'yes']. Use class labels detected by sklearn model"
) != -1)
elif framework == RDS:
assert (str(stdo_stde).find(
"Wrong class labels. Use class labels according to your dataset"
) != -1)
def test_custom_models_with_drum_prediction_server(
self,
resources,
framework,
problem,
language,
nginx,
docker,
tmp_path,
):
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
with DrumServerRun(
"unstructured",
resources.class_labels(framework, problem),
custom_model_dir,
docker,
nginx=nginx,
) as run:
input_dataset = resources.datasets(framework, problem)
for ret_mode in ["text", "binary"]:
for endpoint in [
"/predictUnstructured/", "/predictionsUnstructured/"
]:
# do predictions
url = run.url_server_address + endpoint
data = open(input_dataset, "rb").read()
params = {"ret_mode": ret_mode}
response = requests.post(url=url, data=data, params=params)
assert response.ok
if ret_mode == "text":
assert response.text == "10"
else:
assert 10 == int.from_bytes(response.content,
byteorder="big")
def _drum_with_monitoring(resources, framework, problem, language, docker,
tmp_path):
"""
We expect the run of drum to be ok, since mlops is assumed to be installed.
"""
custom_model_dir = _create_custom_model_dir(
resources,
tmp_path,
framework,
problem,
language,
)
mlops_spool_dir = tmp_path / "mlops_spool"
os.mkdir(str(mlops_spool_dir))
input_dataset = resources.datasets(framework, problem)
output = tmp_path / "output"
cmd = "{} score --code-dir {} --input {} --output {} --target-type {}".format(
ArgumentsOptions.MAIN_COMMAND,
custom_model_dir,
input_dataset,
output,
resources.target_types(problem),
)
monitor_settings = (
"spooler_type=filesystem;directory={};max_files=1;file_max_size=1024000"
.format(mlops_spool_dir))
cmd += ' --monitor --model-id 555 --deployment-id 777 --monitor-settings="{}"'.format(
monitor_settings)
if problem == BINARY:
cmd = _cmd_add_class_labels(
cmd,
resources.class_labels(framework, problem),
target_type=resources.target_types(problem),
)
if docker:
cmd += " --docker {} --verbose ".format(docker)
return cmd, input_dataset, output, mlops_spool_dir
