def test_script_req():
'''Tests that Python scripts can be included using Requirements'''
def predict(x: int) -> int:
return x
model = Model(predict=predict)
model_name = 'my-model'
# tests that individual script and directory of scripts are both gathered
reqs = Requirements(scripts=_abspath('./user_module.py', './user_package'))
with _dump_model(model, model_name, reqs) as dump_dir:
_verify_files(dump_dir,
('scripts/user_provided/user_package_module.py',
'scripts/user_provided/__init__.py',
'scripts/user_provided/user_module.py'))
bad_reqs = Requirements(
scripts=_abspath('./user_module.py', './user_package', 'not_real.py'))
with pytest.raises(AcumosError, match='does not exist'):
with _dump_model(model, model_name, bad_reqs) as dump_dir:
pass
bad_reqs = Requirements(
scripts=_abspath('./user_module.py', './user_package', './att.png'))
with pytest.raises(AcumosError, match='is invalid'):
with _dump_model(model, model_name, bad_reqs) as dump_dir:
pass
def test_dump_model():
'''Tests dump model utility, including generated artifacts'''
def predict(x: int) -> int:
return user_function(x)
model = Model(predict=predict)
model_name = 'my-model'
reqs = Requirements(reqs=['wronglib'],
req_map={'wronglib': 'scipy'},
packages=[_USER_PACKAGE_DIR])
with _dump_model(model, model_name, reqs) as dump_dir:
assert set(listdir(dump_dir)) == set(_REQ_FILES)
metadata = load_artifact(dump_dir,
'metadata.json',
module=json,
mode='r')
schema = _load_schema(SCHEMA_VERSION)
validate(metadata, schema)
# test that a user-provided library was included and correctly mapped
assert 'scipy' in {
r['name']
for r in metadata['runtime']['dependencies']['pip']['requirements']
}
# test that custom package was bundled
_verify_files(
dump_dir,
('scripts/user_provided/user_package/user_package_module.py',
'scripts/user_provided/user_package/__init__.py',
'scripts/user_provided/user_module.py'))
def test_custom_package():
'''Tests that custom packages can be included, wrapped, and loaded'''
def my_transform(x: int, y: int) -> int:
return user_package_module.add_numbers(x, y)
model = Model(transform=my_transform)
model_name = 'my-model'
# load should fail without requirements
with pytest.raises(
Exception,
match='Module user_package was detected as a dependency'):
with _dump_model(model, model_name) as dump_dir:
pass
reqs = Requirements(packages=[_USER_PACKAGE_DIR])
with _dump_model(model, model_name, reqs) as dump_dir:
run_command(
[sys.executable, _MODEL_LOADER_HELPER, dump_dir, 'user_package'])
def test_custom_script():
'''Tests that custom modules can be included, wrapped, and loaded'''
def predict(x: int) -> int:
return user_function(x)
model = Model(predict=predict)
model_name = 'my-model'
with _dump_model(model, model_name) as dump_dir:
run_command(
[sys.executable, _MODEL_LOADER_HELPER, dump_dir, 'user_module'])
def test_raw_type(func, f_in, f_out, in_media_type, out_media_type, in_is_raw,
out_is_raw):
'''Tests to make sure that supported raw data type models are working correctly'''
model = Model(transform=func)
model_name = 'my-model'
with TemporaryDirectory() as tdir:
with _dump_model(model, model_name) as dump_dir:
_copy_dir(dump_dir, tdir, model_name)
copied_dump_dir = path_join(tdir, model_name)
metadata_file_path = path_join(copied_dump_dir, 'metadata.json')
with open(metadata_file_path) as metadata_file:
metadata_json = json.load(metadata_file)
assert metadata_json['methods']['transform']['input'][
'media_type'] == in_media_type
assert metadata_json['methods']['transform']['output'][
'media_type'] == out_media_type
wrapped_model = load_model(copied_dump_dir)
if in_is_raw:
wrapped_return = wrapped_model.transform.from_raw(f_in)
else:
arguments = model.transform.input_type(*f_in)
arguments_pb_msg = _pack_pb_msg(arguments,
wrapped_model.transform._module)
wrapped_return = wrapped_model.transform.from_pb_msg(
arguments_pb_msg)
if out_is_raw:
ret = wrapped_return.as_raw()
else:
ret_pb_msg = wrapped_return.as_pb_msg()
ret = _unpack_pb_msg(model.transform.output_type, ret_pb_msg).value
assert ret == f_out
def _generic_test(func,
in_,
out,
wrapped_eq=eq,
pb_mg_eq=eq,
pb_bytes_eq=eq,
dict_eq=eq,
json_eq=eq,
preload=None,
reqs=None,
skip=None):
'''Reusable wrap test routine with swappable equality functions'''
model = Model(transform=func)
model_name = 'my-model'
with TemporaryDirectory() as tdir:
with _dump_model(model, model_name, reqs) as dump_dir:
_copy_dir(dump_dir, tdir, model_name)
if preload is not None:
preload()
copied_dump_dir = path_join(tdir, model_name)
wrapped_model = load_model(copied_dump_dir)
TransIn = model.transform.input_type
TransOut = model.transform.output_type
trans_in = TransIn(*in_)
trans_out = TransOut(*out)
trans_in_pb = _pack_pb_msg(trans_in, wrapped_model.transform._module)
trans_out_pb = _pack_pb_msg(trans_out, wrapped_model.transform._module)
trans_in_pb_bytes = trans_in_pb.SerializeToString()
trans_out_pb_bytes = trans_out_pb.SerializeToString()
trans_in_dict = MessageToDict(trans_in_pb)
trans_out_dict = MessageToDict(trans_out_pb)
trans_in_json = MessageToJson(trans_in_pb, indent=0)
trans_out_json = MessageToJson(trans_out_pb, indent=0)
# test all from / as combinations
for as_method_name, as_data_expected, eq_func in (
('as_wrapped', trans_out, wrapped_eq), ('as_pb_msg', trans_out_pb,
pb_mg_eq),
('as_pb_bytes', trans_out_pb_bytes,
pb_bytes_eq), ('as_dict', trans_out_dict,
dict_eq), ('as_json', trans_out_json, json_eq)):
for from_method_name, from_data in (('from_wrapped', trans_in),
('from_pb_msg', trans_in_pb),
('from_pb_bytes',
trans_in_pb_bytes),
('from_dict', trans_in_dict),
('from_json', trans_in_json)):
if skip is not None and skip(as_method_name, from_method_name):
logger.info("Skipping {} -> {}".format(
from_method_name, as_method_name))
continue
from_method = getattr(wrapped_model.transform,
from_method_name)
resp = from_method(from_data)
as_data_method = getattr(resp, as_method_name)
as_data = as_data_method()
assert eq_func(as_data, as_data_expected)
def keras_evaluate(config):
taskComplete = False
useSklearn = True
listImages = []
if 'image_list' in config and config['image_list']:
dfImages = pd.read_csv(config['image_list'], header=None, names=['file'], delimiter=",")
listImages = dfImages['file'].tolist()
config['image'] = listImages[0]
X = create_sample(config['image'])
if useSklearn:
# formulate the pipelien to be used
from image_classifier.keras_model.prediction_formatter import Formatter
model, reqs = model_create_pipeline(config['model_path'], config['label_path'],
config['num_top_predictions'])
if 'push_address' in config and 'auth_address' in config and config['push_address']:
from acumos.session import AcumosSession
session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address'])
print("Pushing new model to upload '{:}', auth '{:}'...".format(config['push_address'], config['auth_address']))
session.push(model, MODEL_NAME, reqs) # creates ./my-iris.zip
taskComplete = True
if 'dump_model' in config and config['dump_model']:
from acumos.session import AcumosSession
from os import makedirs
if not os.path.exists(config['dump_model']):
makedirs(config['dump_model'])
print("Dumping new model to '{:}'...".format(config['dump_model']))
session = AcumosSession()
session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip
taskComplete = True
preds = None
if not taskComplete: # means we need to run a prediction/classify
import tempfile
from acumos.session import _dump_model, _copy_dir
from os.path import join as path_join
from acumos.wrapped import load_model
if not listImages:
listImages = [config['image']]
preds = None
# temporarily wrap model to a temp directory (to get 'wrapped' functionality)
with tempfile.TemporaryDirectory() as tdir: # create temp dir
with _dump_model(model, MODEL_NAME, reqs) as dump_dir: # dump model to temp dir
_copy_dir(dump_dir, tdir, MODEL_NAME) # relocate for load_model below
model_dir = path_join(tdir, MODEL_NAME)
wrapped_model = load_model(model_dir) # load to wrapped model
type_in = wrapped_model.classify._input_type
for idx in range(len(listImages)):
curImage = listImages[idx]
print("Attempting classification of image [{:}]: {:}...".format(idx, curImage))
X = create_sample(curImage)
classify_in = type_in(*tuple(col for col in X.values.T))
pred_raw = wrapped_model.classify.from_wrapped(classify_in).as_wrapped()
# already a wrapped response
predNew = pd.DataFrame(np.column_stack(pred_raw), columns=pred_raw._fields)
predNew[Formatter.COL_NAME_IDX] = idx
if preds is None:
preds = predNew
else:
preds = preds.append(predNew, ignore_index=True)
preds.reset_index(drop=True, inplace=True)
"""
Disable non-sklearn path for now
else:
from image_classifier.keras import inception_v4
from image_classifier.keras.image_decoder import ImageDecoder
# Load test image!
img = ImageDecoder.get_processed_image_keras_file(config['image']) # load image through keras
# img = evaluate_image.get_processed_image_cv(config['image'])
# Run prediction on test image
model, model_path = inception_v4.create_model(weights='imagenet', include_top=True, model_path=model_path)
preds = model.predict(img)
"""
return preds