def test_model_serde():
model = skil.Model('keras_mnist.h5', name='bar')
model.save('model.json')
recov = skil.Model.load('model.json')
assert recov.get_config() == model.get_config()
def test_serving():
ws = _get_ws()
exp = skil.Experiment(ws)
save_model()
model = skil.Model('model.h5', experiment=exp)
dep = skil.Deployment(ws.skil)
service = model.deploy(dep)
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_test = x_test.reshape(10000, 784)
x_test = x_test.astype('float32')
x_test /= 255
service.predict_single(x_test[0])
service.predict(x_test[:10])
service.stop()
time.sleep(4)
with pytest.raises(Exception):
service.predict(x_test[:10])
service.start()
time.sleep(4)
service.predict(x_test[:10])
service.delete()
os.remove('model.h5')
def test_model_creation_2():
sk = skil.Skil()
work_space = skil.WorkSpace(sk)
exp = skil.Experiment(work_space)
model = skil.Model('keras_mnist.h5', experiment=exp)
work_space.delete()
exp.delete()
model.delete()
def test_service_serde():
dep = skil.Deployment()
model = skil.Model('keras_mnist.h5', name='bar')
service = model.deploy(dep)
service.save('service.json')
recov = skil.Service.load('service.json')
assert recov.get_config() == service.get_config()
def test_model_by_id():
ws = _get_ws()
exp = skil.Experiment(ws, name='test_exp2')
with open('model.h5', 'w') as f:
f.write('')
model = skil.Model('model.h5', name='test_model', experiment=exp)
model_id = model.id
model2 = skil.get_model_by_id(exp, model_id)
assert model.name == model2.name
os.remove('model.h5')
def test_service_creation():
sk = skil.Skil()
work_space = skil.WorkSpace(sk)
exp = skil.Experiment(work_space)
model = skil.Model('keras_mnist.h5', experiment=exp)
model.add_evaluation(0.95)
dep = skil.Deployment(sk)
model.deploy(dep)
work_space.delete()
exp.delete()
model.delete()
dep.delete()
def test_base_inference_job():
skil_server = skil.Skil()
model = skil.Model('keras_mnist.h5')
res = skil.resources.compute.EMR(skil_server, 'name244', 'region', 'creds',
'id')
conf = InferenceJobConfiguration(model, 32, res, res, './', "DSP")
# job = InferenceJob(skil_server, conf)
# job.inference_config()
# with pytest.raises(Exception):
# job.run()
res.delete()
def load(cls, file_name):
config = deserialize_config(file_name)
skil_server = skil.Skil.from_config()
experiment = skil.experiments.get_experiment_by_id(
skil_server, config['experiment_id'])
model = skil.Model(model_id=config['model_id'],
experiment=experiment,
create=False)
model.name = config['model_name']
deployment = skil.get_deployment_by_id(skil, config['deployment_id'])
model_entity = skil_client.ModelEntity(id=config['model_entity_id'])
return Service(skil=skil_server,
model=model,
deployment=deployment,
model_entity=model_entity)
def test_base_training_job():
skil_server = skil.Skil()
model = skil.Model('keras_mnist.h5')
res = skil.resources.compute.EMR(skil_server, 'name234', 'region', 'creds',
'id')
conf = TrainingJobConfiguration(model, 10, "acc", "EvalDSP", res, res,
'./', "DSP")
distributed_config = ParameterAveraging(8, 32)
# TODO "jobArgs" does not get recognize"
# job = TrainingJob(skil_server, conf, distributed_config)
# job._training_job_args()
# with pytest.raises(Exception):
# job.run()
res.delete()
def test_model_creation_2():
ws = _get_ws()
exp = skil.Experiment(ws)
model = skil.Model('keras_mnist.h5', experiment=exp)
import skil
import numpy as np
skil_server = skil.Skil()
work_space = skil.WorkSpace(skil_server)
experiment = skil.Experiment(work_space)
transform = skil.Transform(transform='iris_tp.json', experiment=experiment)
model = skil.Model(model='iris_model.h5', experiment=experiment)
deployment = skil.Deployment(skil_server)
pipeline = skil.Pipeline(deployment, model, transform)
with open('iris.data', 'r') as f:
data = np.array(f.readlines())
print(pipeline.predict(data))
import skil
import cv2
skil_server = skil.Skil()
model = skil.Model('yolo_v2.pb', name='yolo-tf', model_id='yolo-3493723')
deployment = skil.Deployment(skil_server, 'yolo')
service = model.deploy(deployment, input_names=['input'], output_names=['output'], scale=2)
cap = cv2.VideoCapture(0)
while True:
_, image = cap.read()
detection = service.detect_objects(image)
image = skil.utils.yolo.annotate_image(image, detection)
cv2.imshow('yolo', image)
def test_model_deletion():
model = skil.Model('keras_mnist.h5')
model.delete()
import skil
from keras.models import model_from_config
import json
# Load Keras model you want to train
with open('keras_config.json', 'r') as f:
model = model_from_config(json.load(f))
model.compile(loss='categorical_crossentropy', optimizer='sgd')
# Create a SKIL model from it
skil_server = skil.Skil()
ws = skil.WorkSpace(skil_server)
experiment = skil.Experiment(ws)
model = skil.Model(model, model_id='keras_mnist_mlp_42',
name='keras', experiment=experiment)
# Register compute and storage resources.
s3 = skil.resources.storage.S3(
skil_server, 's3_resource', 'bucket_name', 'region')
emr = skil.resources.compute.EMR(
skil_server, 'emr_cluster', 'region', 'credential_uri', 'cluster_id')
# Define your general training setup
training_config = skil.jobs.TrainingJobConfiguration(
skil_model=model, num_epochs=10, eval_type='ROC_MULTI_CLASS',
storage_resource=s3, compute_resource=emr,
data_set_provider_class='MnistProvider',
eval_data_set_provider_class='MnistProvider',
output_path='.')
# Optionally specify a distributed training config.
def test_service_creation():
ws = _get_ws()
exp = skil.Experiment(ws)
model = skil.Model('keras_mnist.h5', experiment=exp)
dep = skil.Deployment(ws.skil)
model.deploy(dep)
def test_model_creation_1():
model = skil.Model('keras_mnist.h5')
model.delete()
import skil
import cv2
model = skil.Model('yolo.pb', model_id='yolo_42', name='yolo')
service = model.deploy(skil.Deployment(),
input_names=['input'],
output_names=['output'])
image = cv2.imread("say_yolo_again.jpg")
detection = service.detect_objects(image)
image = skil.utils.yolo.annotate_image(image, detection)
cv2.imwrite('annotated.jpg', image)
import skil
import cv2
model = skil.Model('yolo_v2.pb', model_id='yolo_1', name='yolo')
service = model.deploy(skil.Deployment(),
input_names=['input'],
output_names=['output'])
image = cv2.imread("say_yolo_again.jpg")
detection = service.detect_objects(image)
image = skil.utils.yolo.annotate_image(image, detection)
cv2.imwrite('annotated.jpg', image)
