def test_e2e(Skil):
model_path = './dummy.pb'
skil_server = Skil()
skil_server.upload_model(model_path)
ws = WorkSpace(skil_server, 'jupyter_ws')
experiment = Experiment(ws, 'test_exp')
model = Model(model_path, experiment)
model.add_evaluation(accuracy=0.93)
deployment = Deployment(skil_server, 'test_deployment')
model.deploy(deployment, start_server=False)
from skil import Skil, WorkSpace, Experiment, Model, Deployment
from keras.datasets import mnist
skil_server = Skil()
work_space = WorkSpace(skil_server)
experiment = Experiment(work_space)
model = Model('model.pb', model_id='tf_model', experiment=experiment)
deployment = Deployment(skil_server)
service = model.deploy(deployment,
input_names=['input'],
output_names=['output'])
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_test = x_test.reshape(10000, 784)
x_test = x_test.astype('float32')
x_test /= 255
print(service.predict_single(x_test[0]))
print(service.predict(x_test[:10]))
def test_skil_default_model(Skil):
skil_server = Skil()
deployment = Deployment(skil_server)
from skil import Skil, WorkSpace, Experiment, Model, Deployment
import json
# Initialize a new experiment in a workspace
skil_server = Skil()
ws = WorkSpace(skil_server)
experiment = Experiment(ws)
deployment = Deployment(skil_server, "keras_models")
skil_models = []
for epoch in range(2):
model_name = 'model_{epoch:02d}.hdf5'.format(epoch=epoch + 1)
model = Model(model_name, id=epoch, experiment=experiment)
model.deploy(start_server=False, deployment=deployment)
skil_models.append(model)
with open('history.json', 'r') as f:
history = json.loads(f.read())
acc = history.get('val_acc')
best_model = skil_models[acc.index(max(acc))]
best_model.serve()
from skil import Skil, WorkSpace, Experiment, Model, Deployment
import json
# Initialize a new experiment in a workspace
skil_server = Skil()
work_space = WorkSpace(skil_server, name='mnist-workspace')
experiment = Experiment(work_space, name='mnist-experiment')
deployment = Deployment(skil_server, name='mnist-deployment')
# Deploy all models with SKIL
skil_services = []
for epoch in range(2):
model_name = 'model_{epoch:02d}.hdf5'.format(epoch=epoch + 1)
model = Model(model_name, model_id=epoch, experiment=experiment)
service = model.deploy(start_server=False, deployment=deployment)
skil_services.append(service)
# Serve the best one
with open('history.json', 'r') as f:
history = json.loads(f.read())
acc = history.get('val_acc')
best_service = skil_services[acc.index(max(acc))]
best_service.start()
work_space = skil.get_workspace_by_id(None, skil_server, skil_conf['workspace_id'])
experiment = skil.get_experiment_by_id(work_space, skil_conf['experiment_id'])
model = None
if sys.argv[1] == '--id':
model = skil.get_model_by_id(None, experiment, sys.argv[2]) # TODO: Doesn't work in 0.2.5.
model.name = 'mnist'
elif sys.argv[1] == '--file':
model = SkilModel(sys.argv[2], name='mnist', id=str(uuid.uuid1()), experiment=experiment)
else:
print('Usage: python eval_and_test.py --id [model_id] | --file [path]')
sys.exit(1)
deployment = None
if skil_conf['deployment_id'] == '':
deployment = Deployment(skil_server, "mnist_models")
skil_conf['deployment_id'] = deployment.id
with open('.skil', 'w') as f:
json.dump(skil_conf, f)
else:
deployment = skil.get_deployement_by_id(skil_server, skil_conf['deployment_id'])
service = None
try:
service = model.deploy(deployment=deployment)
except Exception:
print('Service with name already exists reusing...')
service = Service(skil_server, "mnist", deployment, {})
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_test = x_test.reshape(10000, 784)
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
batch_size=128,
epochs=10,
verbose=1,
validation_data=(x_test, y_test))
_, acc = model.evaluate(x_test, y_test)
# Load the experiment from file (alternatively create one with Experiment())
# Make sure to run: "pyskil init-experiment -f exp.json" before
experiment = Experiment.load('exp.json')
# Save the model to SKIL
skil_model = SkilModel(model, experiment=experiment)
# Add accuracy as evaluation metric to SKIL
skil_model.add_evaluation(accuracy=acc, name="MNIST test set")
# Load the deployment from file (or create one with Deployment()) and deploy model as service
# Make sure to run: "pyskil init-deployment -f dep.json" before
deployment = Deployment.load('dep.json')
service = skil_model.deploy(deployment=deployment)
# Evaluate from scratch, using SKIL
num_correct = 0
for i in range(10000):
result = service.predict_single(x_test[i])
num_correct += np.argmax(result) == np.argmax(y_test[i])
print("Test Accuracy: %s" % (float(num_correct) / 10000, ))