def __init__(self,
host="localhost",
port=8080,
sname="",
description="",
mllib="caffe",
service_parameters_input=None,
service_parameters_mllib=None,
service_parameters_output=None,
model=None,
tmp_dir=None):
self.host = host
self.port = port
self.sname = sname
self.model = model
self.description = description
self.mllib = mllib
self.tmp_dir = tmp_dir
self.service_parameters_input = service_parameters_input
self.service_parameters_mllib = service_parameters_mllib
self.service_parameters_output = service_parameters_output
self.n_pred = 0
self.n_fit = 0
self.calls = []
self.answers = []
# self.train_logs = None
super(AbstractModels, self).__init__(self.host, self.port)
if self.sname:
for service in self.get_info()['head']['services']:
if service['name'] == self.sname.lower(
): # DD lowercases services' name
self.delete_service(self.sname, clear="mem")
else:
self.sname = "pyDD_{}".format(time_utils.fulltimestamp())
self.description = self.sname
# Check if a repository is given otherwise creates one
if "repository" not in self.model or not self.model["repository"]:
self.repository = tempfile.mkdtemp(prefix="pydd_",
dir=self.tmp_dir)
self.model["repository"] = self.repository
os_utils._create_dirs([self.model["repository"]])
else:
assert os.path.exists(
self.model["repository"]), "{} does not exist".format(
self.model["repository"])
json_dump = self.create_service(self.sname, self.model,
self.description, self.mllib,
self.service_parameters_input,
self.service_parameters_mllib,
self.service_parameters_output)
self.answers.append(json_dump)
with open("{}/model.json".format(self.model["repository"])) as f:
self.calls = [json.loads(line, encoding="utf-8") for line in f]
def test_predict_from_model(self):
snames = ['svm_predict_from_model', 'array_predict_from_model']
model_repo = [
os.path.abspath('model_svm'),
os.path.abspath('model_array')
]
params = {'nclasses': n_classes, 'gpu': True}
params.update(connec_param)
train_path = os.path.abspath('x_train.svm')
test_path = os.path.abspath('x_test.svm')
datasets.dump_svmlight_file(x_train, y_train, train_path)
datasets.dump_svmlight_file(x_test, y_test, test_path)
# We make sure model repo does not exist
for folder in model_repo:
if os.path.exists(folder):
os_utils._remove_dirs([folder])
os_utils._create_dirs(model_repo)
# Create model, make sure the sname is not used by the server
clf_svm = MLPfromSVM(sname=snames[0],
repository=model_repo[0],
**params)
clf_array = MLPfromArray(sname=snames[1],
repository=model_repo[1],
**params)
clf_svm.fit([train_path, test_path], iterations=300)
clf_array.fit(x_train,
y_train,
validation_data=[(x_test, y_test)],
iterations=300)
y_pred_svm = clf_svm.predict(test_path)
y_pred_array = clf_array.predict(x_test)
# Load from existing model
params = {'nclasses': n_classes, 'finetuning': True, 'template': None}
params.update(connec_param)
clf_svm = MLPfromSVM(sname=snames[0],
repository=model_repo[0],
**params)
clf_array = MLPfromArray(sname=snames[1],
repository=model_repo[1],
**params)
assert np.array_equal(y_pred_svm, clf_svm.predict(test_path))
assert np.array_equal(y_pred_array, clf_array.predict(x_test))
os_utils._remove_files([train_path, test_path])
os_utils._remove_dirs(model_repo)
from sklearn import datasets, preprocessing, model_selection, metrics
# Parameters
test_size = 0.2
seed = 1337
host = 'localhost'
port = 8080
nclasses = 10
sname = 'predict_from_model'
model_repo = os.path.abspath('trained_model')
# We make sure model repo does not exist
if os.path.exists(model_repo):
os_utils._remove_dirs([model_repo])
os_utils._create_dirs([model_repo])
# create dataset
X, Y = datasets.load_digits(return_X_y=True)
X = preprocessing.StandardScaler().fit_transform(X)
xtr, xte, ytr, yte = model_selection.train_test_split(X,
Y,
test_size=test_size,
random_state=seed)
# create and save train.svm and test.svm
tr_f = os.path.abspath('x_train.svm')
te_f = os.path.abspath('x_test.svm')
datasets.dump_svmlight_file(xtr, ytr, tr_f)
datasets.dump_svmlight_file(xte, yte, te_f)
def __init__(self,
host='localhost',
port=8080,
sname='',
mllib='caffe',
description='',
repository='',
templates='../templates/caffe',
connector='svm',
nclasses=None,
ntargets=None,
gpu=False,
gpuid=0,
template='mlp',
layers=[50],
activation='relu',
dropout=0.5,
regression=False,
finetuning=False,
db=True):
self.host = host
self.port = port
self.sname = sname
self.mllib = mllib
self.description = description
self.repository = repository
self.templates = templates
self.connector = connector
self.nclasses = nclasses
self.ntargets = ntargets
self.gpu = gpu
self.gpuid = gpuid
self.template = template
self.layers = layers
self.activation = activation
self.dropout = dropout
self.regression = regression
self.finetuning = finetuning
self.db = db
self.params = {
'host': self.host,
'port': self.port,
'sname': self.sname,
'mllib': self.mllib,
'description': self.description,
'repository': self.repository,
'templates': self.templates,
'connector': self.connector,
'nclasses': self.nclasses,
'ntargets': self.ntargets,
'gpu': self.gpu,
'gpuid': self.gpuid,
'template': self.template,
'layers': self.layers,
'activation': self.activation,
'dropout': self.dropout,
'regression': self.regression,
'finetuning': self.finetuning,
'db': self.db,
}
self.n_pred = 0
self.n_fit = 0
self.calls = []
self.answers = []
self.model = {
'templates': self.templates,
'repository': self.repository
}
self.service_parameters_mllib = {
'nclasses': self.nclasses,
'ntargets': self.ntargets,
'gpu': self.gpu,
'gpuid': self.gpuid,
'template': self.template,
'layers': self.layers,
'activation': self.activation,
'dropout': self.dropout,
'regression': self.regression,
'finetuning': self.finetuning,
'db': self.db
}
self.service_parameters_input = {'connector': self.connector}
self.service_parameters_output = {}
super(genericMLP, self).__init__(self.host, self.port)
if self.sname == '':
self.sname = "pyDD_MLP_{}".format(time_utils.fulltimestamp())
self.description = self.sname
else:
self.delete_service(self.sname, "mem")
tmp_dir = tempfile.mkdtemp()
self.data_folder = "{}/data".format(tmp_dir)
if self.model['repository'] == '':
self.model['repository'] = "{}/model".format(tmp_dir)
os_utils._create_dirs([self.model['repository'], self.data_folder])
json_dump = self.create_service(self.sname, self.model,
self.description, self.mllib,
self.service_parameters_input,
self.service_parameters_mllib,
self.service_parameters_output)
self.answers.append(json_dump)
with open("{}/model.json".format(self.model['repository'])) as f:
self.calls = [json.loads(line, encoding='utf-8') for line in f]
from pydd.connectors import SVMConnector
from pydd.utils import os_utils, lmdb_utils
from sklearn import datasets, metrics, model_selection, preprocessing
# Parameters
seed = 1337
np.random.seed(seed) # for reproducibility
n_classes = 10
params = {"port": 8080, "nclasses": n_classes, "gpu": True}
split_params = {"test_size": 0.2, "random_state": seed}
folder = "train-from-lmdb"
if os.path.exists(folder):
os_utils._remove_dirs([folder])
os_utils._create_dirs([folder])
# create dataset
X, y = datasets.load_digits(n_class=n_classes, return_X_y=True)
X = preprocessing.StandardScaler().fit_transform(X)
xtr, xte, ytr, yte = model_selection.train_test_split(X, y, **split_params)
# create and save train.svm and test.svm
tr_f = os.path.abspath("{}/x_train.svm".format(folder))
te_f = os.path.abspath("{}/x_test.svm".format(folder))
datasets.dump_svmlight_file(xtr, ytr, tr_f)
datasets.dump_svmlight_file(xte, yte, te_f)
# create lmdb dataset
tr_lmdb = os.path.abspath("{}/train.lmdb".format(folder))
params = {
'host': 'localhost',
'port': 8085,
'nclasses': 10,
'layers': [100, 100],
'activation': 'relu',
'dropout': 0.2,
'db': True,
'gpu': True
}
# We make sure model repo does not exist
for folder in model_repo:
if os.path.exists(folder):
os_utils._remove_dirs([folder])
os_utils._create_dirs(model_repo)
# Create dataset
X, Y = datasets.load_digits(return_X_y=True)
X = preprocessing.StandardScaler().fit_transform(X)
x_train, x_test, y_train, y_test = model_selection.train_test_split(
X, Y, test_size=0.2, random_state=1337)
# Save data in .svm format
train_path = os.path.abspath('x_train.svm')
test_path = os.path.abspath('x_test.svm')
datasets.dump_svmlight_file(x_train, y_train, train_path)
datasets.dump_svmlight_file(x_test, y_test, test_path)
# Create model, make sure the sname is not used by the server
clf_svm = MLPfromSVM(sname=snames[0], repository=model_repo[0], **params)