def build(self):
print(
'***************************build model***************************'
)
self.model = ModelWrapper.model(conf,
train=True,
vocab_size=self.vocab_size,
labels_num=1)
adam_optimizer = tf.keras.optimizers.Adam(lr=1e-3,
decay=1e-6,
clipvalue=5)
self.model.compile(loss='binary_crossentropy',
optimizer=adam_optimizer,
metrics=['binary_crossentropy', 'accuracy'])
self.model.summary()
def build(self):
print(
'***************************build model***************************'
)
self.model = ModelWrapper.model(conf,
train=True,
vocab_size=self.vocab_size,
labels_num=self.labels_num)
# model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=["accuracy"]) #optimizer=Adam() keras.optimizers.Adam(lr=args.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
self.model.compile(loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(lr=0.05,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8),
metrics=["accuracy"])
self.model.summary()
def setUp(self):
self.flask_app = app.test_client()
# Adding users to authenticate against
self.username = "******"
self.password = "******"
app.config['USERS'][self.username] = self.password
app.config['MODEL'] = ModelWrapper.load(GIT_COMMIT)
# in case of using data versioning, don't forget to pull test data from remote data repo
# read held back test data
self.test_data = pd.read_csv("resources/test.csv")
self.min_performance = 0.8
self.predictor = app.config['MODEL']
def build(self, model_file=None):
print(
'***************************build model***************************'
)
self.model = ModelWrapper.model(conf,
train=True,
vocab_size=self.vocab_size,
labels_num=self.labels_num)
adam_optimizer = tf.keras.optimizers.Adam(lr=1e-3,
decay=1e-6,
clipvalue=5)
#self.model.compile(loss=[self.focal_loss(gamma=2., alpha=.25)], optimizer=self.optimizer, metrics=self.metrics)
self.model.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=self.metrics)
if not model_file is None:
self.model.load_weights(model_file, by_name=True)
self.model.summary()
def train(model_name=GIT_COMMIT,
train_data='resources/train.csv',
dvc_data_repo=None,
dvc_ssh_user=None,
dvc_ssh_password=None):
"""This function is the training entry point and should not be removed. Executes the
*ModelWrapper.prep_and_train* and saves the model using *joblib*. MANDATORY
Notes
-----
This function can also be used for your local exploration/development. See the
# >>> if __name__ and "__main__"
Parameters
----------
model_name : String
given the trained model a name to store. Default: git_commit id.
train_data : String
path where to find the train data.
dvc_data_repo : String
dvc_ssh_user : String
dvc_ssh_password : String
Examples
--------
In order to pull /optional) dependencies over the integrated data versioning provided by dvc,
do:
$ syncer = DataSync(dvc_data_repo, dvc_ssh_user, dvc_ssh_password)
$ syncer.pull_data_dependency(train_data)
This will synchronize the dvc data dependencies and training can be conducted
"""
# where to get the data -> hard coded for now
data = pd.read_csv(train_data)
# initiate & train model
logging.getLogger(__name__).info("Starting classification training...")
classification_model = ModelWrapper()
classification_model.prep_and_train(data)
logging.getLogger(__name__).info(
"Starting classification training... Done")
logging.getLogger(__name__).info("Persisting the model...")
classification_model.save(model_name)
from flask import jsonify, request
from services.infrastructure.environment import training_host_url, debug_mode
from services.infrastructure.flask import init_flask
from services.infrastructure.git_info import GIT_COMMIT
from services.infrastructure.logging import initialize_logging
from services.infrastructure.environment import prediction_auth
from model.model_wrapper import ModelWrapper
# initialize flask application
TRAINING_POD_URL = training_host_url()
app, _executor, auth = init_flask()
app.config['USERS'] = prediction_auth()
app.config["MODEL"] = ModelWrapper.load(GIT_COMMIT)
@app.route('/predict', methods=['POST'])
@auth.login_required
def predict():
"""Provides the endpoint for getting new predictions from the developed model using a POST
request json.
HTTP Request Parameters
-----------------------
data : json
Json post containing at least the *ModelWrapper.source_features*. Otherwise a error will
be thrown
HTTP Response
import os
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from model.model_wrapper import ModelWrapper
from model.models import Base
class Connector:
def __init__(self, db):
self.engine = create_engine(db)
Session = sessionmaker(bind=self.engine)
self.session = Session()
def update_model(self):
Base.metadata.create_all(self.engine)
if __name__ == '__main__':
con = Connector(db=os.environ['DB'])
con.update_model()
con.session.add(ModelWrapper.add_user('user', '#1', '*****@*****.**'))
try:
con.session.commit()
except sqlalchemy.exc.IntegrityError:
pass
logging.getLogger(__name__).info("Persisting the model...")
classification_model.save(model_name)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Training model and saving it")
parser.add_argument("--input",
"-i",
type=str,
help="Input file for training",
default="resources/train.csv")
parser.add_argument("--output",
"-o",
type=str,
help="output model name",
default="local")
parsed_args = parser.parse_args()
train(model_name=GIT_COMMIT, train_data=parsed_args.input)
model = ModelWrapper.load(GIT_COMMIT)
# load test Dataframe
test_df = pd.read_csv("resources/test.csv")
print(test_df.to_dict(orient='list'))
res = model.prep_and_predict(test_df.to_dict(orient='list'))
print(res, test_df['Species'].values)
print(accuracy_score(res['prediction'], test_df["Species"].values))