def __test_model_save_load_helper__(self, model):
with util.TempDirectory() as f:
model.save(f)
m2 = tc.load_model(f)
self.assertItemsEqual(model._list_fields(), m2._list_fields())
for key in model._list_fields():
if type(model._get(key)) is SGraph:
self.assertItemsEqual(
model._get(key).summary(),
m2._get(key).summary())
self.assertItemsEqual(
model._get(key).get_fields(),
m2._get(key).get_fields())
elif type(model._get(key)) is SFrame:
sf1 = model._get(key)
sf2 = m2._get(key)
self.assertEqual(len(sf1), len(sf2))
self.assertItemsEqual(sf1.column_names(),
sf2.column_names())
df1 = sf1.to_dataframe()
print(df1)
df2 = sf2.to_dataframe()
print(df2)
df1 = df1.set_index(df1.columns[0])
df2 = df2.set_index(df2.columns[0])
assert_frame_equal(df1, df2)
else:
if (type(model._get(key)) is pd.DataFrame):
assert_frame_equal(model._get(key), m2._get(key))
else:
self.assertEqual(model._get(key), m2._get(key))
def setUpClass(self):
# Create an example containing synthetic data along
# with fitted models (with default parameters).
docs = generate_bar_example(num_documents=1000, seed=12345)
models = []
# Test a model that used CGS
m = topic_model.create(docs, num_topics=10)
models.append(m)
# Test a model with many topics
m = topic_model.create(docs,
method='cgs',
num_topics=100,
num_iterations=2)
models.append(m)
m = topic_model.create(docs,
method='alias',
num_topics=100,
num_iterations=2)
models.append(m)
# Test a model serialized after using CGS
with test_util.TempDirectory() as f:
m.save(f)
m2 = turicreate.load_model(f)
models.append(m2)
# Save
examples['synthetic'] = {'docs': docs, 'models': models}
self.docs = examples['synthetic']['docs']
self.models = examples['synthetic']['models']
def test_save_and_load(self):
"""
Make sure saving and loading retains everything.
"""
filename = 'save_file{}'.format(uuid.uuid4())
self.model.save(filename)
self.model = tc.load_model(filename)
try:
self.test_get()
print("Get passed")
self.test_coefficients()
print("Coefficients passed")
self.test_summary()
print("Summary passed")
self.test_repr()
print("Repr passed")
self.test_predict()
print("Predict passed")
self.test_classify()
print("Classify passed")
self.test_evaluate()
print("Evaluate passed")
self.test__list_fields()
print("List fields passed")
shutil.rmtree(filename)
except:
self.assertTrue(False, "Failed during save & load diagnostics")
def test_save_and_load(self):
"""
Make sure saving and loading retains things.
"""
filename = 'save_file%s' % (str(uuid.uuid4()))
self.model.save(filename)
self.model = tc.load_model(filename)
try:
self.test_summary()
print("Summary passed")
self.test_repr()
print("Repr passed")
self.test_predict()
print("Predict passed")
self.test_evaluate()
print("Evaluate passed")
self.test_extract_features()
print("Extract features passed")
self.test_feature_importance()
print("Feature importance passed")
self.test__list_fields()
print("List field passed")
self.test_get()
print("Get passed")
shutil.rmtree(filename)
except:
self.assertTrue(False, "Failed during save & load diagnostics")
def test_save_and_load(self):
"""
Make sure saving and loading retains everything.
"""
test_methods_list = [
func
for func in dir(self)
if callable(getattr(self, func)) and func.startswith("test")
]
test_methods_list.remove("test_save_and_load")
with test_util.TempDirectory() as filename:
self.model.save(filename)
self.model = None
self.model = tc.load_model(filename)
print("Repeating all test cases after model delete and reload")
for test_method in test_methods_list:
try:
getattr(self, test_method)()
print("Save and Load:", test_method, "has passed")
except unittest.SkipTest:
pass
except Exception as e:
self.assertTrue(
False,
"After model save and load, method "
+ test_method
+ " has failed with error: "
+ str(e),
)
def predictPrice(self):
scalerX = joblib.load(self.ticker1 + "-" + self.ticker2 +
'-scalerX.gz')
scalerY = joblib.load(self.ticker1 + "-" + self.ticker2 +
'-scalerY.gz')
myFeatures = []
for i in range(self.win, self.past * self.win):
myFeatures.append('lag-' + str(i))
# Automatically picks the right model based on your data.
model = tc.load_model(self.ticker1 + "-" + self.ticker2 +
'-regression')
hist = self.load_data(self.ticker1, self.period)
current = np.array(hist.iloc[-1 * (self.past - 1) *
self.win:].values).reshape(1, -1)
#print(current)
current = scalerX.transform(current)
current = tc.SFrame(current)
future = model.predict(current)
#print(hist)
price = hist['Close'].values
current = tc.SFrame(price[-1 * (self.past - 1) * self.win:])
future = model.predict(current)
future = np.expand_dims(future, axis=0)
print(future[0][0])
future = scalerY.inverse_transform(future)
print("predicted price after transform is {}".format(future[0][0]))
return future[0][0]
def test_sparse_nn(self):
X = tc.util.generate_random_sframe(100, "ssszzz")
Y = X.copy()
X = X.add_row_number()
m = tc.extensions._sparse_nn()
m.train(X, "id")
for i, row in enumerate(Y):
res = m.query(row, 1)
self.assertEqual(res, {i: 1.0})
# Save and load
model_file = tempfile.gettempdir() + "/sparse_nn.model"
m.save(model_file)
m2 = tc.load_model(model_file)
for i, row in enumerate(Y):
res = m2.query(Y[i], 1)
self.assertEqual(res, {i: 1.0})
# in case file not exist
try:
shutil.rmtree(model_file)
except (OSError, IOError) as e:
if e.errno == errno.EEXIST:
pass
else:
raise e
def __init__(self):
self.imgframe = tc.load_sframe('model/final/final.sframe')
self.model = tc.load_model('model/final/final_model')
self.sample = tc.Image()
self.results = SFrame()
self.rows = SArray()
self.pathlist = []
self.distance_list = []
def get_classification(filename):
modelPath = os.path.join(os.getcwd(), app.config['ML_MODEL'])
print('Model Path : ', modelPath)
model = tc.load_model(modelPath)
print('Analysing image..')
sfData = tc.image_analysis.load_images(filename, with_path=True)
classify = model.classify(sfData)
return classify
def test_save_and_load(self):
with test_util.TempDirectory() as filename:
self.model.save(filename)
self.model = tc.load_model(filename)
self.test_predict()
self.test_get()
self.test_summary()
self.test__list_fields()
def test_save_and_load(self):
"""
Ensure that model saving and loading retains all model information.
"""
with test_util.TempDirectory() as f:
self.model.save(f)
self.model = tc.load_model(f)
loaded_model = tc.load_model(f)
self.test__list_fields()
print("Saved model list fields passed")
self.test_get()
print("Saved model get passed")
self.test_summaries()
print("Saved model summaries passed")
def test_save_and_load(self):
with test_util.TempDirectory() as filename:
self.model.save(filename)
self.model = tc.load_model(filename)
self.test_stylize_success()
print("Stylize passed")
self.test_get_styles_success()
print("Get styles passed")
def predict(user_id, meal_id):
my_model = tc.load_model("my_model_file")
prediction = my_model.predict(dataset={"id": user_id, "meal__id": meal_id})
dict = {}
for item in prediction:
dict["prediction"] = item
dict["user_id"] = str(user_id)
dict["meal_id"] = str(meal_id)
return dict
def predictions_step1_page():
# try:
tc.config.set_num_gpus(0)
model_id = request.args.get('model_id')
my_model = TrainedModel.query.filter_by(id=model_id).first()
my_data = UserData.query.filter_by(project_id=my_model.project_id).all()
if my_data[0].user_id is not current_user.id:
flash('Opps! Do data found', 'error')
return redirect(request.referrer)
form = UserProfileForm(request.form, obj=current_user)
if request.method == 'POST':
data_id = request.form['data_set_id']
my_data = UserData.query.filter_by(id=data_id).first()
data_frame = tc.load_sframe(my_data.sname)
if my_model.features['model_type'] == 'deep':
tfrm = data_frame.to_dataframe()
tfrm = tfrm.sort_values(by=[my_model.features["session_id"], my_model.features["time_field"]])
data_frame = tc.SFrame(data=tfrm)
data_frame[str(my_model.features["session_id"])] = data_frame[str(my_model.features["session_id"])].astype(int)
model = tc.load_model(my_model.mname)
predictions = model.predict(data_frame).to_numpy()
my_dict = Predictions()
my_dict.model_id = my_model.id
my_dict.user_id = current_user.id
my_dict.path = my_model.path
my_dict.input_file = my_data.name
my_predictions = []
for item in predictions:
my_predictions.append(str(item))
my_dict.predictions = my_predictions
origs = []
for item in data_frame[str(my_model.features['target'])]:
origs.append(str(item))
# Make sure the predictions only overwrite blank values
if request.form['mode'] == "fill":
size = len(predictions)
for x in range(0, size):
if origs[x] is not None:
predictions[x] = origs[x]
my_dict.originals = origs
data_frame = safely_add_col('Predicted_Value', predictions, data_frame)
my_dict.oname = os.path.join(my_dict.path, str(uuid.uuid4()) + "_model_predictions.csv")
data_frame.save(my_dict.oname, format='csv')
db.session.add(my_dict)
db.session.commit()
# Redirect to home page
return redirect(url_for('model.prediction_page', dict=my_dict.id))
return render_template('pages/models/predict_step1.html',
my_data=my_data,
my_model=my_model,
form=form)
def test_save_and_load(self):
for index in range(len(self.models)):
old_model, data = self.models[index], self.trains[index]
with test_util.TempDirectory() as filename:
old_model.save(filename)
new_model = _tc.load_model(filename)
old_preds = old_model.predict(data)
new_preds = new_model.predict(data)
assert (new_preds.dtype == old_preds.dtype
and (new_preds == old_preds).all())
def recommendByItems(self, itemId, topN=10):
if os.path.exists(self.ItemModelPath):
model = tc.load_model(self.ItemModelPath)
else:
model = self.trainItemBasedCF()
results = model.get_similar_items(items=[itemId], k=topN)
items = self.getRecommendJson(topN, results['similar'])
return items
def getPopularForUser(self, userId, topN=10):
if os.path.exists(self.PopularModelPath):
model = tc.load_model(self.PopularModelPath)
else:
model = self.trainPopular()
results = model.recommend(users=[userId], k=topN, diversity=random.uniform(1, 3))
items = self.getRecommendJson(topN, results[self.itemKey])
return items
def main():
# define the folder structure we inherited from SageMaker pipelines
data_path = Path("/opt/ml/processing").resolve()
model_dir = data_path/"model"
test_dir = data_path/"test"
output_eval_dir = data_path/"evaluation"
output_mlmodel_dir = data_path/"mlmodel"
output_images_dir = data_path/"eval_images"
# load and untar the model file
model_path = model_dir/"model.tar.gz"
with tarfile.open(model_path) as tar:
tar.extractall(path=".")
model = tc.load_model("bitti.model")
logging.info("Loaded the model from %s", str(model_path))
# load and score the testing data
test_path = test_dir/"bitti_test.sframe"
test_data = tc.SFrame(str(test_path)) # don't pass raw Path instances
metrics = model.evaluate(test_data)
logging.info("Evaluating the model on test data: %s", metrics)
mAP = metrics['mean_average_precision_50']
# TODO: generate a few images for model card here
predictions = model.predict(test_data)
for image, pred in zip(test_data[:EVAL_N_IMAGES],
predictions[:EVAL_N_IMAGES]):
draw_predictions(image['image'].pixel_data, pred,
output_images_dir/image['name'])
# generate and save the evaluation report
report_dict = {"regression_metrics": {"mAP": {"value": mAP}}}
Path(output_eval_dir).mkdir(parents=True, exist_ok=True)
evaluation_path = output_eval_dir/"evaluation.json"
# convert turicreate model to coreml format and add metadata
mlmodel_path = str(output_mlmodel_dir/'bitti.mlmodel')
model.export_coreml(mlmodel_path)
model = coremltools.models.MLModel(mlmodel_path)
model.author = 'Solita Oy'
model.license = 'TBD'
model.short_description = 'BITTI magazine logo detector'
model.versionString = '1.0'
model.save(mlmodel_path)
with open(evaluation_path, "w") as f:
f.write(json.dumps(report_dict))
if evaluation_path.exists():
logging.info("Successfully dumped evaluation JSON to `%s`.",
str(evaluation_path))
else:
logging.error("Failed writing the evaluation file!")
logging.info("Evaluation script finished. Storing mAP=%.2f"
" into the evaluation report", mAP)
def test_save_and_load(self):
with TempDirectory() as filename:
self.model.save(filename)
new_model = tc.load_model(filename)
self.assertEqual(self.model.feature, new_model.feature)
old_model_probs = self.model.predict(self.data['audio'], output_type='probability_vector')
new_model_probs = new_model.predict(self.data['audio'], output_type='probability_vector')
for a, b in zip(old_model_probs, new_model_probs):
np.testing.assert_array_almost_equal(a, b, decimal=6)
def img_search(self):
test_data = tc.image_analysis.load_images('/Users/jiang/Pictures/test',
with_path=True,
recursive=True)
test_data.explore()
loaded_model = tc.load_model('picture_model')
similar_images = loaded_model.query(test_data, label='path', k=3)
return similar_images
def test_exception(self):
# load model from empty file
with util.TempDirectory() as tmp_empty_file:
with self.assertRaises(IOError):
tc.load_model(tmp_empty_file)
# load model from non-existing file
if (os.path.exists('./tmp_model-%d' % temp_number)):
shutil.rmtree('./tmp_model-%d' % temp_number)
with self.assertRaises(IOError):
tc.load_model('./tmp_model-%d' % temp_number)
# save model to invalid url
restricted_place = None
if sys.platform == 'win32':
restricted_place = 'C:\\Windows\\System32\\config\\RegBack\\testmodel'
else:
restricted_place = '/root/tmp/testmodel'
for url in ['http://test', restricted_place]:
with self.assertRaises(IOError):
self.pr_model.save(url)
def test_save_and_load(self):
with test_util.TempDirectory() as filename:
self.model.save(filename)
self.model = tc.load_model(filename)
self.test_predict()
print("Predict passed")
self.test_get()
print("Get passed")
self.test_summary()
print("Summary passed")
self.test__list_fields()
print("List fields passed")
def test_gpu_save_load_export(self):
old_num_gpus = tc.config.get_num_gpus()
gpu_options = set([old_num_gpus, 0, 1])
for in_gpus in gpu_options:
for out_gpus in gpu_options:
tc.config.set_num_gpus(in_gpus)
model = tc.image_similarity.create(self.sf, feature=self.feature,
label=None, model=self.pre_trained_model)
with TempDirectory() as filename:
model.save(filename)
tc.config.set_num_gpus(out_gpus)
model = tc.load_model(filename)
tc.config.set_num_gpus(old_num_gpus)
def test_gpu_save_load_export(self):
old_num_gpus = tc.config.get_num_gpus()
gpu_options = set([old_num_gpus, 0, 1])
for in_gpus in gpu_options:
for out_gpus in gpu_options:
tc.config.set_num_gpus(in_gpus)
model = tc.style_transfer.create(self.style_sf, self.content_sf, max_iterations=1)
with test_util.TempDirectory() as path:
model.save(path)
tc.config.set_num_gpus(out_gpus)
model = tc.load_model(path)
model.export_coreml(os.path.join(path, 'model.mlmodel'))
tc.config.set_num_gpus(old_num_gpus)
def classified_write():
"""use model to classify target dataset by
two classes FAKE/REAL, and write to db dataframe"""
base = pd.read_csv(config.csv_news)
model = tc.load_model('model_fake_or_real')
target = tc.SFrame.read_csv(config.csv_news, verbose=False)\
.remove_column('link').remove_column('date')
classify = model.classify(tc.SFrame(target))
classified = pd.DataFrame(classify)
base['classified'] = classified['class']
base['probability'] = classified['probability'].round(2)
# return print(target, flush=True)
return base.to_csv(config.csv_classified, index=False)
def test_gpu_save_load_export(self):
old_num_gpus = tc.config.get_num_gpus()
gpu_options = set([old_num_gpus, 0, 1])
for in_gpus in gpu_options:
tc.config.set_num_gpus(in_gpus)
model = tc.object_detector.create(self.sf, max_iterations=1)
for out_gpus in gpu_options:
with test_util.TempDirectory() as path:
model.save(path)
tc.config.set_num_gpus(out_gpus)
model = tc.load_model(path)
model.export_coreml(os.path.join(path, "model.mlmodel"))
tc.config.set_num_gpus(old_num_gpus)
def query(request):
if request.user.is_authenticated:
body = json.loads(request.body)
users_to_recommend = [body["uid"]]
n_rec = body["n_rec"]
model = tc.load_model("static/model")
recom = model.recommend(users=["uid"], k=n_rec)
recom = pd.DataFrame(recom)
recom = recom["productID"]
# print(list(recom))
return JsonResponse({"users_to_recommend": users_to_recommend, "recom": list(recom)})
else:
return HttpResponse("You need to login to do that.")
def load_model( self , model , model_name , train ):
if model is None:
model_path = "./models/%s" % model_name
if not file_exists( model_path ):
model = train()
model.save( model_path )
else:
model = tc.load_model( model_path );
return model
def test_basic_save_load(self):
# save and load the pagerank model
with util.TempDirectory() as tmp_pr_model_file:
self.pr_model.save(tmp_pr_model_file)
pr_model2 = tc.load_model(tmp_pr_model_file)
self.__assert_model_equals__(self.pr_model, pr_model2)
# save and load the connected_component model
with util.TempDirectory() as tmp_cc_model_file:
self.cc_model.save(tmp_cc_model_file)
cc_model2 = tc.load_model(tmp_cc_model_file)
self.__assert_model_equals__(self.cc_model, cc_model2)
# handle different types of urls.
# TODO: test hdfs and s3 urls.
for url in [
'./tmp_model-%d' % temp_number,
'/tmp/tmp_model-%d' % temp_number,
'~/tmp/tmp_model-%d' % temp_number
]:
self.pr_model.save(url)
self.__assert_model_equals__(self.pr_model, tc.load_model(url))
def main():
# df = read_data()
# df_side_info = create_side_info(df)
# df_side_info = categorize_column(df_side_info, ['race', 'diet', 'allergy', 'groceries'])
# df_turi, df_side_info = to_recommendation_format(df, df_side_info)
# df_turi = df_turi.loc[df_turi['rating'] != -2][0:20]
recommendations= []
form = UserDem()
input_dict = {}
if form.is_submitted(): # form.validate_on_submit():
for field in form:
input_dict[field.name] = [field.data]
# flash(f'Account created for {form.Age.data}!', 'success')
#aging = form.Age.data
#grocery = form.Groceries.data
input_dict.pop('submit')
input_dict.pop('csrf_token')
#name= input_dict.pop('user_id')
# return redirect(url_for('project1'))
input_dict['user_id'][0] = convert_to_num(input_dict['user_id'][0])
name= input_dict['user_id'][0]
new_user_info= tc.SFrame(input_dict)
print(new_user_info)
#name= input_dict.pop('user_id')[0]
# rating = request.form.get("ratings", None)
# grocery= request.form.get("glist", None)
loaded_model = tc.load_model('Foodpollo_Recommender')
df_turi= loaded_model.recommend([name], new_user_data=new_user_info)
df_turi = df_turi.to_dataframe()
df_turi= df_turi[['item_id', 'rank']].head(10)
for i in range(df_turi.shape[0]):
if df_turi.iloc[i, 0] in vals:
extracted_name = re.sub('enjoy_', "", df_turi.iloc[i, 0])
extracted_name += ' recipes'
# print('Your recommended meals for {} are \n {} \n'.format(extracted_name,random.sample(vals[df_turi.iloc[i,0]],2)))
recommendations.append('Your recommended meals for {} are {}'.format(extracted_name,
random.sample(vals[df_turi.iloc[i, 0]], 2)))
#return render_template('project1.html', movie_names=[df_turi.to_html(classes='data')], titles=df_turi.columns.values, form=form)
return render_template('project1.html', recommendations=recommendations, form=form)
