def test_detect_submodule_in_deployment(self): yh = Yhat("greg", "test", "http://api.yhathq.com/") _, bundle = yh.deploy("TestModel", TestModel, globals(), sure=True, dry_run=True) self.assertEqual(len(bundle['modules']), 8)
def index(): if request.method == 'POST': yh = Yhat("USERNAME", "APIKEY", "http://cloud.yhathq.com/") pred = yh.predict("BeerRec", { "beers": request.json['beers'], "n": request.json['n'] }) return Response(json.dumps(pred), mimetype='application/json') else: # static files css_url = url_for('static', filename='css/main.css') jquery_url = url_for('static', filename='js/jquery-1.10.2.min.js') beers_url = url_for('static', filename='js/beers.js') highlight_url = url_for('static', filename='js/code.js') js_url = url_for('static', filename='js/main.js') return render_template('index.html', css_url=css_url, jquery_url=jquery_url, beers_url=beers_url, js_url=js_url, highlight_url=highlight_url)
from yhat import YhatModel, Yhat import nltk bot = nltk.chat.eliza.eliza_chatbot # bot = nltk.chat.iesha.iesha_chatbot class ChatBot(YhatModel): def execute(self, data): text = data['text'] reply = bot.respond(text) return {"reply": reply} print ChatBot().execute({"text": "I'm feeling sad."}) yh = Yhat("greg", "foo", "http://cloud.yhathq.com/") print yh.deploy("ChatBot", ChatBot, globals())
features = [{"name": "x", "na_filler": 0}, {"name": "z", "na_filler": fill_z}] class MyOtherClass: def hello(self, x): return "hello: %s" % str(x) REQS = open("reqs.txt").read() ### <DEPLOYMENT START> ### # @preprocess(in_type=dict, out_type=pd.DataFrame, null_handler=features) class MyModel(YhatModel): REQUIREMENTS = REQS @preprocess(out_type=pd.DataFrame) def execute(self, data): return predict(data) # "push" to server would be here data = {"x": 1, "z": None} if __name__ == '__main__': creds = credentials.read() yh = Yhat(creds['username'], creds['apikey']) yh.deploy_to_file("mynewmodel", MyModel, globals())
get_sims(["Sierra Nevada Pale Ale", "120 Minute IPA", "Coors Light"]) # Index([u'Samuel Adams Boston Lager', u'Sierra Nevada Celebration Ale', u'90 Minute IPA', u'Arrogant Bastard Ale', u'Stone IPA (India Pale Ale)', u'60 Minute IPA', u'HopDevil Ale', u'Stone Ruination IPA', u'Sierra Nevada Bigfoot Barleywine Style Ale', u'Storm King Stout', u'Samuel Adams Winter Lager', u'Samuel Adams Summer Ale', u'Prima Pils', u'Anchor Steam Beer', u'Old Rasputin Russian Imperial Stout', u'Samuel Adams Octoberfest', ...], dtype='object') from yhat import Yhat, YhatModel, preprocess class BeerRecommender(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): beers = data.get("beers") suggested_beers = get_sims(beers) result = [] for beer in suggested_beers: result.append({"beer": beer}) return result BeerRecommender().execute({ "beers": ["Sierra Nevada Pale Ale", "120 Minute IPA", "Stone Ruination IPA"] }) yh = Yhat("USERNAME", "APIKEY", "http://cloud.yhathq.com") yh.deploy("BeerRecommender", BeerRecommender, globals()) yh.predict("BeerRecommender", { "beers": ["Sierra Nevada Pale Ale", "120 Minute IPA", "Stone Ruination IPA"] })
# create and train a classifier nbayes = MultinomialNB(fit_prior=False) nbayes.fit(train_twitter_tfidf, train.liked_content.tolist()) # prep the test data, then create a confusion matrix to examine the results test_twitter_tfidf = vec.transform(test.text) preds = nbayes.predict(test_twitter_tfidf) print pd.crosstab(test.liked_content, preds) from yhat import Yhat, YhatModel, preprocess class TwitterRanker(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): tweet = data['tweet_content'] data = vec.transform([tweet]) pred = nbayes.predict(data) prob = nbayes.predict_proba(data) prob = { "ham": round(prob[0][0], 4), "spam": 1 - round(prob[0][0], 4) } return {"pred": pred[0], "prob": prob} yh = Yhat("YOUR_USERNAME", "YOUR_APIKEY", "http://cloud.yhathq.com/") yh.deploy("twitterRanker", TwitterRanker, globals())
import os from yhat import Yhat, YhatModel, preprocess from foo.foo import print_foo from module import function_in_same_dir class HelloWorld(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): me = data['name'] greeting = "Hello %s!" % me print_foo(me) return {"greeting": greeting, "nine": function_in_same_dir()} username = os.environ["USERNAME"] apikey = os.environ["APIKEY"] endpoint = os.environ["OPS_ENDPOINT"] print "%s:%s:%s" % ( username, apikey, endpoint, ) yh = Yhat(username, apikey, endpoint) yh.deploy("HelloWorldPkg", HelloWorld, globals(), sure=True, verbose=1)
def execute(self, data): P = matrix(data['risk_aversion'] * returns_cov.as_matrix()) q = matrix(-exp_returns['mean'].as_matrix()) G = matrix(0.0, (len(q), len(q))) G[::len(q) + 1] = -1.0 h = matrix(0.0, (len(q), 1)) A = matrix(1.0, (1, len(q))) b = matrix(1.0) solution = solvers.qp(P, q, G, h, A, b) expected_return = exp_returns['mean'].dot(solution['x'])[0] variance = sum(solution['x'] * returns_cov.as_matrix().dot(solution['x']))[0] investments = {} for i, amount in enumerate(solution['x']): # Ignore values that appear to have converged to 0. if amount > 10e-5: investments[countries[i]] = amount * 100 return { 'risk_aversion': data['risk_aversion'], 'investments': investments, 'expected_return': expected_return, 'variance': variance } yh = Yhat('USERNAME', 'APIKEY', 'http://cloud.yhathq.com/') yh.deploy('CurrencyPortfolio', CurrencyPortfolio, globals())
return p[0:n_recs] get_sims(["Sierra Nevada Pale Ale", "60 Minute IPA"]) from yhat import Yhat, YhatModel, preprocess class BeerRecommender(YhatModel): REQUIREMENTS=['numpy==1.11.3', 'pandas==0.19.2', 'scikit-learn==0.18.1'] def execute(self, data): beers = data.get("beers") n_recs = data.get("n_recs") prob = data.get("prob") unique = data.get("unique") suggested_beers = get_sims(beers, n_recs, prob, unique) result = suggested_beers.to_dict(orient='records') return result model = BeerRecommender() model.execute({'beers':["Sierra Nevada Pale Ale"],'n_recs':10}) yh = Yhat("colin", "ce796d278f4840e30e763413d8b4baa4", "http://do-sb-dev-master.x.yhat.com/") print yh.deploy("BeerRecommender", BeerRecommender, globals(), autodetect=False, sure=True) # print yh.predict("BeerRecommender", {"beers": ["Sierra Nevada Pale Ale", # "120 Minute IPA", "Stone Ruination IPA"]})
data = data[features] prob = glm.predict_proba(data)[0][1] if prob > 0.3: decline_code = "Credit score too low" else: decline_code = "" odds = glm.predict_log_proba(data)[0][1] score = calculate_score(odds) output = { "prob_default": [prob], "decline_code": [decline_code], "score": [score] } return output df_term[features].head() test = { "last_fico_range_low": 705, "last_fico_range_high": 732, "home_ownership": "MORTGAGE" } LoanModel().execute(test) yh = Yhat("colin", "d325fc5bcb83fc197ee01edb58b4b396", "https://sandbox.c.yhat.com/") yh.deploy("LendingClub", LoanModel, globals(), True)
decline_code = "Credit score too low" else: decline_code = "" odds = glm.predict_log_proba(data)[0][1] score = calculate_score(odds) output = { "prob_default": [prob], "decline_code": [decline_code], "score": [score] } return output df_term[features].head() test = { "last_fico_range_low": 705, "last_fico_range_high": 732, "home_ownership": "MORTGAGE" } LoanModel().execute(test) yh = Yhat("demo-master", "4a662eb13647cfb9ed4ca36c5e95c7b3", "https://sandbox.yhathq.com/") yh.deploy("LendingClub", LoanModel, globals(), True)
import pandas as pd class ChurnModel(YhatModel): # Type casts incoming data as a dataframe @preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame) def execute(self, data): # Collect customer meta data response = data[['Area Code', 'Phone']] charges = ['Day Charge', 'Eve Charge', 'Night Charge', 'Intl Charge'] response['customer_worth'] = data[charges].sum(axis=1) # Convert yes no columns to bool data[yes_no_cols] = data[yes_no_cols] == 'yes' # Create feature space X = data[features].as_matrix().astype(float) X = scaler.transform(X) # Make prediction churn_prob = clf.predict_proba(X) response['churn_prob'] = churn_prob[:, 1] # Calculate expected loss by churn response['expected_loss'] = response['churn_prob'] * response[ 'customer_worth'] response = response.sort('expected_loss', ascending=False) # Return response DataFrame return response yh = Yhat("*****@*****.**", "eb27a1a0836fea54ffa7c994917e5c92", "http://cloud.yhathq.com/") response = yh.deploy("PythonChurnModel", ChurnModel, globals())
### Actually deploying our model to Yhat from yhat import Yhat, YhatModel class ProductClassifier(YhatModel): def execute(self, data): if "texts" not in data: return {} texts = data["texts"] return make_prediction(texts) # example handling a single record example = {"texts": {"text": "Alpo dog food"}} pp.pprint(ProductClassifier().execute(example)) # example handling multiple records example = {"texts": [{"text": "Alpo dog food"}, {"text": "Diet Coke"}]} pp.pprint(ProductClassifier().execute(example)) YHAT_USERNAME = "" YHAT_APIKEY = "" try: yh = Yhat(YHAT_USERNAME, YHAT_APIKEY, "http://cloud.yhathq.com/") except: print "Please add in your YHAT_USERNAME and YHAT_APIKEY" sys.exit(1) print yh.deploy("ProductClassifier", ProductClassifier, globals())
weights = [float(w) / sum(weights) for w in weights] return (beer, weights) def predict(self, data): beer, weights = data results = [] for beer_cmp in self.beers: if beer != beer_cmp: dist = calc_distance(self.simple_distances, beer, beer_cmp, weights) results.append((beer, beer_cmp, dist)) return sorted(results, key=lambda x: x[2]) #Deploy to Yhat yh = Yhat("{USERNAME}", "{APIKEY}") br = BeerRec(simple_distances=simple_distances, beers=beers, udfs=[calc_distance]) yh.deploy("PydataBeerRec", br) #Test it Out yh.predict("PydataBeerRec", 1, { "beer": "Coors Light", "weights": [1, 1, 1, 1] }) yh.predict("PydataBeerRec", 1, { "beer": "Coors Light", "weights": [2, 1, 0, 0] })
from example_app.models.yhat_model import TestModel, Foo from yhat import Yhat import json yh = Yhat("greg", "foo", "http://api.yhat.com/") TestModel().execute(1) # _, bundle = yh.deploy("Foo", TestModel, globals(), dry_run=True) yh.deploy("Foo", TestModel, globals(), verbose=2) # print json.dumps(bundle, indent=2)
#!/usr/bin/env python from flask import Flask, request, render_template, url_for, Response, json from yhat import Yhat from uuid import uuid4 import numpy as np from bandits import EpsilonGreedy app = Flask(__name__) yh = Yhat("__username__", "__apikey__", "http://cloud.yhathq.com/") arms = ["EuclideanBeerRec", "CosineBeerRec", "CorrelationBeerRec"] eg = EpsilonGreedy(3) ids = {} @app.route('/', methods=['GET', 'POST']) def index(): if request.method == 'POST': arm = eg.choose_arm() arm_name = arms[arm] u_id = str(uuid4()) pred = yh.predict(arm_name, {"beers": request.json['beers']}) ids[u_id] = {'arm': arm, 'arm_name': arm_name} return Response(json.dumps({ 'result': pred['result'], 'uid': u_id }), mimetype='application/json') else:
def is_poor_coverage(row): pct_null = float(row.isnull().sum()) / row.count() return pct_null < 0.8 df_head[df_head.apply(is_poor_coverage, axis=1)] df = df[df.apply(is_poor_coverage, axis=1)] df['year_issued'] = df.issue_d.apply(lambda x: int(x.split("-")[0])) df_term = df[df.year_issued < 2012] features = ['last_fico_range_low', 'last_fico_range_high', 'home_ownership'] yh = Yhat("demo-master", "3b0160e10f6d7a94a2528b11b1c9bca1", "https://sandbox.c.yhat.com/") for i, row in df_term[features][:500].iterrows(): # some models require vectorized data, others don't # non-vectorized # row = row.to_dict() # {'is_rent': True, 'last_fico_range_low': 785, 'last_fico_range_high': 789} # vectorized row = { k: [v] for k, v in row.to_dict().items() } # {'is_rent': [True], 'last_fico_range_low': [785], 'last_fico_range_high': [789]} print yh.predict("LendingClub", row) time.sleep(.05)
def convert_prob_to_score(p): """ takes a probability and converts it to a score Example: convert_prob_to_score(0.1) > 340 """ odds = (1 - p) / p return np.log(odds) * (40 / np.log(2)) + 340 ##Deploying to Yhat from yhat import BaseModel, Yhat yh = Yhat("greg", "abcd1234") class LoanModel(BaseModel): def transform(self, newdata): df = pd.DataFrame(newdata) # handle nulls here # df['monthly_income'] = self.income_imputer.predict(df[[]]) df['number_of_dependents'] = df['number_of_dependents'].fillna(0) return df def predict(self, df): data = df[self.features] result = {} p = self.clf.predict_proba(data) p = p[::, 1]
red_upper = np.array([50, 56, 200], dtype = "uint8") mask = cv2.inRange(image, red_lower, red_upper) output = cv2.bitwise_and(image, image, mask = mask) output_gray = rgb2gray(output) total_red = np.sum(output_gray) y, x = ndimage.center_of_mass(output_gray) data = { "x": x, "y": y, "xmax": output_gray.shape[1], "ymax": output_gray.shape[0], "total_red": total_red, "time": time.time() } return data from yhat import Yhat, YhatModel class DroneModel(YhatModel): REQUIREMENTS = [ "opencv" ] def execute(self, data): return get_coords(data['image64']) yh = Yhat(username, apikey, url) yh.deploy("DroneModel", DroneModel, globals(), True)
def test_deployment(self): yh = Yhat("foo", "bar", "http://api.yhathq.com/") _, bundle = yh.deploy("HelloWorld", HelloWorld, globals(), dry_run=True) self.assertTrue(True)
#!/usr/bin/env python from flask import Flask, request, render_template, url_for, Response, json from yhat import Yhat import os app = Flask(__name__) yh = Yhat(os.environ.get("YHAT_USERNAME"), os.environ.get("YHAT_APIKEY"), os.environ.get("YHAT_URL")) @app.route('/', methods=['GET', 'POST']) def index(): if request.method == 'POST': # print request.json['beers'] try: pred = yh.predict("BeerRecommender", {"beers": request.json['beers'], "n": request.json['n']}) return Response(json.dumps(pred), mimetype='application/json') except Exception, e: print e return Response(json.dumps({"error": str(e)}), mimetype='application/json') else: # static files css_url = url_for('static', filename='css/main.css') jquery_url = url_for('static', filename='js/jquery-1.10.2.min.js') beers_url = url_for('static', filename='js/beers.js') highlight_url = url_for('static', filename='js/code.js') js_url = url_for('static', filename='js/main.js') return render_template('index.html', css_url=css_url, jquery_url=jquery_url, beers_url=beers_url, js_url=js_url, highlight_url=highlight_url)
{ 's1':1, 's2':1, 's3':1, 's4':1, 's5':1, 'w1':1, 'w2':1, 'w3':1, 'w4':1, 'k1':1, 'k2':1, 'k3':1, 'k4':1, 'k5':1, 'k6':1, 'k7':1, 'k8':1, 'k9':1, 'k10':1, 'k11':1, 'k12':1, 'k13':1, 'k14':1, 'k15':1 } test_data = pd.read_csv(open('data/test.csv', 'r'), quotechar='"') sub_data = pd.read_csv(open('data/sampleSubmission.csv', 'r'), quotechar='"') if not np.alltrue(test_data['id'] == sub_data['id']): raise Exception("IDs do not match") yh = Yhat(username, apikey) variabless = sub_data.columns[1:] raw_tweets = test_data['tweet'].tolist() for variable in variables: model_version = best_model[variable] model_name = "TweetClassifier_%s" % (variable, ) results_from_server = yh.raw_predict(model_name, model_version, raw_tweets) pred = results_from_server['prediction']['scores'] sub_data[variable] = pred try: sub_data.to_csv(open(sub_file, 'w'), index=False) except IOError: sys.stderr.write("IO error: could not write data to file")
from yhat import Yhat, YhatModel, preprocess import os USERNAME = os.environ["USERNAME"] APIKEY = os.environ["APIKEY"] URL = os.environ["URL"] class HelloWorld(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): me = data['name'] greeting = "Hello " + str(me) + "!" return {"greeting": greeting} yh = Yhat(USERNAME, APIKEY, URL) yh.deploy("Gitmodel", HelloWorld, globals(), True)
features = [{"name": "x", "na_filler": 0}, {"name": "z", "na_filler": fill_z}] class MyOtherClass: def hello(self, x): return "hello: %s" % str(x) REQS = open("reqs.txt").read() ### <DEPLOYMENT START> ### # @preprocess(in_type=dict, out_type=pd.DataFrame, null_handler=features) class MyModel(YhatModel): REQUIREMENTS = REQS @preprocess(out_type=pd.DataFrame) def execute(self, data): return predict(data) # "push" to server would be here data = {"x": 1, "z": None} if __name__ == '__main__': creds = credentials.read() yh = Yhat(creds['username'], creds['apikey'], "http://localhost:3000/") yh.deploy("mynewmodel", MyModel, globals())
from yhat import YhatModel, Yhat, preprocess # from first import hello as h2 import first as f2 from first import Support from another.testfile import bye def goodbye(y): bye() print y, "goodbye!" class Example(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): goodbye(x) return Support().hello(10) # return h2(data) from first import x yh = Yhat("greg", "fCVZiLJhS95cnxOrsp5e2VSkk0GfypZqeRCntTD1nHA", "http://api.yhathq.com/") yh.deploy_to_file("Example", Example, globals())
data = pd.DataFrame(data) data = data[features] prob = glm.predict_proba(data)[0][1] if prob > 0.3: decline_code = "Credit score too low" else: decline_code = "" odds = glm.predict_log_proba(data)[0][1] score = calculate_score(odds) output = { "prob_default": [prob], "decline_code": [decline_code], "score": [score] } return output df_term[features].head() test = { "last_fico_range_low": 705, "last_fico_range_high": 732, "home_ownership": "MORTGAGE" } LoanModel().execute(test) yh = Yhat("austin", os.environ.get("SCIENCEOPS_API_KEY"), "https://sandbox.c.yhat.com/") yh.deploy("LendingClub", LoanModel, globals(), True)
class ChurnModel(YhatModel): # Type casts incoming data as a dataframe @preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame) def execute(self, data): # Collect customer meta data response = data[['Area Code', 'Phone']] charges = ['Day Charge', 'Eve Charge', 'Night Charge', 'Intl Charge'] response['customer_worth'] = data[charges].sum(axis=1) # Convert yes no columns to bool data[yes_no_cols] = data[yes_no_cols] == 'yes' # Create feature space X = data[features].as_matrix().astype(float) X = scaler.transform(X) # Make prediction churn_prob = clf.predict_proba(X) response['churn_prob'] = churn_prob[:, 1] # Calculate expected loss by churn response['expected_loss'] = response['churn_prob'] * response[ 'customer_worth'] response = response.sort('expected_loss', ascending=False) # Return response DataFrame return response yh = Yhat("e[at]yhathq.com", " MY APIKEY ", "http://cloud.yhathq.com/") print "Deploying model" response = yh.deploy("PythonChurnModel", ChurnModel, globals()) print json.dumps(response, indent=2)
def parse_tweet(tweet): trees = nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(tweet))) for tree in trees.subtrees(): etype = None if tree.node == "PERSON": etype = "PERSON" elif tree.node == "GPE": etype = "PLACE" if etype is not None: ne = " ".join([leaf[0] for leaf in tree.leaves()]) tweet = tweet.replace(ne, "<" + etype + ">" + ne + "</" + etype + ">") return tweet class Tagger(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, raw): tweet = raw['text'] tagged = parse_tweet(tweet) raw['tagged'] = tagged return raw tg = Tagger() yh = Yhat("greg", "mykey", YHAT_URL) print yh.deploy("NamedEntityTagger" + str(i), Tagger, globals())
from yhat import Yhat, YhatModel, preprocess x = range(10) class HelloWorld(YhatModel): @preprocess(in_type=dict, out_type=dict) def execute(self, data): print x[:10] me = data['name'] greeting = "Hello " + str(me) + "!" return {"greeting": greeting, "x": x} # yh = Yhat("greg", "fCVZiLJhS95cnxOrsp5e2VSkk0GfypZqeRCntTD1nHA", "http://cloud.yhathq.com/") yh = Yhat("greg", "9207b9a2dd9d48848b139b729d4354bc", "http://localhost:8080/") yh.deploy("NewZippedModel", HelloWorld, globals())
class ChurnModel(YhatModel): @preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame) def execute(self, data): response = pd.DataFrame(data) charges = ['day_charge', 'eve_charge', 'night_charge', 'intl_charge'] response['customer_worth'] = data[charges].sum(axis=1) # Convert yes no columns to bool data[yes_no_cols] = data[yes_no_cols] == 'yes' # Create feature space X = data[features].as_matrix().astype(float) X = scaler.transform(X) # Make prediction churn_prob = clf.predict_proba(X) response['churn_prob'] = churn_prob[:, 1] # Calculate expected loss by churn response['expected_loss'] = response['churn_prob'] * response[ 'customer_worth'] response = response.sort('expected_loss', ascending=False) response = response[['customer_worth', 'churn_prob', 'expected_loss']] # Return response DataFrame return response yh = Yhat(raw_input("Yhat username: "******"Yhat apikey: "), "http://sandbox.yhathq.com/") print "Deploying model" response = yh.deploy("PythonChurnModel", ChurnModel, globals()) print df_to_json(churn_df[:1])