def test_detect_submodule_in_deployment(self):
yh = Yhat("greg", "test", "http://api.yhathq.com/")
_, bundle = yh.deploy("TestModel",
TestModel,
globals(),
sure=True,
dry_run=True)
self.assertEqual(len(bundle['modules']), 8)
def index():
if request.method == 'POST':
yh = Yhat("USERNAME", "APIKEY", "http://cloud.yhathq.com/")
pred = yh.predict("BeerRec", {
"beers": request.json['beers'],
"n": request.json['n']
})
return Response(json.dumps(pred), mimetype='application/json')
else:
# static files
css_url = url_for('static', filename='css/main.css')
jquery_url = url_for('static', filename='js/jquery-1.10.2.min.js')
beers_url = url_for('static', filename='js/beers.js')
highlight_url = url_for('static', filename='js/code.js')
js_url = url_for('static', filename='js/main.js')
return render_template('index.html',
css_url=css_url,
jquery_url=jquery_url,
beers_url=beers_url,
js_url=js_url,
highlight_url=highlight_url)
from yhat import YhatModel, Yhat
import nltk
bot = nltk.chat.eliza.eliza_chatbot
# bot = nltk.chat.iesha.iesha_chatbot
class ChatBot(YhatModel):
def execute(self, data):
text = data['text']
reply = bot.respond(text)
return {"reply": reply}
print ChatBot().execute({"text": "I'm feeling sad."})
yh = Yhat("greg", "foo", "http://cloud.yhathq.com/")
print yh.deploy("ChatBot", ChatBot, globals())
features = [{"name": "x", "na_filler": 0}, {"name": "z", "na_filler": fill_z}]
class MyOtherClass:
def hello(self, x):
return "hello: %s" % str(x)
REQS = open("reqs.txt").read()
### <DEPLOYMENT START> ###
# @preprocess(in_type=dict, out_type=pd.DataFrame, null_handler=features)
class MyModel(YhatModel):
REQUIREMENTS = REQS
@preprocess(out_type=pd.DataFrame)
def execute(self, data):
return predict(data)
# "push" to server would be here
data = {"x": 1, "z": None}
if __name__ == '__main__':
creds = credentials.read()
yh = Yhat(creds['username'], creds['apikey'])
yh.deploy_to_file("mynewmodel", MyModel, globals())
get_sims(["Sierra Nevada Pale Ale", "120 Minute IPA", "Coors Light"])
# Index([u'Samuel Adams Boston Lager', u'Sierra Nevada Celebration Ale', u'90 Minute IPA', u'Arrogant Bastard Ale', u'Stone IPA (India Pale Ale)', u'60 Minute IPA', u'HopDevil Ale', u'Stone Ruination IPA', u'Sierra Nevada Bigfoot Barleywine Style Ale', u'Storm King Stout', u'Samuel Adams Winter Lager', u'Samuel Adams Summer Ale', u'Prima Pils', u'Anchor Steam Beer', u'Old Rasputin Russian Imperial Stout', u'Samuel Adams Octoberfest', ...], dtype='object')
from yhat import Yhat, YhatModel, preprocess
class BeerRecommender(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
beers = data.get("beers")
suggested_beers = get_sims(beers)
result = []
for beer in suggested_beers:
result.append({"beer": beer})
return result
BeerRecommender().execute({
"beers":
["Sierra Nevada Pale Ale", "120 Minute IPA", "Stone Ruination IPA"]
})
yh = Yhat("USERNAME", "APIKEY", "http://cloud.yhathq.com")
yh.deploy("BeerRecommender", BeerRecommender, globals())
yh.predict("BeerRecommender", {
"beers":
["Sierra Nevada Pale Ale", "120 Minute IPA", "Stone Ruination IPA"]
})
# create and train a classifier
nbayes = MultinomialNB(fit_prior=False)
nbayes.fit(train_twitter_tfidf, train.liked_content.tolist())
# prep the test data, then create a confusion matrix to examine the results
test_twitter_tfidf = vec.transform(test.text)
preds = nbayes.predict(test_twitter_tfidf)
print pd.crosstab(test.liked_content, preds)
from yhat import Yhat, YhatModel, preprocess
class TwitterRanker(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
tweet = data['tweet_content']
data = vec.transform([tweet])
pred = nbayes.predict(data)
prob = nbayes.predict_proba(data)
prob = {
"ham": round(prob[0][0], 4),
"spam": 1 - round(prob[0][0], 4)
}
return {"pred": pred[0], "prob": prob}
yh = Yhat("YOUR_USERNAME", "YOUR_APIKEY", "http://cloud.yhathq.com/")
yh.deploy("twitterRanker", TwitterRanker, globals())
import os
from yhat import Yhat, YhatModel, preprocess
from foo.foo import print_foo
from module import function_in_same_dir
class HelloWorld(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
me = data['name']
greeting = "Hello %s!" % me
print_foo(me)
return {"greeting": greeting, "nine": function_in_same_dir()}
username = os.environ["USERNAME"]
apikey = os.environ["APIKEY"]
endpoint = os.environ["OPS_ENDPOINT"]
print "%s:%s:%s" % (
username,
apikey,
endpoint,
)
yh = Yhat(username, apikey, endpoint)
yh.deploy("HelloWorldPkg", HelloWorld, globals(), sure=True, verbose=1)
def execute(self, data):
P = matrix(data['risk_aversion'] * returns_cov.as_matrix())
q = matrix(-exp_returns['mean'].as_matrix())
G = matrix(0.0, (len(q), len(q)))
G[::len(q) + 1] = -1.0
h = matrix(0.0, (len(q), 1))
A = matrix(1.0, (1, len(q)))
b = matrix(1.0)
solution = solvers.qp(P, q, G, h, A, b)
expected_return = exp_returns['mean'].dot(solution['x'])[0]
variance = sum(solution['x'] *
returns_cov.as_matrix().dot(solution['x']))[0]
investments = {}
for i, amount in enumerate(solution['x']):
# Ignore values that appear to have converged to 0.
if amount > 10e-5:
investments[countries[i]] = amount * 100
return {
'risk_aversion': data['risk_aversion'],
'investments': investments,
'expected_return': expected_return,
'variance': variance
}
yh = Yhat('USERNAME', 'APIKEY', 'http://cloud.yhathq.com/')
yh.deploy('CurrencyPortfolio', CurrencyPortfolio, globals())
return p[0:n_recs]
get_sims(["Sierra Nevada Pale Ale", "60 Minute IPA"])
from yhat import Yhat, YhatModel, preprocess
class BeerRecommender(YhatModel):
REQUIREMENTS=['numpy==1.11.3',
'pandas==0.19.2',
'scikit-learn==0.18.1']
def execute(self, data):
beers = data.get("beers")
n_recs = data.get("n_recs")
prob = data.get("prob")
unique = data.get("unique")
suggested_beers = get_sims(beers, n_recs, prob, unique)
result = suggested_beers.to_dict(orient='records')
return result
model = BeerRecommender()
model.execute({'beers':["Sierra Nevada Pale Ale"],'n_recs':10})
yh = Yhat("colin", "ce796d278f4840e30e763413d8b4baa4", "http://do-sb-dev-master.x.yhat.com/")
print yh.deploy("BeerRecommender", BeerRecommender, globals(), autodetect=False, sure=True)
# print yh.predict("BeerRecommender", {"beers": ["Sierra Nevada Pale Ale",
# "120 Minute IPA", "Stone Ruination IPA"]})
data = data[features]
prob = glm.predict_proba(data)[0][1]
if prob > 0.3:
decline_code = "Credit score too low"
else:
decline_code = ""
odds = glm.predict_log_proba(data)[0][1]
score = calculate_score(odds)
output = {
"prob_default": [prob],
"decline_code": [decline_code],
"score": [score]
}
return output
df_term[features].head()
test = {
"last_fico_range_low": 705,
"last_fico_range_high": 732,
"home_ownership": "MORTGAGE"
}
LoanModel().execute(test)
yh = Yhat("colin", "d325fc5bcb83fc197ee01edb58b4b396",
"https://sandbox.c.yhat.com/")
yh.deploy("LendingClub", LoanModel, globals(), True)
decline_code = "Credit score too low"
else:
decline_code = ""
odds = glm.predict_log_proba(data)[0][1]
score = calculate_score(odds)
output = {
"prob_default": [prob],
"decline_code": [decline_code],
"score": [score]
}
return output
df_term[features].head()
test = {
"last_fico_range_low": 705,
"last_fico_range_high": 732,
"home_ownership": "MORTGAGE"
}
LoanModel().execute(test)
yh = Yhat("demo-master", "4a662eb13647cfb9ed4ca36c5e95c7b3",
"https://sandbox.yhathq.com/")
yh.deploy("LendingClub", LoanModel, globals(), True)
import pandas as pd
class ChurnModel(YhatModel):
# Type casts incoming data as a dataframe
@preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame)
def execute(self, data):
# Collect customer meta data
response = data[['Area Code', 'Phone']]
charges = ['Day Charge', 'Eve Charge', 'Night Charge', 'Intl Charge']
response['customer_worth'] = data[charges].sum(axis=1)
# Convert yes no columns to bool
data[yes_no_cols] = data[yes_no_cols] == 'yes'
# Create feature space
X = data[features].as_matrix().astype(float)
X = scaler.transform(X)
# Make prediction
churn_prob = clf.predict_proba(X)
response['churn_prob'] = churn_prob[:, 1]
# Calculate expected loss by churn
response['expected_loss'] = response['churn_prob'] * response[
'customer_worth']
response = response.sort('expected_loss', ascending=False)
# Return response DataFrame
return response
yh = Yhat("*****@*****.**", "eb27a1a0836fea54ffa7c994917e5c92",
"http://cloud.yhathq.com/")
response = yh.deploy("PythonChurnModel", ChurnModel, globals())
### Actually deploying our model to Yhat
from yhat import Yhat, YhatModel
class ProductClassifier(YhatModel):
def execute(self, data):
if "texts" not in data:
return {}
texts = data["texts"]
return make_prediction(texts)
# example handling a single record
example = {"texts": {"text": "Alpo dog food"}}
pp.pprint(ProductClassifier().execute(example))
# example handling multiple records
example = {"texts": [{"text": "Alpo dog food"}, {"text": "Diet Coke"}]}
pp.pprint(ProductClassifier().execute(example))
YHAT_USERNAME = ""
YHAT_APIKEY = ""
try:
yh = Yhat(YHAT_USERNAME, YHAT_APIKEY, "http://cloud.yhathq.com/")
except:
print "Please add in your YHAT_USERNAME and YHAT_APIKEY"
sys.exit(1)
print yh.deploy("ProductClassifier", ProductClassifier, globals())
weights = [float(w) / sum(weights) for w in weights]
return (beer, weights)
def predict(self, data):
beer, weights = data
results = []
for beer_cmp in self.beers:
if beer != beer_cmp:
dist = calc_distance(self.simple_distances, beer, beer_cmp,
weights)
results.append((beer, beer_cmp, dist))
return sorted(results, key=lambda x: x[2])
#Deploy to Yhat
yh = Yhat("{USERNAME}", "{APIKEY}")
br = BeerRec(simple_distances=simple_distances,
beers=beers,
udfs=[calc_distance])
yh.deploy("PydataBeerRec", br)
#Test it Out
yh.predict("PydataBeerRec", 1, {
"beer": "Coors Light",
"weights": [1, 1, 1, 1]
})
yh.predict("PydataBeerRec", 1, {
"beer": "Coors Light",
"weights": [2, 1, 0, 0]
})
from example_app.models.yhat_model import TestModel, Foo
from yhat import Yhat
import json
yh = Yhat("greg", "foo", "http://api.yhat.com/")
TestModel().execute(1)
# _, bundle = yh.deploy("Foo", TestModel, globals(), dry_run=True)
yh.deploy("Foo", TestModel, globals(), verbose=2)
# print json.dumps(bundle, indent=2)
#!/usr/bin/env python
from flask import Flask, request, render_template, url_for, Response, json
from yhat import Yhat
from uuid import uuid4
import numpy as np
from bandits import EpsilonGreedy
app = Flask(__name__)
yh = Yhat("__username__", "__apikey__", "http://cloud.yhathq.com/")
arms = ["EuclideanBeerRec", "CosineBeerRec", "CorrelationBeerRec"]
eg = EpsilonGreedy(3)
ids = {}
@app.route('/', methods=['GET', 'POST'])
def index():
if request.method == 'POST':
arm = eg.choose_arm()
arm_name = arms[arm]
u_id = str(uuid4())
pred = yh.predict(arm_name, {"beers": request.json['beers']})
ids[u_id] = {'arm': arm, 'arm_name': arm_name}
return Response(json.dumps({
'result': pred['result'],
'uid': u_id
}),
mimetype='application/json')
else:
def is_poor_coverage(row):
pct_null = float(row.isnull().sum()) / row.count()
return pct_null < 0.8
df_head[df_head.apply(is_poor_coverage, axis=1)]
df = df[df.apply(is_poor_coverage, axis=1)]
df['year_issued'] = df.issue_d.apply(lambda x: int(x.split("-")[0]))
df_term = df[df.year_issued < 2012]
features = ['last_fico_range_low', 'last_fico_range_high', 'home_ownership']
yh = Yhat("demo-master", "3b0160e10f6d7a94a2528b11b1c9bca1",
"https://sandbox.c.yhat.com/")
for i, row in df_term[features][:500].iterrows():
# some models require vectorized data, others don't
# non-vectorized
# row = row.to_dict() # {'is_rent': True, 'last_fico_range_low': 785, 'last_fico_range_high': 789}
# vectorized
row = {
k: [v]
for k, v in row.to_dict().items()
} # {'is_rent': [True], 'last_fico_range_low': [785], 'last_fico_range_high': [789]}
print yh.predict("LendingClub", row)
time.sleep(.05)
def convert_prob_to_score(p):
"""
takes a probability and converts it to a score
Example:
convert_prob_to_score(0.1)
> 340
"""
odds = (1 - p) / p
return np.log(odds) * (40 / np.log(2)) + 340
##Deploying to Yhat
from yhat import BaseModel, Yhat
yh = Yhat("greg", "abcd1234")
class LoanModel(BaseModel):
def transform(self, newdata):
df = pd.DataFrame(newdata)
# handle nulls here
# df['monthly_income'] = self.income_imputer.predict(df[[]])
df['number_of_dependents'] = df['number_of_dependents'].fillna(0)
return df
def predict(self, df):
data = df[self.features]
result = {}
p = self.clf.predict_proba(data)
p = p[::, 1]
red_upper = np.array([50, 56, 200], dtype = "uint8")
mask = cv2.inRange(image, red_lower, red_upper)
output = cv2.bitwise_and(image, image, mask = mask)
output_gray = rgb2gray(output)
total_red = np.sum(output_gray)
y, x = ndimage.center_of_mass(output_gray)
data = {
"x": x,
"y": y,
"xmax": output_gray.shape[1],
"ymax": output_gray.shape[0],
"total_red": total_red,
"time": time.time()
}
return data
from yhat import Yhat, YhatModel
class DroneModel(YhatModel):
REQUIREMENTS = [
"opencv"
]
def execute(self, data):
return get_coords(data['image64'])
yh = Yhat(username, apikey, url)
yh.deploy("DroneModel", DroneModel, globals(), True)
def test_deployment(self):
yh = Yhat("foo", "bar", "http://api.yhathq.com/")
_, bundle = yh.deploy("HelloWorld", HelloWorld, globals(), dry_run=True)
self.assertTrue(True)
#!/usr/bin/env python
from flask import Flask, request, render_template, url_for, Response, json
from yhat import Yhat
import os
app = Flask(__name__)
yh = Yhat(os.environ.get("YHAT_USERNAME"), os.environ.get("YHAT_APIKEY"), os.environ.get("YHAT_URL"))
@app.route('/', methods=['GET', 'POST'])
def index():
if request.method == 'POST':
# print request.json['beers']
try:
pred = yh.predict("BeerRecommender", {"beers": request.json['beers'],
"n": request.json['n']})
return Response(json.dumps(pred), mimetype='application/json')
except Exception, e:
print e
return Response(json.dumps({"error": str(e)}),
mimetype='application/json')
else:
# static files
css_url = url_for('static', filename='css/main.css')
jquery_url = url_for('static', filename='js/jquery-1.10.2.min.js')
beers_url = url_for('static', filename='js/beers.js')
highlight_url = url_for('static', filename='js/code.js')
js_url = url_for('static', filename='js/main.js')
return render_template('index.html', css_url=css_url,
jquery_url=jquery_url, beers_url=beers_url,
js_url=js_url, highlight_url=highlight_url)
{
's1':1, 's2':1, 's3':1, 's4':1, 's5':1,
'w1':1, 'w2':1, 'w3':1, 'w4':1,
'k1':1, 'k2':1, 'k3':1, 'k4':1, 'k5':1,
'k6':1, 'k7':1, 'k8':1, 'k9':1, 'k10':1,
'k11':1, 'k12':1, 'k13':1, 'k14':1, 'k15':1
}
test_data = pd.read_csv(open('data/test.csv', 'r'), quotechar='"')
sub_data = pd.read_csv(open('data/sampleSubmission.csv', 'r'), quotechar='"')
if not np.alltrue(test_data['id'] == sub_data['id']):
raise Exception("IDs do not match")
yh = Yhat(username, apikey)
variabless = sub_data.columns[1:]
raw_tweets = test_data['tweet'].tolist()
for variable in variables:
model_version = best_model[variable]
model_name = "TweetClassifier_%s" % (variable, )
results_from_server = yh.raw_predict(model_name, model_version, raw_tweets)
pred = results_from_server['prediction']['scores']
sub_data[variable] = pred
try:
sub_data.to_csv(open(sub_file, 'w'), index=False)
except IOError:
sys.stderr.write("IO error: could not write data to file")
from yhat import Yhat, YhatModel, preprocess
import os
USERNAME = os.environ["USERNAME"]
APIKEY = os.environ["APIKEY"]
URL = os.environ["URL"]
class HelloWorld(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
me = data['name']
greeting = "Hello " + str(me) + "!"
return {"greeting": greeting}
yh = Yhat(USERNAME, APIKEY, URL)
yh.deploy("Gitmodel", HelloWorld, globals(), True)
features = [{"name": "x", "na_filler": 0}, {"name": "z", "na_filler": fill_z}]
class MyOtherClass:
def hello(self, x):
return "hello: %s" % str(x)
REQS = open("reqs.txt").read()
### <DEPLOYMENT START> ###
# @preprocess(in_type=dict, out_type=pd.DataFrame, null_handler=features)
class MyModel(YhatModel):
REQUIREMENTS = REQS
@preprocess(out_type=pd.DataFrame)
def execute(self, data):
return predict(data)
# "push" to server would be here
data = {"x": 1, "z": None}
if __name__ == '__main__':
creds = credentials.read()
yh = Yhat(creds['username'], creds['apikey'], "http://localhost:3000/")
yh.deploy("mynewmodel", MyModel, globals())
from yhat import YhatModel, Yhat, preprocess
# from first import hello as h2
import first as f2
from first import Support
from another.testfile import bye
def goodbye(y):
bye()
print y, "goodbye!"
class Example(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
goodbye(x)
return Support().hello(10)
# return h2(data)
from first import x
yh = Yhat("greg", "fCVZiLJhS95cnxOrsp5e2VSkk0GfypZqeRCntTD1nHA",
"http://api.yhathq.com/")
yh.deploy_to_file("Example", Example, globals())
data = pd.DataFrame(data)
data = data[features]
prob = glm.predict_proba(data)[0][1]
if prob > 0.3:
decline_code = "Credit score too low"
else:
decline_code = ""
odds = glm.predict_log_proba(data)[0][1]
score = calculate_score(odds)
output = {
"prob_default": [prob],
"decline_code": [decline_code],
"score": [score]
}
return output
df_term[features].head()
test = {
"last_fico_range_low": 705,
"last_fico_range_high": 732,
"home_ownership": "MORTGAGE"
}
LoanModel().execute(test)
yh = Yhat("austin", os.environ.get("SCIENCEOPS_API_KEY"), "https://sandbox.c.yhat.com/")
yh.deploy("LendingClub", LoanModel, globals(), True)
class ChurnModel(YhatModel):
# Type casts incoming data as a dataframe
@preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame)
def execute(self, data):
# Collect customer meta data
response = data[['Area Code', 'Phone']]
charges = ['Day Charge', 'Eve Charge', 'Night Charge', 'Intl Charge']
response['customer_worth'] = data[charges].sum(axis=1)
# Convert yes no columns to bool
data[yes_no_cols] = data[yes_no_cols] == 'yes'
# Create feature space
X = data[features].as_matrix().astype(float)
X = scaler.transform(X)
# Make prediction
churn_prob = clf.predict_proba(X)
response['churn_prob'] = churn_prob[:, 1]
# Calculate expected loss by churn
response['expected_loss'] = response['churn_prob'] * response[
'customer_worth']
response = response.sort('expected_loss', ascending=False)
# Return response DataFrame
return response
yh = Yhat("e[at]yhathq.com", " MY APIKEY ", "http://cloud.yhathq.com/")
print "Deploying model"
response = yh.deploy("PythonChurnModel", ChurnModel, globals())
print json.dumps(response, indent=2)
def parse_tweet(tweet):
trees = nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(tweet)))
for tree in trees.subtrees():
etype = None
if tree.node == "PERSON":
etype = "PERSON"
elif tree.node == "GPE":
etype = "PLACE"
if etype is not None:
ne = " ".join([leaf[0] for leaf in tree.leaves()])
tweet = tweet.replace(ne,
"<" + etype + ">" + ne + "</" + etype + ">")
return tweet
class Tagger(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, raw):
tweet = raw['text']
tagged = parse_tweet(tweet)
raw['tagged'] = tagged
return raw
tg = Tagger()
yh = Yhat("greg", "mykey", YHAT_URL)
print yh.deploy("NamedEntityTagger" + str(i), Tagger, globals())
from yhat import Yhat, YhatModel, preprocess
x = range(10)
class HelloWorld(YhatModel):
@preprocess(in_type=dict, out_type=dict)
def execute(self, data):
print x[:10]
me = data['name']
greeting = "Hello " + str(me) + "!"
return {"greeting": greeting, "x": x}
# yh = Yhat("greg", "fCVZiLJhS95cnxOrsp5e2VSkk0GfypZqeRCntTD1nHA", "http://cloud.yhathq.com/")
yh = Yhat("greg", "9207b9a2dd9d48848b139b729d4354bc", "http://localhost:8080/")
yh.deploy("NewZippedModel", HelloWorld, globals())
class ChurnModel(YhatModel):
@preprocess(in_type=pd.DataFrame, out_type=pd.DataFrame)
def execute(self, data):
response = pd.DataFrame(data)
charges = ['day_charge', 'eve_charge', 'night_charge', 'intl_charge']
response['customer_worth'] = data[charges].sum(axis=1)
# Convert yes no columns to bool
data[yes_no_cols] = data[yes_no_cols] == 'yes'
# Create feature space
X = data[features].as_matrix().astype(float)
X = scaler.transform(X)
# Make prediction
churn_prob = clf.predict_proba(X)
response['churn_prob'] = churn_prob[:, 1]
# Calculate expected loss by churn
response['expected_loss'] = response['churn_prob'] * response[
'customer_worth']
response = response.sort('expected_loss', ascending=False)
response = response[['customer_worth', 'churn_prob', 'expected_loss']]
# Return response DataFrame
return response
yh = Yhat(raw_input("Yhat username: "******"Yhat apikey: "),
"http://sandbox.yhathq.com/")
print "Deploying model"
response = yh.deploy("PythonChurnModel", ChurnModel, globals())
print df_to_json(churn_df[:1])
