def test_init(self):
client = Client("http://example.com:9004")
self.assertEqual(client._endpoint, "http://example.com:9004")
client = Client("http://example.com/", 10.0)
self.assertEqual(client._endpoint, "http://example.com/")
client = Client(endpoint="https://example.com/", query_timeout=-10.0)
self.assertEqual(client._endpoint, "https://example.com/")
self.assertEqual(client.query_timeout, 0.0)
# valid name tests
with self.assertRaises(ValueError):
Client('')
with self.assertRaises(TypeError):
Client(1.0)
with self.assertRaises(ValueError):
Client("*#")
with self.assertRaises(TypeError):
Client()
with self.assertRaises(ValueError):
Client("http:/www.example.com/")
with self.assertRaises(ValueError):
Client("httpx://www.example.com:9004")
def main():
client = ipp.Client(profile="default")
dview = client[:]
dview.execute("import bodo")
dview.execute("import numpy as np")
dview.execute("import pandas as pd")
dview.execute("import os")
dview.execute(inspect.getsource(lr_from_sql))
tp_client = Client('http://localhost:8080/')
tp_client.deploy('lr_snowflake',
lr_snowflake,
'Logistic regression from Snowflake table',
override=True)
def clean_text(input_df):
'''
This function create preprocessed PAR and output the new dataframe.
Called in Tableau Prep
Args:
------
whole dataframe from Tableau
Returns:
--------
Returns processed pandas dataframe
'''
client = Client("http://10.155.94.140:9004/")
processed = client.query('clean_text',
input_df['X_PAR_COMMENTS'].tolist())['response']
input_df['PROCESSED_PAR'] = processed
output_df = input_df
# return the entire df
return output_df
def deploy_model(funcName, func, funcDescription):
# running from deploy_models.py
config_file_path = sys.argv[1] if len(
sys.argv) > 1 else get_default_config_file_path()
port, auth_on, prefix = parse_config(config_file_path)
connection = Client(f"{prefix}://localhost:{port}/")
if auth_on:
# credentials are passed in from setup.py
user, passwd = sys.argv[2], sys.argv[3] if len(
sys.argv) == 4 else get_creds()
connection.set_credentials(user, passwd)
connection.deploy(funcName, func, funcDescription, override=True)
print(f"Successfully deployed {funcName}")
def deploy_model(funcName, func, funcDescription):
# running from deploy_models.py
if len(sys.argv) > 1:
config_file_path = sys.argv[1]
else:
config_file_path = get_default_config_file_path()
port, auth_on, prefix = parse_config(config_file_path)
connection = Client(f'{prefix}://localhost:{port}/')
if auth_on:
# credentials are passed in from setup.py
if len(sys.argv) == 4:
user, passwd = sys.argv[2], sys.argv[3]
# running Sentiment Analysis independently
else:
user, passwd = get_creds()
connection.set_credentials(user, passwd)
connection.deploy(funcName, func, funcDescription, override=True)
print(f'Successfully deployed {funcName}')
from tabpy.tabpy_tools.client import Client
client = Client('http://localhost:9004/')
client.set_credentials('kikin', 'karate10')
# Deploying a Function
# * Add Function
def add(x, y):
import numpy as np
return np.add(x, y).tolist()
client.deploy('add', add, 'Adds two numbers x and y', override=True)
# Import tabpy client for deployment
from tabpy.tabpy_tools.client import Client
# Server URL (This would be the host and port on which you are running the TabPy server)
client = Client('http://localhost:9004/')
# Define the function tested above
def clustering(x, y):
import numpy as np
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
X = np.column_stack([x, y])
X = StandardScaler().fit_transform(X)
db = DBSCAN(eps=0.3, min_samples=3).fit(X)
return db.labels_.tolist()
# Deploy the model to TabPy server
# Add Override = True if you are deploying the model again
client.deploy(
'clustering', clustering,
'Returns cluster Ids for each data point specified by the pairs in x and y'
)
"""
Check if the model is model is deployed on the TabPy server at the URL below:
Server URL (This would be the host and port on which you are running the TabPy server):
http://localhost:9004/endpoints
"""
# Sample Data
x = [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10]
from tabpy.tabpy_tools.client import Client
client = Client('http://localhost:8080/')
def clustering(x, y):
import numpy as np
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
X = np.column_stack([x, y])
X = StandardScaler().fit_transform(X)
db = DBSCAN(eps=1, min_samples=3).fit(X)
return db.labels_.tolist()
if __name__ == "__main__":
client.deploy(
'clustering',
clustering,
'Returns cluster Ids for each data point specified by the '
'pairs in x and y'
)
#!/usr/bin/env python
# coding: utf-8
import pandas as pd
from scipy.spatial import KDTree
from tabpy.tabpy_tools.client import Client
connection = Client('http://localhost:9004/')
def rec(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11,
arg12, arg13, arg14, arg15, arg16):
_arg1 = arg1[0]
_arg2 = arg2[0]
_arg3 = arg3[0]
_arg4 = arg4[0]
_arg5 = arg5[0]
_arg6 = arg6[0]
_arg7 = arg7[0]
_arg8 = arg8[0]
_arg9 = arg9[0]
_arg10 = arg10[0]
_arg11 = arg11[0]
_arg12 = arg12[0]
_arg13 = arg13[0]
_arg14 = arg14[0]
_arg15 = arg15[0]
_arg16 = arg16[0]
data = pd.read_csv('data_cleaned_and_pruned.csv', delimiter="|")
df = pd.concat([
df_mode, df_customer_category, df_region, df_category, df_quantity,
df_ratio
],
axis=1)
print(df)
df_dummy = pd.get_dummies(df)
# 作成したモデルを利用して予測値を取得
y_pred = clf.predict_proba(df_dummy)
# 予測結果をリストとして返す
return y_pred[:, 1].tolist()
if __name__ == '__main__':
from tabpy.tabpy_tools.client import Client
client = Client('http://localhost:9004/')
_desision_tree()
client.deploy('sub', sub, 'Simple subtraction function.', override=True)
client.deploy('clustering',
clustering,
'Returns cluster Ids for each data point specified by the '
'pairs in x and y',
override=True)
client.deploy('decision_tree',
decision_tree,
'decision_tree',
override=True)
def setUp(self):
self.client = Client("http://example.com/")
self.client._service = Mock() # TODO: should spec this
class TestClient(unittest.TestCase):
def setUp(self):
self.client = Client("http://example.com/")
self.client._service = Mock() # TODO: should spec this
def test_init(self):
client = Client("http://example.com:9004")
self.assertEqual(client._endpoint, "http://example.com:9004")
client = Client("http://example.com/", 10.0)
self.assertEqual(client._endpoint, "http://example.com/")
client = Client(endpoint="https://example.com/", query_timeout=-10.0)
self.assertEqual(client._endpoint, "https://example.com/")
self.assertEqual(client.query_timeout, 0.0)
# valid name tests
with self.assertRaises(ValueError):
Client('')
with self.assertRaises(TypeError):
Client(1.0)
with self.assertRaises(ValueError):
Client("*#")
with self.assertRaises(TypeError):
Client()
with self.assertRaises(ValueError):
Client("http:/www.example.com/")
with self.assertRaises(ValueError):
Client("httpx://www.example.com:9004")
def test_get_status(self):
self.client._service.get_status.return_value = "asdf"
self.assertEqual(self.client.get_status(), "asdf")
def test_query_timeout(self):
self.client.query_timeout = 5.0
self.assertEqual(self.client.query_timeout, 5.0)
self.assertEqual(self.client._service.query_timeout, 5.0)
def test_query(self):
self.client._service.query.return_value = "ok"
self.assertEqual(self.client.query("foo", 1, 2, 3), "ok")
self.client._service.query.assert_called_once_with("foo", 1, 2, 3)
self.client._service.query.reset_mock()
self.assertEqual(self.client.query("foo", a=1, b=2, c=3), "ok")
self.client._service.query.assert_called_once_with("foo",
a=1,
b=2,
c=3)
def test_get_endpoints(self):
self.client._service.get_endpoints.return_value = "foo"
self.assertEqual(self.client.get_endpoints("foo"), "foo")
self.client._service.get_endpoints.assert_called_once_with("foo")
def test_get_endpoint_upload_destination(self):
self.client._service.get_endpoint_upload_destination.return_value = \
{"path": "foo"}
self.assertEqual(self.client._get_endpoint_upload_destination(), "foo")
def test_set_credentials(self):
username, password = "******", "password"
self.client.set_credentials(username, password)
self.client._service.set_credentials.assert_called_once_with(
username, password)
class TestClient(unittest.TestCase):
def setUp(self):
self.client = Client("http://example.com/")
self.client._service = Mock() # TODO: should spec this
def test_init(self):
client = Client("http://example.com:9004")
self.assertEqual(client._endpoint, "http://example.com:9004")
client = Client("http://example.com/", 10.0)
self.assertEqual(client._endpoint, "http://example.com/")
client = Client(endpoint="https://example.com/", query_timeout=-10.0)
self.assertEqual(client._endpoint, "https://example.com/")
self.assertEqual(client.query_timeout, 0.0)
# valid name tests
with self.assertRaises(ValueError):
Client('')
with self.assertRaises(TypeError):
Client(1.0)
with self.assertRaises(ValueError):
Client("*#")
with self.assertRaises(TypeError):
Client()
with self.assertRaises(ValueError):
Client("http:/www.example.com/")
with self.assertRaises(ValueError):
Client("httpx://www.example.com:9004")
def test_get_status(self):
self.client._service.get_status.return_value = "asdf"
self.assertEqual(self.client.get_status(), "asdf")
def test_query_timeout(self):
self.client.query_timeout = 5.0
self.assertEqual(self.client.query_timeout, 5.0)
self.assertEqual(self.client._service.query_timeout, 5.0)
def test_query(self):
self.client._service.query.return_value = "ok"
self.assertEqual(self.client.query("foo", 1, 2, 3), "ok")
self.client._service.query.assert_called_once_with("foo", 1, 2, 3)
self.client._service.query.reset_mock()
self.assertEqual(self.client.query("foo", a=1, b=2, c=3), "ok")
self.client._service.query.assert_called_once_with("foo",
a=1,
b=2,
c=3)
def test_get_endpoints(self):
self.client._service.get_endpoints.return_value = "foo"
self.assertEqual(self.client.get_endpoints("foo"), "foo")
self.client._service.get_endpoints.assert_called_once_with("foo")
def test_get_endpoint_upload_destination(self):
self.client._service.get_endpoint_upload_destination.return_value = \
{"path": "foo"}
self.assertEqual(self.client._get_endpoint_upload_destination(), "foo")
def test_set_credentials(self):
username, password = "******", "password"
self.client.set_credentials(username, password)
self.client._service.set_credentials.assert_called_once_with(
username, password)
def test_check_invalid_endpoint_name(self):
endpoint_name = 'Invalid:model:@name'
with self.assertRaises(ValueError) as err:
_check_endpoint_name(endpoint_name)
self.assertEqual(
err.exception.args[0],
f'endpoint name {endpoint_name } can only contain: '
'a-z, A-Z, 0-9, underscore, hyphens and spaces.')
# Import packages
from tabpy.tabpy_tools.client import Client
client = Client("http://10.155.94.140:9004/")
def clean_text(text_list):
"""
Clean text with spacy library
"""
import pandas as pd
import numpy as np
import spacy
# configure stopword path
stopword_path = "/home/nusintern/project/nus/scripts/stopwords.txt"
# Import language model
nlp = spacy.load('en_core_web_sm', disable=['tagger','parser', 'ner'])
#Load custom stopwords
custom_stopwords = []
with open(stopword_path) as f:
custom_stopwords = f.read().splitlines()
custom_stopwords = set(custom_stopwords)
return [process_text(x, nlp, custom_stopwords) for x in text_list]
def process_text(text, nlp, stopwords):
'''
This function performs text data preprocessing, including tokenizing the text, converting text to lower case, removing
punctuation, removing digits, removing stop words, stemming the tokens, then converting the tokens back to strings.
