def main():
from tatk.pipelines.text_classification.text_classifier import TextClassifier
from sklearn.linear_model import LogisticRegression
import pandas
init()
df = pandas.DataFrame(data=[['please add your good text here.']],
columns=['tweets'])
#input1 = pandas.DataFrame(data=[['please add your good text here.']], columns=['tweets'])
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Generate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='service-schema.json')
print("Schema generated")
def main():
# Generating random 28x28 pixels to use as sample input
sample_input = (np.random.rand(28, 28, 1) * 255) #.astype('uint8')
sample_input = sample_input.reshape(1, 28, 28,
1) # Reshaping to match training data
# Calling init() and run()
init()
inputs = {"input_array": SampleDefinition(DataTypes.NUMPY, sample_input)}
result_string = run(sample_input)
print("resultString = " + str(result_string))
# Generating the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='outputs/schema.json')
print('Schema generated')
def main():
# Test the init and run functions using test data
init()
test_doc_text = "🍪"
category = run(test_doc_text)
print(category)
test_doc_text = "tomato pizza"
# Generate the schema file (schema.json) needed for AML operationalization
inputs = {"doc_text": SampleDefinition(DataTypes.STANDARD, test_doc_text)}
generate_schema(run_func=run,
inputs=inputs,
filepath='./outputs/schema.json')
block_blob_service.create_blob_from_path('embeddings', 'schema.json',
'./outputs/schema.json')
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
sentences = ['This is a sentence.', 'This is another one.']
# Test the output of the functions
init()
result = run(sentences)
inputs = {"sentences": SampleDefinition(DataTypes.STANDARD, sentences)}
#Generate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='infersent_service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(data=[[190, 60, 38]], columns=['height', 'width', 'shoe_size'])
# Test the functions' output
init()
input1 = pandas.DataFrame([[190, 60, 38]])
print("Result: " + run(input1))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
# Generate the service_schema.json
generate_schema(run_func=run, inputs=inputs, filepath='service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import numpy as np
text_entry = 'This is a sentence. This is an awesome sentence!'
# Test the output of the functions
init()
print("Result: " + run(text_entry))
inputs = {"text_entry": SampleDefinition(DataTypes.STANDARD, text_entry)}
#Generate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='./outputs/cnn_service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
inputs = {
"input_df": SampleDefinition(DataTypes.PANDAS, yourinputdataframe)
}
generate_schema(run_func=run,
inputs=inputs,
filepath='service_schema.json')
print("Schema generated")
if __name__ == "__main__":
main()
def generate_api_schema():
import os
print("create schema")
d = {
'num-of-doors': [4],
'fuel-type': [1],
'width': [68.9],
'height': [55.5],
'num-of-cylinders': [6],
'engine-type': [0],
'horsepower': [106]
}
df = pd.DataFrame(data=d)
input = df.to_json()
inputs = {"input": SampleDefinition(DataTypes.STANDARD, input)}
os.makedirs('outputs', exist_ok=True)
print(
generate_schema(inputs=inputs,
filepath="outputs/schema.json",
run_func=run))
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(data=["What a waste of time and money! The story was not realistic at all! Actually it was completely far fetched!"], columns=['Text'])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
input1 = pandas.DataFrame(data=["What a waste of time and money! The story was not realistic at all! Actually it was completely far fetched!"], columns=['Text'])
print("The input {0} created the following output: {1}".format(input1['Text'], run(input1)))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Genereate the schema
generate_schema(run_func=run, inputs=inputs, filepath='./outputs/service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(data=[[380, 120, 76]],
columns=['indicator1', 'NF1', 'cellprofiling'])
# Check the output of the function
init()
input1 = pandas.DataFrame([[380, 120, 76]])
print("Result: " + run(input1))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
# Generate the service_schema.json
generate_schema(run_func=run,
inputs=inputs,
filepath='output/service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(data=[[1,1,1,1,1,1,1,1,1,1]], columns=['MAX_ACCX', 'MAX_ACCY', 'MAX_ACCZ', 'MAX_GYROX', 'MAX_GYROY', 'MAX_GYROZ', 'ACC_AVG', 'ACC_VAR', 'GYRO_AVG', 'GYRO_VAR'])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
input1 = pandas.DataFrame([[1,1,1,1,1,1,1,1,1,1]])
print("The input {0} created the following output: {1}".format(input1.values, run(input1)))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Genereate the schema
generate_schema(run_func=run, inputs=inputs, filepath='./outputs/service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(data=[[3.0, 3.6, 1.3, 0.25]], columns=['sepal length', 'sepal width','petal length','petal width'])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
input1 = pandas.DataFrame([[3.0, 3.6, 1.3, 0.25]])
print("Result: " + run(input1))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Genereate the schema
generate_schema(run_func=run, inputs=inputs, filepath='./outputs/service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
df = pandas.DataFrame(
data=[[3.0, 3.6, 1.3, 0.25]],
columns=['sepal length', 'sepal width', 'petal length', 'petal width'])
# Test the output of the functions
init()
input1 = pandas.DataFrame([[3.0, 3.6, 1.3, 0.25]])
print("Result: " + run(input1))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Genereate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='service_schema.json')
print("Schema generated")
def create_schema():
# This is used to define the schema and example
df = pandas.DataFrame(data=[[10, 5]], columns=["x1", "x2"])
init()
# This is used to execute the run to get the schema of the prediction
input1 = pandas.DataFrame(data=[[10, 5]])
run(input1)
print(os.getcwd())
print(df)
print(input1)
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
# The prepare statement writes the scoring file (main.py) and
# the scchema file (service_schema.json) the the output folder.
#prepare(run_func=run, init_func=init, input_types=inputs, )
generate_schema(run_func=run,
inputs=inputs,
filepath='./outputs/service_schema.json')
print("Schema generated")
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import numpy as np
df = np.array([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
input1 = np.array([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]])
print("Result: " + run(input1))
inputs = {"input_df": SampleDefinition(DataTypes.NUMPY, df)}
# Genereate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='./outputs/sgd_automated_learn_v2.json')
print("Schema generated")
def main():
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'false'
os.environ["AML_MODEL_DC_STORAGE_ENABLED"] = 'false'
# create the outputs folder
os.makedirs('./outputs', exist_ok=True)
# Read in json, mod bus sample msg
input_msg = read_msg()
# Debugging - remove when deploying
#print (" ");
#print ("Input Json:")
#print (input_msg);
# Test init function
init()
# Write out json, sample response msg
output_msg = run(input_msg)
# Debugging - remove when deploying
#print (" ");
#print ("Output Json:")
print(output_msg)
# Sample input string
input_str = {
"input_str": SampleDefinition(DataTypes.STANDARD, input_msg)
}
# Generate swagger document for web service
generate_schema(run_func=run,
inputs=input_str,
filepath='./outputs/service_schema.json')
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
# Anomaly
df = pandas.DataFrame(data=[[33.66995566, 2.44341267, 21.39450979, 26]], columns=['machine_temperature', \
'machine_pressure','ambient_temperature','ambient_humidity'])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
# Anomaly
#input1 = '{ "machine": { "temperature": 33.66995566, "pressure": 2.44341267 }, \
# "ambient": { "temperature": 21.39450979, "humidity": 26 },\
# "timeCreated": "2017-10-27T18:14:02.4911177Z" }'
# Normal
#input1 = '{ "machine": { "temperature": 31.16469009, "pressure": 2.158002669 }, \
# "ambient": { "temperature": 21.17794693, "humidity": 25 },\
# "timeCreated": "2017-10-27T18:14:02.4911177Z" }'
input2 = pandas.DataFrame(data=[[31.16469009, 2.158002669, 21.17794693, 25]], columns=['machine_temperature', \
'machine_pressure','ambient_temperature','ambient_humidity'])
print("Result: " + str(run(input2)))
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, df)}
generate_schema(run_func=run,
inputs=inputs,
filepath='./outputs/service_schema.json')
print("Schema generated")
return_str = str_webmap_count + str(pred[0])
return json.dumps(return_str)
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
import pandas
from arcgis_search import search_fire
df = pandas.DataFrame(data=[[3.0, 3.6, 1.3, 0.25]], columns=['sepal length', 'sepal width','petal length','petal width'])
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
input1 = 'forest'
input2 = pandas.DataFrame([[3.7, 2.6, 1.3, 0.25]])
print("Result: " + run(input1, input2))
inputs = {'input_str':SampleDefinition(DataTypes.STRING, input1),
"input_df": SampleDefinition(DataTypes.PANDAS, df)}
#Genereate the schema
generate_schema(run_func=run, inputs=inputs, filepath='./outputs/service_schema.json')
print("Schema generated")
if __name__ == "__main__":
main()
# Implement test code to run in IDE or Azure ML Workbench
if __name__ == '__main__':
# Import the logger only for Workbench runs
from azureml.logging import get_azureml_logger
import numpy as np
import pandas as pd
columns = ['TimeSpentOnWeb', 'TimeSpentOnProductPage']
sample_input = pd.DataFrame(data=[(5.1, 3.5)], columns=columns)
init()
#input = "{}"
result = run(sample_input)
print("The predicted Product which will be Purchased is -", str(result))
##Generating Schema
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, sample_input)}
print(
generate_schema(inputs=inputs,
filepath="service-schema.json",
run_func=run))
logger = get_azureml_logger()
logger.log("Result", result)
## To use your existing model management account
## az ml account modelmanagement set -n dsazmlexpmodelmgmt -g datascience
## Setup a NEW Model management
##az ml account modelmanagement create -l [Azure region, e.g. eastus2] -n [your account name] -g [resource group name] --sku-instances [number of instances, e.g. 1] --sku-name [Pricing tier for example S1]
## Next setup your local or cluster mode deployment
## az ml env setup -l eastus2 -n dslocal -g datascience
## az ml env show -l eastus2 -n dslocal -g datascience
from sklearn.externals import joblib
global model
model = joblib.load(RandomForest_model_file)
def run(input_df):
import json
input = input_df.as_matrix()
try:
pred = model.predict(input)
return json.dumps(str(pred[0]))
except Exception as e:
return (str(e))
####################
# Main function
####################
if __name__ == '__main__':
init()
X_str = '[{"capital_loss":-0.219095674,"hours_per_week":0.7559573744,"education_num":-0.4507068474,"capital_gain":-0.1480462751,"age":-0.0311032178,"Federal-gov":0.0,"Local-gov":0.0,"Private":1.0,"Self-emp-inc":0.0,"Self-emp-not-inc":0.0,"State-gov":0.0,"Without-pay":0.0,"Divorced":0.0,"Married-AF-spouse":0.0,"Married-civ-spouse":1.0,"Married-spouse-absent":0.0,"Never-married":0.0,"Separated":0.0,"Widowed":0.0,"Adm-clerical":0.0,"Armed-Forces":0.0,"Craft-repair":0.0,"Exec-managerial":0.0,"Farming-fishing":1.0,"Handlers-cleaners":0.0,"Machine-op-inspct":0.0,"Other-service":0.0,"Priv-house-serv":0.0,"Prof-specialty":0.0,"Protective-serv":0.0,"Sales":0.0,"Tech-support":0.0,"Transport-moving":0.0,"Husband":1.0,"Not-in-family":0.0,"Other-relative":0.0,"Own-child":0.0,"Unmarried":0.0,"Wife":0.0,"Amer-Indian-Eskimo":0.0,"Asian-Pac-Islander":0.0,"Black":0.0,"Other":0.0,"White":1.0,"Female":0.0,"Male":1.0,"Canada":0.0,"El-Salvador":0.0,"Germany":0.0,"Mexico":0.0,"Philippines":0.0,"Puerto-Rico":0.0,"United-States":1.0}]'
X_test = pandas.read_json(X_str)
# Get predictions
y_pred = run(X_test)
inputs = {"input_df": SampleDefinition(DataTypes.PANDAS, X_test)}
# The prepare statement writes the scoring file (main.py) and
# the schema file (service_schema.json) the the output folder.
generate_schema(run_func=run, inputs=inputs, filepath = 'service_schema.json')
logger = get_azureml_logger()
logger.log("amlrealworld.uciincome.score", "true")
def load_image_into_numpy_array(image):
(im_width, im_height) = image.size
return np.array(image.getdata()).reshape(
(im_height, im_width, 3)).astype(np.uint8)
# Implement test code to run in IDE or Azure ML Workbench
if __name__ == '__main__':
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
# Import the logger only for Workbench runs
#from azureml.logging import get_azureml_logger
#logger = get_azureml_logger()
init()
pilImg = Image.open("yourimage.jpg")
base64ImgString = pilImgToBase64(pilImg)
np_imgstring = np.array([base64ImgString], dtype=np.unicode)
inputs = {"input_array": SampleDefinition(DataTypes.NUMPY, np_imgstring)}
resultString = run(np_imgstring)
print("resultString = " + str(resultString))
# Genereate the schema
generate_schema(run_func=run,
inputs=inputs,
filepath='service_schema.json')
print("Schema generated.")
f.close()
#========================= CREATE WEB SERVICE SCHEMA =========================
def run(inputData):
import json
prediction = classifier.predict(input_df)
prediction = "%s %d" % (str(input_df), classifier)
return json.dumps(str(prediction))
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
inputs = {"inputData": SampleDefinition(DataTypes.NUMPY, X_test)}
print(generate_schema(run_func=run, inputs=inputs, filepath="./outputs/schema.json"))
'''
#========================= VISUALISING THE RESULTS =========================
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
# Visualising the Training set results
X_set, y_set = X_train, y_train
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape),
alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
# Plot ROC curve
localPrediction = prediction.select(' income',
'scored_probabilities').toPandas()
y_true = localPrediction[' income'] == ' >50K'
y_pred = [elem[1] for elem in localPrediction['scored_probabilities']]
plot_roc(y_true, y_pred)
print("******** SAVE THE MODEL ***********")
model.write().overwrite().save("./outputs/AdultCensus.mml")
# save model in wasb if running in HDI.
#model.write().overwrite().save("wasb:///models/AdultCensus.mml")
# create web service schema
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
# Define the input dataframe
sample = spark.createDataFrame(
[('10th', 'Married-civ-spouse', 35.0)],
[' education', ' marital-status', ' hours-per-week'])
inputs = {"input_df": SampleDefinition(DataTypes.SPARK, sample)}
# Create the schema file (service_schema.json) the the output folder.
import score_mmlspark
generate_schema(run_func=score_mmlspark.run,
inputs=inputs,
filepath='./outputs/service_schema.json')
def main():
from azureml.api.schema.dataTypes import DataTypes
from azureml.api.schema.sampleDefinition import SampleDefinition
from azureml.api.realtime.services import generate_schema
print('Entered main function:')
print(os.getcwd())
amlWBSharedDir = os.environ['AZUREML_NATIVE_SHARE_DIRECTORY']
print(amlWBSharedDir)
def get_files_in_dir(crt_dir):
return( [f for f in os.listdir(crt_dir) if os.path.isfile(os.path.join(crt_dir, f))])
fully_trained_weights_dir=os.path.join(
amlWBSharedDir,
os.path.join(*(['chestxray', 'output', 'trained_models_weights'])))
crt_models = get_files_in_dir(fully_trained_weights_dir)
print(fully_trained_weights_dir)
print(crt_models)
test_images_dir=os.path.join(
amlWBSharedDir,
os.path.join(*(['chestxray', 'data', 'ChestX-ray8', 'test_images'])))
test_images = get_files_in_dir(test_images_dir)
print(test_images_dir)
print(len(test_images))
# score in local mode (i.e. here in main function)
model = azure_chestxray_utils.build_DenseNetImageNet201_model()
model.load_weights(os.path.join(
fully_trained_weights_dir, densenet_weights_file_name))
print('Model weoghts loaded!')
import cv2
cv2_image = cv2.imread(os.path.join(test_images_dir,test_images[0]))
x, serialized_cam_image = get_image_score_and_serialized_cam(cv2_image, model)
file_bytes = np.asarray(bytearray(serialized_cam_image.read()), dtype=np.uint8)
recovered_image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
# x = model.predict(cv2_image[None,:,:,:])
print(test_images[0])
print(x)
print(recovered_image.shape)
# score in local mode (i.e. here in main function) using encoded data
encoded_image = as_string_b64encoded_pickled(cv2_image)
df_for_api = pd.DataFrame(data=[[encoded_image]], columns=[as_string_b64encoded_pickled_data_column_name])
del encoded_image
del cv2_image
del serialized_cam_image
input_df = df_for_api[as_string_b64encoded_pickled_data_column_name][0]
input_cv2_image = unpickled_b64decoded_as_bytes(input_df);
x, serialized_cam_image = get_image_score_and_serialized_cam(input_cv2_image, model)
file_bytes = np.asarray(bytearray(serialized_cam_image.read()), dtype=np.uint8)
recovered_image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
# x = model.predict(input_cv2_image[None,:,:,:])
print('After encoding and decoding:')
print(x)
print(recovered_image.shape)
del model
# now create the post deployment env, i.e. score using init() and run()
crt_dir = os.getcwd()
working_dir = os.path.join(crt_dir, 'tmp_cam_deploy')
if not os.path.exists(working_dir):
os.makedirs(working_dir)
import shutil
shutil.copyfile(
os.path.join( fully_trained_weights_dir,densenet_weights_file_name),
os.path.join( working_dir,densenet_weights_file_name))
os.chdir(working_dir)
# Turn on data collection debug mode to view output in stdout
os.environ["AML_MODEL_DC_DEBUG"] = 'true'
# Test the output of the functions
init()
print("Result: " + run(df_for_api))
# #Generate the schema
data_for_schema = {"input_df": SampleDefinition(DataTypes.PANDAS, df_for_api)}
schema_file = os.path.join(fully_trained_weights_dir, 'chest_XRay_cam_service_schema.json')
generate_schema(run_func=run, inputs=data_for_schema, filepath=schema_file)
print("Schema saved in " +schema_file)
input_df = spark.createDataFrame(
[["this grant will provide funding for biostatistics and neuroscience"],
["this is an example abstract"]],
schema=sch)
print(run(input_df))
# COMMAND ----------
# MAGIC %md
# MAGIC ### Write the Schema to Blob
# COMMAND ----------
# define the input data frame
inputs = {
"input_df": SampleDefinition(DataTypes.SPARK,
input_df.select(input_features))
}
# The Generate_Schema will attempt to write the file to the databricks cluster.
# But we will take the results and write it to our blob storage account defined in the `train_on_dbr.py` file.
json_schema = generate_schema(run_func=run,
inputs=inputs,
filepath='service_schema.json')
with open("/dbfs/mnt/misc/service_schema.json", 'w') as f:
f.write(json.dumps(json_schema))
# COMMAND ----------
# Take a look at the schema
print(json.dumps(json_schema, indent=2))
