def set_username():
FLAGS.username = getpass.getuser()
users_dataset = dsx_core_utils.get_remote_data_set_info(FLAGS.users_dataset)
users_df = load_dataset(users_dataset)
for i in range(0, len(users_df)):
if str(users_df["USER_ID"][i]) == FLAGS.username:
# print(users_df.iloc[i])
FLAGS.username = extract_username(users_df["USERNAME"][i])
FLAGS.name = extract_username(users_df["NAME"][i])
def init_steps():
global dataSet
global dataSource
global conn
dataSet = dsx_core_utils.get_remote_data_set_info('submissions')
dataSource = dsx_core_utils.get_data_source_info(dataSet['datasource'])
if (sys.version_info >= (3, 0)):
conn = jaydebeapi.connect(dataSource['driver_class'], dataSource['URL'], [dataSource['user'], dataSource['password']])
else:
conn = jaydebeapi.connect(dataSource['driver_class'], [dataSource['URL'], dataSource['user'], dataSource['password']])
load_dataset()
def save_record(dataframe):
print("IN save_record >>>>>>>>")
for i in range(0, len(dataframe)):
if dataframe["USERNAME"][i] == FLAGS.username:
dataSet = dsx_core_utils.get_remote_data_set_info(FLAGS.submissions_dataset)
insert_query = "INSERT INTO "+ (dataSet['schema'] + '.' if (len(dataSet['schema'].strip()) != 0) else '') + dataSet['table']
insert_query = insert_query + "(USERNAME, NAME, ACCURACY, ENTRIES, LAST) VALUES ('{0}', '{1}', {2}, {3}, '{4}')"
insert_query = insert_query.format(dataframe["USERNAME"][i], dataframe["NAME"][i],dataframe["ACCURACY"][i],dataframe["ENTRIES"][i],dataframe["LAST"][i])
curs = conn.cursor()
curs.execute(insert_query)
# load_dataset(dataSet)
curs.close()
print("YOUR SUBMISSION SAVED SUCCESSFULLY >>>>>>>>>>>>")
def update_record(dataframe):
print("IN update_record >>>>>>>>")
for i in range(0, len(dataframe)):
if dataframe["USERNAME"][i] == FLAGS.username:
# print(submissions_df.iloc[i])
dataSet = dsx_core_utils.get_remote_data_set_info(FLAGS.submissions_dataset)
update_query = "UPDATE "+ (dataSet['schema'] + '.' if (len(dataSet['schema'].strip()) != 0) else '') + dataSet['table']
update_query = update_query +" SET ENTRIES = {0}, ACCURACY = {1}, LAST = '{2}' WHERE USERNAME = '******'"
update_query = update_query.format(dataframe["ENTRIES"][i], dataframe["ACCURACY"][i], dataframe["LAST"][i], dataframe["USERNAME"][i])
curs = conn.cursor()
curs.execute(update_query)
# load_dataset(dataSet)
curs.close()
print("YOUR SUBMISSION UPDATED SUCCESSFULLY >>>>>>>>>>>>")
# #### Retrieve all of the user accounts
#
# ```SQL
# SELECT * FROM 'USER999.AWS_ACCOUNTS'
# ```
#
# > Note: The USER999.AWS_ACCOUNTS dataset is a virtual table mapped to the STOCKS.AWS_ACCOUNTS table running on a Db2 instance running on AWS
# In[1]:
import dsx_core_utils, requests, jaydebeapi, os, io, sys
from pyspark.sql import SparkSession
import pandas as pd
df = None
dataSet = dsx_core_utils.get_remote_data_set_info('USER999.AWS_ACCOUNTS')
dataSource = dsx_core_utils.get_data_source_info(dataSet['datasource'])
if (sys.version_info >= (3, 0)):
conn = jaydebeapi.connect(dataSource['driver_class'], dataSource['URL'],
[dataSource['user'], dataSource['password']])
else:
conn = jaydebeapi.connect(
dataSource['driver_class'],
[dataSource['URL'], dataSource['user'], dataSource['password']])
query = 'select * from "' + (dataSet['schema'] + '"."' if
(len(dataSet['schema'].strip()) != 0) else
'') + dataSet['table'] + '"'
if (dataSet['query']):
query = dataSet['query']
df = pd.read_sql(query, con=conn)
# if published use model path
if(args.get('published').lower() == 'true'):
copy_result = json.loads(published_model_util.copy_model(project_name, "Telco Churn Prediction Model 02"))
if(copy_result['code'] == 200):
model_path = copy_result['path'] + "/model"
published_path = copy_result['path']
else:
raise Exception('Unable to evaluate published model: ' + copy_result['description'])
# create spark context
spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext
# load the input data
if(False):
dataSet = dsx_core_utils.get_remote_data_set_info(input_data.get('dataset'))
dataSource = dsx_core_utils.get_data_source_info(dataSet['datasource'])
dbTableOrQuery = (dataSet['schema'] + '.' if(len(dataSet['schema'].strip()) != 0) else '') + dataSet['table']
dataframe = spark.read.format("jdbc").option("url", dataSource['URL']).option("dbtable",dbTableOrQuery).option("user",dataSource['user']).option("password",dataSource['password']).load()
else:
dataframe = SQLContext(sc).read.csv(input_data , header='true', inferSchema = 'true')
# load the model from disk
model_rf = PipelineModel.load(model_path)
# generate predictions
predictions = model_rf.transform(dataframe)
# Create Evalutation JSON
evaluation = dict()
evaluation["metrics"] = dict()
def get_submissions_dataframe():
submissions_dataset = dsx_core_utils.get_remote_data_set_info(FLAGS.submissions_dataset)
submissions_df = load_dataset(submissions_dataset)
return submissions_df