def get_duals(result, ret_type):
if isinstance(result, str) or not hasattr(result, '__iter__'):
result = [result]
# Transform the result to an iterable of Dual data
if ret_type == ReturnType.String:
return iter([SSE.Dual(strData=col) for col in result])
elif ret_type == ReturnType.Numeric:
return iter([SSE.Dual(numData=col) for col in result])
def _cluster(request, context):
"""
Look for clusters within a dimension using the given features. Scalar function.
Three variants are implemented:
:0: one dimension and a string of numeric features.
:1: two dimensions and one measure. This will only work when used in the Qlik load script.
:2: one dimension, the latitude and longitude.
:
:param request: an iterable sequence of RowData
:param context:
:return: the clustering classification for each row
:Qlik expression examples:
:<AAI Connection Name>.Cluster(dimension, sum(value1) & ';' & sum(value2), 'scaler=standard')
"""
# Get a list from the generator object so that it can be iterated over multiple times
request_list = [request_rows for request_rows in request]
# Get the function id from the header to determine the variant being called
function = ExtensionService._get_function_id(context)
if function == 0:
# The standard variant takes in one dimension and a string of semi-colon separated features.
variant = "standard"
elif function == 1:
# In the second variant features for clustering are produced by pivoting the measure for the second dimension.
variant = "two_dims"
elif function == 2:
# The third variant if for finding clusters in geospatial coordinates.
variant = "lat_long"
# Create an instance of the HDBSCANForQlik class
# This will take the request data from Qlik and prepare it for clustering
clusterer = HDBSCANForQlik(request_list, context, variant=variant)
# Calculate the clusters and store in a Pandas series (or DataFrame in the case of a load script call)
clusters = clusterer.scan()
# Check if the response is a DataFrame.
# This occurs when the load_script=true argument is passed in the Qlik expression.
response_is_df = isinstance(clusters, pd.DataFrame)
# Convert the response to a list of rows
clusters = clusters.values.tolist()
# We convert values to type SSE.Dual, and group columns into a iterable
if response_is_df:
response_rows = [
iter([SSE.Dual(strData=row[0]),
SSE.Dual(numData=row[1])]) for row in clusters
]
else:
response_rows = [iter([SSE.Dual(numData=row)]) for row in clusters]
# Values are then structured as SSE.Rows
response_rows = [SSE.Row(duals=duals) for duals in response_rows]
# Yield Row data as Bundled rows
yield SSE.BundledRows(rows=response_rows)
def _translateScript(request, context):
idNumList = []
translationsList = []
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# grab the text
text = [d.strData for d in row.duals][0]
# grab the id
idNum = [d.numData for d in row.duals][1]
idNumList.append(idNum)
# grab the source
src = [d.strData for d in row.duals][2]
# grab the destination
dest = [d.strData for d in row.duals][3]
translationsList.append(text)
try:
translator = Translator()
translations = translator.translate(translationsList,
src=src,
dest=dest)
except:
pass
resultList = [i.text for i in translations]
logging.info('Records translated: ' + str(len(resultList)))
# Create an iterable of dual with the result
dualsList = []
dualsList.append([SSE.Dual(numData=d) for d in idNumList])
dualsList.append([SSE.Dual(strData=d) for d in resultList])
response_rows = []
for i in range(len(idNumList)):
duals = [dualsList[z][i] for z in range(len(dualsList))]
response_rows.append(SSE.Row(duals=iter(duals)))
# Set and send Table header
table = SSE.TableDescription(name='Translations')
table.fields.add(dataType=SSE.NUMERIC)
table.fields.add(dataType=SSE.STRING)
md = (('qlik-tabledescription-bin', table.SerializeToString()), )
context.send_initial_metadata(md)
yield SSE.BundledRows(rows=response_rows)
def _getPDFTable(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
# empty parameters
extraction_method = None
path = None
template = None
for request_rows in request:
# pull duals from each row, and the numData from duals
for row in request_rows.rows:
if extraction_method is None:
extraction_method = [d.strData for d in row.duals][0]
if path is None:
path = [d.strData for d in row.duals][1]
if template is None:
template = [d.strData for d in row.duals][2]
# read PDF with template
if extraction_method == 'stream':
df_list = read_pdf_with_template(path, template, stream=True)
else:
df_list = read_pdf_with_template(path, template, lattice=True)
final_df = pd.DataFrame()
count = 1
for df in df_list:
df['tableID'] = str(count)
final_df = pd.concat([final_df, df], axis=0, ignore_index=True)
count = count + 1
columns = final_df.columns
# iterate through df columns and format as SSE duals
dualsList = []
for col in columns:
tmpList = final_df[col].tolist()
dualsList.append([
SSE.Dual(strData=d) if type(d) is str else SSE.Dual(numData=d)
for d in tmpList
])
# create response rows
response_rows = []
for i in range(len(tmpList)):
duals = [dualsList[z][i] for z in range(len(dualsList))]
response_rows.append(SSE.Row(duals=iter(duals)))
# return response
yield SSE.BundledRows(rows=response_rows)
def _sum_of_rows(request):
"""
Summarize two parameters row wise. Tensor function.
:param request: an iterable sequence of RowData
:return: the same iterable sequence of row data as received
"""
# Iterate over bundled rows
for request_rows in request:
response_rows = []
# Iterating over rows
for row in request_rows.rows:
# Retrieve the numerical value of the parameters
# Two columns are sent from the client, hence the length of params will be 2
params = [d.numData for d in row.duals]
# Sum over each row
result = sum(params)
# Create an iterable of Dual with a numerical value
duals = iter([SSE.Dual(numData=result)])
# Append the row data constructed to response_rows
response_rows.append(SSE.Row(duals=duals))
# Yield Row data as Bundled rows
yield SSE.BundledRows(rows=response_rows)
def _sum_of_column(request):
"""
Summarize the column sent as a parameter. Aggregation function.
:param request: an iterable sequence of RowData
:return: int, sum if column
"""
params = []
# Iterate over bundled rows
for request_rows in request:
# Iterating over rows
for row in request_rows.rows:
# Retrieve numerical value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.numData for d in row.duals][0]
params.append(param)
# Sum all rows collected the the params variable
result = sum(params)
# Create an iterable of dual with numerical value
duals = iter([SSE.Dual(numData=result)])
# Yield the row data constructed
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def _ws_single(request, context):
"""
Single Row Processing for Websockets
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
logging.info('Entering {} TimeStamp: {}' .format(function_name, datetime.now().strftime("%H:%M:%S.%f")))
#Start by Gathering Environmental Varaiable
host = socket.gethostname()
ip_addr = socket.gethostbyname(host)
ws_url = config.get(q_function_name, 'ws_url')
token = config.get(q_function_name, 'token')
user_name= config.get(q_function_name, 'username')
ws_route= config.get(q_function_name, 'ws_route')
bCache= config.get(q_function_name, 'cache')
logging.debug('Pringint Route for WS {}' .format(ws_route))
logging.debug("Caching is set to {}" .format(bCache))
if (bCache.lower()=="true"):
logging.info("Caching ****Enabled*** for {}" .format(q_function_name))
else:
logging.info("Caching ****Disabled**** for {}" .format(q_function_name))
md = (('qlik-cache', 'no-store'),)
context.send_initial_metadata(md)
#In Future we will use the Token for Liencensing and Throttling
#Currently we are using Comblination of host+ipaddr+username for Client Identification
ws_url = ws_url + host +'_'+ ip_addr+'_'+ user_name+'_'
logging.debug('Websocket URL : {}' .format(ws_url))
ws = create_connection(ws_url)
response_rows = []
for request_rows in request:
# Iterate over rows
# Default code
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
result = ''
if (len(param) ==0):
logging.debug('Parameters are Empty')
result = 'Error'
else:
payload = '{"action":"'+ ws_route +'","data":"' + param + '"}'
logging.debug('Showing Payload: {}'.format(payload))
ws.send(payload)
#logging.info('Show Payload Response: {}'.format(resp.text))
resp = json.loads(ws.recv())
logging.debug(resp)
result = resp['result']
logging.debug('Show Result: {}'.format(result))
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
ws.close()
logging.info('Exiting {} TimeStamp: {}' .format(function_name, datetime.now().strftime("%H:%M:%S.%f")))
def _translate(request, context):
translationsList = []
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# grab the text
text = [d.strData for d in row.duals][0]
# grab the source
src = [d.strData for d in row.duals][1]
# grab the destination
dest = [d.strData for d in row.duals][2]
translationsList.append(text)
try:
translator = Translator()
translations = translator.translate(translationsList,
src=src,
dest=dest)
except:
pass
resultList = [i.text for i in translations]
logging.info('Records translated: ' + str(len(resultList)))
# Create an iterable of dual with the result
duals = iter([[SSE.Dual(strData=d)] for d in resultList])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=d) for d in duals])
def evaluate(context, script, ret_type, params=[]):
"""
Evaluates a script with given parameters.
:param context: the context sent from client
:param script: script to evaluate
:param ret_type: return data type
:param params: params to evaluate. Default: []
:return: a RowData of string dual
"""
if ret_type == ReturnType.String:
# Evaluate script
result = eval(script, {'args': params})
# Transform the result to an iterable of Dual data with a string value
duals = iter([SSE.Dual(strData=result)])
# Create row data out of duals
return SSE.BundledRows(rows=[SSE.Row(duals=duals)])
else:
# This plugin does not support other return types than string
# Make sure the error handling, including logging, works as intended in the client
msg = 'Return type {} is not supported in this plugin.'.format(
ret_type)
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details(msg)
# Raise error on the plugin-side
raise grpc.RpcError(grpc.StatusCode.UNIMPLEMENTED, msg)
def _get_sql(request):
"""
GetSQL function, tensor
"""
# Iterate over bundled rows
for request_rows in request:
response_rows = []
# Iterating over rows
for row in request_rows.rows:
# Retrieve numerical value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
# This is pretty costly.. remove it when you see SQL generating normally
print(param)
# Execute SQL command
cursor.execute(param)
results = cursor.fetchall()
results = results[0][0]
# Create an iterable of dual with numerical value
duals = iter([SSE.Dual(numData=results)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data constructed
yield SSE.BundledRows(rows=response_rows)
def _max_of_columns_2(request, context):
"""
Find max of each column. This is a table function.
:param request: an iterable sequence of RowData
:param context:
:return: a table with numerical values, two columns and one row
"""
result = [_MINFLOAT] * 2
# Iterate over bundled rows
for request_rows in request:
# Iterating over rows
for row in request_rows.rows:
# Retrieve the numerical value of each parameter
# and update the result variable if it's higher than the previously saved value
for i in range(0, len(row.duals)):
result[i] = max(result[i], row.duals[i].numData)
# Create an iterable of dual with numerical value
duals = iter([SSE.Dual(numData=r) for r in result])
# Set and send Table header
table = SSE.TableDescription(name='MaxOfColumns', numberOfRows=1)
table.fields.add(name='Max1', dataType=SSE.NUMERIC)
table.fields.add(name='Max2', dataType=SSE.NUMERIC)
md = (('qlik-tabledescription-bin', table.SerializeToString()), )
context.send_initial_metadata(md)
# Yield the row data constructed
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def _no_cache(request, context):
"""
Cache disabled. Add the datetime stamp to the end of each string value.
:param request:
:param context: used for disabling the cache in the header.
:return: string
"""
# Disable caching.
md = (('qlik-cache', 'no-store'),)
context.send_initial_metadata(md)
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
# Join with current timedate stamp
result = param + ' ' + datetime.now().isoformat()
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def _hello_world_aggr(request, context):
"""
Aggregates the parameters to a single comma separated string.
:param request: iterable sequence of bundled rows
:return: string
"""
params = []
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
params.append(param)
# Aggregate parameters to a single string
result = ', '.join(params)
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def numbers_to_duals(*args):
"""
Converts a number of numbers to a list of Duals.
"""
duals = [SSE.Dual(numData=n) for n in list(args)]
return duals
def strings_to_duals(*args):
"""
Converts a number of strings to a list of Duals.
"""
duals = [SSE.Dual(strData=s) for s in list(args)]
return duals
def _prophet_seasonality(request, context):
"""
Provide the seasonality component of the Prophet timeseries forecast. Scalar function.
:param request: an iterable sequence of RowData
:param context: not used for now
:return: the forecasted value for each row
:
:Qlik expression example:
:<AAI Connection Name>.Prophet_Seasonality(Month, $(vConcatSeries), $(vHolidays), 'seasonality=yearly, freq=MS, debug=true')
:The fourth argument in the Qlik expression is a string of parameters.
:This should take the form of a comma separated string:
:e.g 'seasonality=yearly, freq=MS, debug=true' or 'seasonality=weekly, freq=D'
:
:Parameters accepted for the Prophet() function are: cap, floor, changepoint_prior_scale, interval_width,
:lower_window, upper_window
:
:Parameters accepted for the make_future_dataframe() function are: freq
:
:For more information on these parameters go here: https://facebook.github.io/prophet/docs/quick_start.html
:
:Additional parameters used are: return, take_log, debug
:
:cap = 1000 : A logistic growth model can be defined using cap and floor. Values should be double or integer
:changepoint_prior_scale = 0.05 : Decrease if the trend changes are being overfit, increase for underfit
:interval_width = 0.08 : Set the width of the uncertainty intervals
:lower_window = -1 : Only used with holidays. Extend the holiday by certain no. of days prior to the date.
:upper_window = 1 : Only used with holidays. Extend the holiday by certain no. of days after the date.
:freq = MS : The frequency of the time series. e.g. MS for Month Start. See the possible options here:
: : http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
:return = yhat : Any of the options in the forecast result. You can see these options with debug=true
: : yhat, yhat_upper, yhat_lower : Forecast, upper and lower limits
: : y_then_yhat, y_then_yhat_upper, y_then_yhat_lower : Return forecast only for forecast periods
: : trend, trend_upper, trend_lower : Trend component of the timeseries
: : seasonal, seasonal_upper, seasonal_lower: Seasonal component of the timeseries
:take_log = false : Apply logarithm to the values before the forecast. Default is true
:debug = true : Print exexution information to the terminal and logs in ..\logs\Prophet Log <n>.txt
"""
# Get a list from the generator object so that it can be iterated over multiple times
request_list = [request_rows for request_rows in request]
# Create an instance of the ProphetForQlik class
# This will take the request data from Qlik and prepare it for forecasting
predictor = ProphetForQlik.init_seasonality(request_list)
# Calculate the forecast and store in a Pandas series
forecast = predictor.predict()
# Values in the series are converted to type SSE.Dual
response_rows = forecast.apply(lambda result: iter([SSE.Dual(numData=result)]))
# Values in the series are converted to type SSE.Row
# The series is then converted to a list
response_rows = response_rows.apply(lambda duals: SSE.Row(duals=duals)).tolist()
# Iterate over bundled rows
for request_rows in request_list:
# Yield Row data as Bundled rows
yield SSE.BundledRows(rows=response_rows)
def _geocode(request, context):
api_selection = None
coordinateList = []
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# The first strData is the coordinate
data = [d.strData for d in row.duals][0]
# Grab selected API
# Possible choices are: Google, Open Street, GeocoderDotUS
if not api_selection:
api_selection = [d.strData for d in row.duals][1]
if 'google' in api_selection.lower():
geolocator = GoogleV3(
api_key="AIzaSyDlLSgsXiYG-zKYTuLxsbj1BNr7XoGpRMk")
elif 'open street' in api_selection.lower():
geolocator = Nominatim(scheme='http')
elif 'geocoderdotus' in api_selection.lower():
geolocator = GeocoderDotUS()
else:
geolocator = Nominatim()
# geocode
for i in range(2):
try:
location = geolocator.geocode(data, timeout=2)
break
except geopy.exc.GeocoderTimedOut:
pass
except geopy.exc.GeocoderQueryError:
pass
except urllib.error.HTTPError:
pass
try:
latitude = location.latitude
except AttributeError:
latitude = 'Unavailable'
try:
longitude = location.longitude
except AttributeError:
longitude = 'Unavailable'
coordinates = '[' + str(longitude) + ', ' + str(latitude) + ']'
coordinateList.append(coordinates)
# Create an iterable of dual with the result
duals = iter([[SSE.Dual(strData=c)] for c in coordinateList])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=d) for d in duals])
def _evaluate(request):
"""
Summarize the column sent as a parameter. Aggregation function.
:param request: an iterable sequence of RowData
:return: int, sum if column
"""
params = []
logging.info('_evaluate')
logging.info('_evaluate request {}'.format(request))
print(request)
# Iterate over bundled rows
for request_rows in request:
print(request_rows.rows)
print(len(request_rows.rows))
# Iterating over rows
logging.info('_evaluate request_rows {}'.format(request_rows.rows))
for row in request_rows.rows:
# Retrieve numerical value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.numData for d in row.duals]
logging.info('_evaluate row {}'.format(param))
params.append(param)
h2o.init()
dir_path = os.path.dirname(
os.path.realpath(__file__)) + "\\kmeans_iris"
#results = h2o.load_model("C:/Users/daniel/Documents/Qlik Advanced Analytics/Examples/Python/H2O/kmeans_iris")
results = h2o.load_model(dir_path)
newData = h2o.H2OFrame(params)
predictedNew = results.predict(newData)
predicted_as_list = h2o.as_list(predictedNew, use_pandas=False)
predicted_as_list.pop(0)
response_rows = []
for result in predicted_as_list:
# Create an iterable of Dual with a numerical value
duals = iter([SSE.Dual(numData=int(result[0]))])
# Append the row data constructed to response_rows
response_rows.append(SSE.Row(duals=duals))
#print(predicted_as_list)
logging.info('_evaluate params {}'.format(params))
logging.info(
'_evaluate predicted_as_list {}'.format(predicted_as_list))
# Sum all rows collected the the params variable
#result = sum(params[0])
# Create an iterable of dual with numerical value
#duals = iter([SSE.Dual(numData=result)])
# Yield the row data constructed
yield SSE.BundledRows(rows=response_rows)
def evaluate(script, ret_type, params=[]):
"""
Evaluates a script with given parameters and construct the result to a Row of duals.
:param script: script to evaluate
:param ret_type: return data type
:param params: params to evaluate. Default: []
:return: a RowData of string dual
"""
# Evaluate script
result = eval(script, {'args': params, 'np': np})
# Transform the result to an iterable of Dual data
if ret_type == ReturnType.String:
duals = iter([SSE.Dual(strData=result)])
elif ret_type == ReturnType.Numeric:
duals = iter([SSE.Dual(numData=result)])
return SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def _linearRegression(request, context):
# clear the log for ARIMA details
## f = open(logger,'w')
## f.write('New function call\n')
# instantiate a list for measure data
dataList = []
for request_rows in request:
# iterate over each request row (contains rows, duals, numData)
## f.write('Request Rows: ' + str(request_rows) + '\n')
# pull duals from each row, and the numData from duals
for row in request_rows.rows:
# the first numData contains the measure data
data = [d.numData for d in row.duals][0]
# try to convert number to float
try:
float(data)
except ValueError:
data = 0
# append each data point to a list and log it
dataList.append(data)
## f.write('Row: ' + str(data) + '\n')
# grab the length of the data list and convert
X_len = len(dataList)
X = np.asarray(range(X_len))
# convert the data into an array
Y = np.asarray(dataList)
# fit linear regression model
mdl = LinearRegression().fit(X.reshape(-1, 1), Y)
# grab m and b from y = mx + b
m = mdl.coef_[0]
b = mdl.intercept_
# calculate regression line points
regressionResults = []
gen = (i for i in range(X_len))
for i in gen:
y = m * i + b
regressionResults.append(y)
# Create an iterable of dual with the result
duals = iter([[SSE.Dual(numData=d)] for d in regressionResults])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=d) for d in duals])
def _getField(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
mytime = datetime.utcnow()
params = []
csvrows = []
csvrows.append(['DateTimeUTC','FieldRef','UserRef','Comment'])
logging.debug("Request: " + str(request) + "\n")
dataUpdate()
#Not optimal but will need to fix later.
global qpiData
global qpiObsfData
global qpiAccess
for request_rows in request:
logging.debug("Request rows: " + str(request_rows) + "\n")
response_rows = []
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
params = [d.strData for d in row.duals]
#Retrieve the requested value from the loaded data here...
try:
if params[2] in qpiAccess:
returnValue = str(qpiData[params[1]][params[0]])
csvrows.append([mytime,params[1],params[2],'Field: ' + params[0] + ' - ' + params[3]])
#Added threading from https://stackoverflow.com/questions/39440015/how-to-do-simple-async-calls-in-python
# needs to be thoroughly tested on LARGE data volumes
with futures.ThreadPoolExecutor(max_workers=1) as executor:
executor.submit(csvlogger,csvrows)
else:
# User is not allowed to see real data, returning obfuscated data and no need to log access.
returnValue = str(qpiObfuscated[params[1]][params[0]])
except:
returnValue = 'Can not retrieve requested data, see log file.'#
logging.error('Error in retrieving data\nNo data exists for ID: ' + params[1] + ' Field: ' + params[0] + ' for User: '******' Comment: ' + params[3])
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=returnValue)])
response_rows.append(SSE.Row(duals=duals))
yield SSE.BundledRows(rows=response_rows)
def _rest_single(request, context):
"""
Rest using single variable
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
url = config.get(q_function_name, 'url')
logging.debug("Rest Url is set to {}".format(url))
bCache = config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
response_rows = []
request_counter = 1
for request_rows in request:
logging.debug('Printing Request Rows - Request Counter {}'.format(
request_counter))
request_counter = request_counter + 1
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
# Join with current timedate stamp
if (len(param) == 0):
logging.info(
'Exiting {} TimeStamp: {} due to Data being Empty '.
format(function_name,
datetime.now().strftime("%H:%M:%S.%f")))
else:
payload = '{"data":"' + param + '"}'
logging.debug('Showing Payload: {}'.format(payload))
resp = requests.post(url, data=payload)
logging.debug('Show Payload Response as Text: {}'.format(
resp.text))
result = resp.text
result = result.replace('"', '')
result = result.strip()
logging.debug('Show Result: {}'.format(result))
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
logging.info('Exiting {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
def _get_duals(row):
"""
Transforms one row to an iterable of duals.
:param result: one row of data
:return: one row of data as an iterable of duals
"""
# Transform the row to an iterable of Dual data
duals = []
for i, col in enumerate(row):
# if the value is null:
if pd.isnull(col):
duals.append(SSE.Dual(numData=np.NaN, strData=''))
# if the value is numeric:
elif isinstance(col, (int, float)):
duals.append(SSE.Dual(numData=col, strData=str(col)))
# if the value is a string:
else:
duals.append(SSE.Dual(numData=np.NaN, strData=str(col)))
return iter(duals)
def _sentiment(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
# initiate vader sentiment analysis
global analyser
sentence_ids = []
scores = []
for request_rows in request:
# pull duals from each row, and the numData from duals
for row in request_rows.rows:
# get sentence id from the row
sentence_id = [d.strData for d in row.duals][0]
# add sentence id to the list
sentence_ids.append(sentence_id)
# get sentence from the row
sentence = [d.strData for d in row.duals][1]
# get score
score = str(analyser.polarity_scores(str(sentence)))
# add score to list
scores.append(score)
print(sentence_ids)
print(scores)
# Create an iterable of dual with the result
dualsList = []
dualsList.append([SSE.Dual(strData=d) for d in sentence_ids])
dualsList.append([SSE.Dual(strData=d) for d in scores])
response_rows = []
for i in range(len(sentence_ids)):
duals = [dualsList[z][i] for z in range(len(dualsList))]
response_rows.append(SSE.Row(duals=iter(duals)))
yield SSE.BundledRows(rows=response_rows)
def _rest_30(request, context):
"""
Aggregates the parameters to a single comma separated string.
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
url = config.get(q_function_name, 'url')
bCache = config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
# Iterate over bundled rows
response_rows = []
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals]
if (len(param) == 0):
logging.debug('Parameters are Empty')
result = 'Error'
#logging.info('Showing Payload: {}'.format(param))
# Aggregate parameters to a single string
# Join payload via =','.join(param)
else:
payload = '{"data":"' + (','.join(param)) + '"}'
logging.debug('Showing Payload: {}'.format(payload))
resp = requests.post(url, data=payload)
logging.debug('Show Payload Response: {}'.format(
resp.text))
result = resp.text
result = result.replace('"', '')
result = result.strip()
logging.debug('Show Result: {}'.format(result))
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
logging.info('Exiting Predict v2 TimeStamp: {}'.format(
datetime.now().strftime("%H:%M:%S.%f")))
def get_duals(result, ret_type):
"""
Transforms one row in qResult to an iterable of duals.
:param result: one row of qResult
:param ret_type: a list containing a data type for each column in qResult
:return: one row of data as an iterable of duals
"""
# result must be iterable
result = [result] if isinstance(
result,
(str, tuple)) or not hasattr(result, '__iter__') else result
# Transform the result to an iterable of Dual data
duals = []
for i, col in enumerate(result):
data_type = ret_type[i]
if data_type == ReturnType.String:
duals.append(SSE.Dual(strData=col))
elif data_type == ReturnType.Numeric:
duals.append(SSE.Dual(numData=col))
elif data_type == ReturnType.Dual:
# col is a tuple with a numeric and string representation
duals.append(SSE.Dual(numData=col[0], strData=col[1]))
return iter(duals)
def _impute(request):
df = pd.DataFrame(columns=['Age', 'High', "Weight"])
# Iterate over bundlwxed rows
i = -1
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
i += 1
Age = [d.numData for d in row.duals][0]
High = [d.numData for d in row.duals][1]
Weight = [d.numData for d in row.duals][2]
df.loc[i] = [Age, High, Weight]
print(df)
df.High[df['High'].isnull()] = np.nan
# use simple imputer with mean strategy
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
df = imp.fit_transform(df)
# transform df to a dataframe
df = pd.DataFrame(df)
# calculate sum of columns for each row , just an additional operation
result = df.sum(axis=1)
df['sum'] = result
# the print statement just to check values in cmd
print(df)
data = df.values.tolist()
response_rows = [
iter([
SSE.Dual(numData=row[0]),
SSE.Dual(numData=row[1]),
SSE.Dual(numData=row[2]),
SSE.Dual(numData=row[3])
]) for row in data
]
response_rows = [SSE.Row(duals=duals) for duals in response_rows]
# Yield Row data as Bundled rows
yield SSE.BundledRows(rows=response_rows)
def _SomaLinha(request):
params = []
for request_rows in request:
for row in request_rows.rows:
p1 = [d.numData for d in row.duals][0]
p2 = [d.numData for d in row.duals][1]
p3 = [d.numData for d in row.duals][2]
params.append(p1)
params.append(p2)
params.append(p3)
result = sum(params)
print('Variaveis: ', params, ' Soma é: ', result)
duals = iter([SSE.Dual(numData=result)])
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
def _predictPlayByPlay(request, context):
df = pd.DataFrame(columns=[
'quarter', 'seconds_elapsed', 'offense_team', 'yardline', 'down',
'yards_to_go', 'home_team', 'away_team', 'curr_home_score',
'curr_away_score'
])
# Iterate over bundled rows
i = -1
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
i += 1
modelName = [d.strData for d in row.duals][0]
gameId = [d.strData for d in row.duals][1]
quarter = [d.strData for d in row.duals][2]
seconds_elapsed = [d.strData for d in row.duals][3]
offense_team = [d.strData for d in row.duals][4]
yardline = [d.strData for d in row.duals][5]
down = [d.strData for d in row.duals][6]
yards_to_go = [d.strData for d in row.duals][7]
home_team = [d.strData for d in row.duals][8]
away_team = [d.strData for d in row.duals][9]
curr_home_score = [d.strData for d in row.duals][10]
curr_away_score = [d.strData for d in row.duals][11]
df.loc[i] = [
quarter,
int(seconds_elapsed), offense_team,
int(yardline),
int(down),
int(yards_to_go), home_team, away_team,
int(curr_home_score),
int(curr_away_score)
]
logging.info(modelName)
model = p.load(open('models\\' + modelName, 'rb'), encoding='latin1')
wp = list(model.predict_wp(df))
# Create an iterable of dual with the result
duals = iter([[SSE.Dual(numData=d)] for d in wp])
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=[SSE.Row(duals=d) for d in duals])
def _encrypt(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
#Encrypt the first parameter here...
encryptedValue = cipher.encrypt(str.encode([d.strData for d in row.duals][0]))
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=encryptedValue)])
yield SSE.BundledRows(rows=[SSE.Row(duals=duals)])
