def _ws_batch(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
logging.info('Entering {} TimeStamp: {}' .format(function_name, datetime.now().strftime("%H:%M:%S.%f")))
host = socket.gethostname()
ip_addr = socket.gethostbyname(host)
logging.debug('Calling qrag.ini section "{}' .format(q_function_name))
ws_url = config.get(q_function_name, 'ws_url')
token = config.get(q_function_name, 'token')
user_name= config.get(q_function_name, 'username')
batch_size = int(config.get(q_function_name, 'batch_size'))
logging.debug('Batch Size {}' .format(batch_size))
ws_route= config.get(q_function_name, 'ws_route')
#ws_route= '"' + ws_route + '"'
logging.info('API Route : {}' .format(ws_route))
# setup Caching
bCache= config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}" .format(bCache))
if (bCache.lower()=="true"):
logging.info("Caching ****Enabled*** for {}" .format(q_function_name))
else:
logging.info("Caching ****Disabled**** for {}" .format(q_function_name))
md = (('qlik-cache', 'no-store'),)
context.send_initial_metadata(md)
ws_url = ws_url + host +'_'+ ip_addr+'_'+ user_name+'_'
logging.debug('Full url for ws: {} '.format(ws_url))
ws = create_connection(ws_url)
response_rows = []
outer_counter = 1
inner_counter = 1
request_counter = 1
for request_rows in request:
logging.debug('Printing Request Rows - Request Counter {}' .format(request_counter))
request_counter+=1
temp = MessageToDict(request_rows)
logging.debug('Temp Message to Dict {}' .format(temp))
test_rows = temp['rows']
logging.debug('Test Rows: {}' .format(test_rows))
request_size = len(test_rows)
logging.debug('Bundled Row Number of Rows - {}' .format(request_size))
batches = list(qlist.divide_chunks(test_rows, batch_size))
for i in batches:
payload_t ={"action":ws_route}
logging.debug('PreFix Route Seletection {}' .format(payload_t))
logging.debug(len(batches))
payload_t["data"] = i
logging.debug('Size of payload {}' .format(pysize.get_size(payload_t)))
logging.debug('Showing Payload: {}'.format(payload_t))
logging.debug('batch number {}'.format(outer_counter))
ws.send(json.dumps(payload_t))
logging.debug('message sent WS')
outer_counter +=1
payload_t.clear()
for j in i:
#logging.debug("Priniting i {}" .format(i))
resp = json.loads(ws.recv())
#logging.debug('Response Type : {}' .format(type(resp)))
logging.debug('Counter: {} Payload Size: {} Payload Response: {}'.format(inner_counter, pysize.get_size(resp), resp))
inner_counter +=1
result = resp['result']
logging.debug('Log Resulst: {}' .format(result))
duals = iter([SSE.Dual(strData=result)])
#logging.debug(duals)
#logging.debug('Printing Duals {}' .format(duals))
#Yield the row data as bundled rows
response_rows.append(SSE.Row(duals=duals))
logging.debug('Exiting Inner Loop: Printing j {}' .format(j))
yield SSE.BundledRows(rows=response_rows)
ws.close()
logging.info('Exiting {} TimeStamp: {}' .format(function_name, datetime.now().strftime("%H:%M:%S.%f")))
def EvaluateScript(self, header, request, context):
"""
Evaluates script provided in the header, given the
arguments provided in the sequence of RowData objects, the request.
:param header:
:param request: an iterable sequence of RowData.
:param context: the context sent from client
:return: an iterable sequence of RowData.
"""
logging.debug('In EvaluateScritp: ScriptEval')
# Retrieve function type
func_type = self.get_func_type(header)
#logging.debug('Function Type {}' .format(func_type))
# Retrieve data types from header
arg_types = self.get_arg_types(header)
#logging.debug('Arg Type {}' .format(arg_types))
ret_type = self.get_return_type(header)
#logging.debug('Return Type {}' .format(ret_type))
conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'config', 'qrag.ini')
##print(conf_file)
logging.info('Location of qrag.ini {}'.format(conf_file))
config.read(conf_file)
batch_size = int(config.get('base', 'batch_size'))
logging.info('Size of Batch Size {}'.format(batch_size))
#print(type(batch_size))
logging.info('EvaluateScript: {} ({} {}) {}'.format(
header.script, arg_types, ret_type, func_type))
# Check if parameters are provided
#print(header.params)
if header.params:
all_rows = []
# Iterate over bundled rows
for request_rows in request:
# Iterate over rows
for row in request_rows.rows:
# Retrieve parameters
#print('row {} jrp' .format(row))
params = self.get_arguments(context, arg_types, row.duals,
header)
all_rows.append(params)
# First element in the parameter list should contain the data of the first parameter.
all_rows = [list(param) for param in zip(*all_rows)]
#print(all_rows)
if arg_types == ArgType.Mixed:
param_datatypes = [param.dataType for param in header.params]
for i, datatype in enumerate(param_datatypes):
if datatype == SSE.DUAL:
# For easier access to the numerical and string representation of duals, in the script, we
# split them to two list. For example, if the first parameter is dual, it will contain two lists
# the first one being the numerical representation and the second one the string.
all_rows[i] = [
list(datatype) for datatype in zip(*all_rows[i])
]
logging.debug(
'Received data from Qlik (args): {}'.format(all_rows))
return_val = self.evaluate(context,
header.script,
ret_type,
params=all_rows)
else:
# No parameters provided
logging.debug('No Parameteres Provided')
#print(ret_type)
result = self.evaluate(context, header.script, ret_type)
print(type(result))
print(sys.getsizeof(result))
#bundledRows = SSE.BundledRows()
if isinstance(result, str) or not hasattr(result, '__iter__'):
# A single value is returned
bundledRows.rows.add(duals=self.get_duals(result, ret_type))
else:
logging.debug('Size of Result {} {}'.format(
len(result), pysize.get_size(result)))
#if(len(result) > )
#result = result[:10000]
batches = list(qlist.divide_chunks(result, batch_size))
for i in batches:
#print("loop")
bundledRows = SSE.BundledRows()
for row in i:
# note that each element of the result should represent a row
#logging.debug(row)
#logging.debug(type(row))
#logging.debug(ret_type)
#Yield the row data as bundled rows
bundledRows.rows.add(duals=self.get_duals(row, ret_type))
#logging.debug('Size of BundledRow {}'.format(sys.getsizeof(bundledRows)))
yield bundledRows
def _rest_dataframe(request, context):
"""
Rest using single variable
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
url = config.get(q_function_name, 'url')
logging.debug("Rest Url is set to {}".format(url))
bCache = config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
response_rows = []
request_counter = 1
token = config.get('base', 'databricks_token')
header = {'Authorization': f'Bearer {token}'}
schema_metadata = config.get(q_function_name, 'schema_metadata')
batch_size = int(config.get(q_function_name, 'batch_size'))
logging.debug('Batch Size {}'.format(batch_size))
logging.debug('Request is type {}. and raw data {}'.format(
type(request), request))
for request_rows in request:
logging.debug('Request row is type {}. and raw data {}'.format(
type(request_rows), request_rows))
logging.debug('Printing Request Rows - Request Counter {}'.format(
request_counter))
request_counter = request_counter + 1
temp = MessageToDict(request_rows)
logging.debug('Temp Message to Dict {}'.format(temp))
test_rows = temp['rows']
logging.debug('Test Rows: {}'.format(test_rows))
request_size = len(test_rows)
logging.debug(
'Bundled Row Number of Rows - {}'.format(request_size))
batches = list(qlist.divide_chunks(test_rows, batch_size))
for i in batches:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
input_str = ""
#print("i in batches: {}" .format(i))
for j in i:
#print("j in i: {} and type {}" .format(j, type(j)))
try:
row = j["duals"][0]["strData"]
except KeyError as e:
logging.info('Key Error Detected: {}'.format(e))
input_str = ""
#print ('print row {} type {}' .format(row, type(row)))
else:
if (not input_str):
input_str = row
else:
input_str = input_str + '\n' + row
#print ('print input_str {} type {}' .format(input_str, type(input_str)))
#if(len(input_str)==0)
param = input_str
#print('****print param2: {}' .format(param2))
#payload2 = databricks.convert_to_df(param2, schema_metadata)
#logging.debug("Print Type {}" .format(type(payload2)))
#logging.debug('*******Showing Payload: {}'.format(payload2))
#resp2 = databricks.score_model(payload2, url, header)
#logging.debug('Show Payload Response as Text: {}'.format(resp2))
#logging.debug("Print Type {}" .format(type(resp2)))
#@result = str(resp[0])
# Join with current timedate stamp
#for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
#param = [d.strData for d in row.duals][0]
if (len(param) == 0):
logging.debug('No Payload')
else:
logging.debug("Showing Param: {}".format(param))
payload = databricks.convert_to_df(param, schema_metadata)
logging.debug("Print Type {}".format(type(payload)))
logging.debug('Showing Payload: {}'.format(payload))
resp = databricks.score_model(payload, url, header)
logging.debug(
'Show Payload Response as Text: {}'.format(resp))
logging.debug("Print Type {}".format(type(resp)))
#result = str(resp[0])
# Create an iterable of dual with the result
for result in resp:
logging.debug('Show Result: {}'.format(result))
duals = iter([SSE.Dual(strData=str(result))])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
logging.info('Exiting {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))