def query(self, statement, channel_name=None):
data = []
metadata = []
is_select = statement.strip().lower().startswith(
'select') # TODO via parser
is_print = statement.strip().lower().startswith('print')
if is_select or is_print:
result = self.client.query(statement)
metadata = [['Row', 'STRING']]
if has_channels() and channel_name:
_send_to_channel(channel_name,
message_type='task.progress',
message_data={
'status': 'running',
'query_id': 1111
})
for line in result:
# columns = line.keys()
# data.append([line[col] for col in columns])
if 'finalMessage' in line:
break
elif 'header' in line:
continue
else:
line = line.strip()[:-1]
if is_select:
data_line = json.loads(line)
if data_line.get('@type') == 'statement_error':
raise KSqlApiException(data_line['message'])
if data_line['row']: # If limit not reached
data.append(data_line['row']['columns'])
else:
data.append([line])
if has_channels() and channel_name:
_send_to_channel(channel_name,
message_type='task.result',
message_data={
'data': data,
'metadata': metadata,
'query_id': 1111
})
# TODO: special message when end of stream
data = []
else:
data, metadata = self._decode_result(self.ksql(statement))
return data, metadata
def execute(self, notebook, snippet):
channel_name = notebook.get('editorWsChannel')
db = self._get_db()
data, description = db.query(snippet['statement'],
channel_name=channel_name)
has_result_set = data is not None
return {
'sync': not (has_channels() and channel_name),
'has_result_set': has_result_set,
'result': {
'has_more':
False,
'data':
data if has_result_set else [],
'meta': [{
'name': col['name'],
'type': col['type'],
'comment': ''
} for col in description] if has_result_set else [],
'type':
'table'
}
}
def query(self, statement, channel_name=None):
data = []
metadata = []
is_select = statement.strip().lower().startswith('select')
if is_select or statement.strip().lower().startswith('print'):
result = self.client.query(statement)
metadata = [['Row', 'STRING']]
if has_channels() and channel_name:
_send_to_channel(channel_name,
message_type='task.progress',
message_data={
'status': 'running',
'query_id': 1111
})
for line in result:
# columns = line.keys()
# data.append([line[col] for col in columns])
if is_select and line: # Empty first 2 lines?
data_line = json.loads(line)
if data_line['row']: # If limit not reached
data.append(data_line['row']['columns'])
else:
data.append([line])
if has_channels() and channel_name:
_send_to_channel(channel_name,
message_type='task.result',
message_data={
'data': data,
'metadata': metadata,
'query_id': 1111
})
# TODO: special message when end of stream
data = []
else:
data, metadata = self._decode_result(self.ksql(statement))
return data, metadata
from __future__ import absolute_import
import logging
import json
from django.utils.translation import ugettext as _
from desktop.lib.i18n import force_unicode
from desktop.conf import has_channels
from kafka.ksql_client import KSqlApi as KSqlClientApi
from notebook.connectors.base import Api, QueryError
LOG = logging.getLogger(__name__)
if has_channels():
from notebook.consumer import _send_to_channel
def query_error_handler(func):
def decorator(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
message = force_unicode(str(e))
raise QueryError(message)
return decorator
class KSqlApi(Api):
def query(self, statement, channel_name=None):
data = []
metadata = []
is_select = statement.strip().lower().startswith(
'select') # TODO via parser
is_print = statement.strip().lower().startswith('print')
if is_select or is_print:
result = self.client.query(statement)
metadata = [{'type': 'STRING', 'name': 'Row', 'comment': None}]
if has_channels() and channel_name:
_send_to_channel(channel_name,
message_type='task.progress',
message_data={
'status': 'running',
'query_id': 1
})
try:
for line in result:
# columns = line.keys()
# data.append([line[col] for col in columns])
if 'finalMessage' in line:
if has_channels(
) and channel_name: # Send results via WS and empty results
_send_to_channel(channel_name,
message_type='task.result',
message_data={
'status': 'finalMessage',
'query_id': 1
})
break
elif 'header' in line:
continue
else:
line = line.strip()
if is_select:
try:
data_line = json.loads(line)
except ValueError as e:
data_line = json.loads(
line[:-1]) # Most probably record is not JSON
if data_line.get('@type') == 'statement_error':
raise KSqlApiException(data_line['message'])
if data_line['row']: # If limit not reached
data.append(data_line['row']['columns'])
else:
data.append([line])
if has_channels(
) and channel_name: # Send results via WS and empty results
_send_to_channel(channel_name,
message_type='task.result',
message_data={
'data': data,
'meta': metadata,
'query_id': 1
})
data = [] # TODO: special message when end of stream
except RuntimeError as e:
if 'generator raised StopIteration' in str(e):
return data, metadata
else:
raise e
else:
data, metadata = self._decode_result(self.ksql(statement))
if has_channels(
) and channel_name: # Send results via WS and empty results
_send_to_channel(channel_name,
message_type='task.result',
message_data={
'data': data,
'meta': metadata,
'query_id': 1
})
data = [] # TODO: special message when end of stream
return data, metadata
