def query(self, statement, channel_name=None): data = [] metadata = [] is_select = statement.strip().lower().startswith( 'select') # TODO via parser is_print = statement.strip().lower().startswith('print') if is_select or is_print: result = self.client.query(statement) metadata = [['Row', 'STRING']] if has_channels() and channel_name: _send_to_channel(channel_name, message_type='task.progress', message_data={ 'status': 'running', 'query_id': 1111 }) for line in result: # columns = line.keys() # data.append([line[col] for col in columns]) if 'finalMessage' in line: break elif 'header' in line: continue else: line = line.strip()[:-1] if is_select: data_line = json.loads(line) if data_line.get('@type') == 'statement_error': raise KSqlApiException(data_line['message']) if data_line['row']: # If limit not reached data.append(data_line['row']['columns']) else: data.append([line]) if has_channels() and channel_name: _send_to_channel(channel_name, message_type='task.result', message_data={ 'data': data, 'metadata': metadata, 'query_id': 1111 }) # TODO: special message when end of stream data = [] else: data, metadata = self._decode_result(self.ksql(statement)) return data, metadata
def execute(self, notebook, snippet): channel_name = notebook.get('editorWsChannel') db = self._get_db() data, description = db.query(snippet['statement'], channel_name=channel_name) has_result_set = data is not None return { 'sync': not (has_channels() and channel_name), 'has_result_set': has_result_set, 'result': { 'has_more': False, 'data': data if has_result_set else [], 'meta': [{ 'name': col['name'], 'type': col['type'], 'comment': '' } for col in description] if has_result_set else [], 'type': 'table' } }
def query(self, statement, channel_name=None): data = [] metadata = [] is_select = statement.strip().lower().startswith('select') if is_select or statement.strip().lower().startswith('print'): result = self.client.query(statement) metadata = [['Row', 'STRING']] if has_channels() and channel_name: _send_to_channel(channel_name, message_type='task.progress', message_data={ 'status': 'running', 'query_id': 1111 }) for line in result: # columns = line.keys() # data.append([line[col] for col in columns]) if is_select and line: # Empty first 2 lines? data_line = json.loads(line) if data_line['row']: # If limit not reached data.append(data_line['row']['columns']) else: data.append([line]) if has_channels() and channel_name: _send_to_channel(channel_name, message_type='task.result', message_data={ 'data': data, 'metadata': metadata, 'query_id': 1111 }) # TODO: special message when end of stream data = [] else: data, metadata = self._decode_result(self.ksql(statement)) return data, metadata
from __future__ import absolute_import import logging import json from django.utils.translation import ugettext as _ from desktop.lib.i18n import force_unicode from desktop.conf import has_channels from kafka.ksql_client import KSqlApi as KSqlClientApi from notebook.connectors.base import Api, QueryError LOG = logging.getLogger(__name__) if has_channels(): from notebook.consumer import _send_to_channel def query_error_handler(func): def decorator(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: message = force_unicode(str(e)) raise QueryError(message) return decorator class KSqlApi(Api):
def query(self, statement, channel_name=None): data = [] metadata = [] is_select = statement.strip().lower().startswith( 'select') # TODO via parser is_print = statement.strip().lower().startswith('print') if is_select or is_print: result = self.client.query(statement) metadata = [{'type': 'STRING', 'name': 'Row', 'comment': None}] if has_channels() and channel_name: _send_to_channel(channel_name, message_type='task.progress', message_data={ 'status': 'running', 'query_id': 1 }) try: for line in result: # columns = line.keys() # data.append([line[col] for col in columns]) if 'finalMessage' in line: if has_channels( ) and channel_name: # Send results via WS and empty results _send_to_channel(channel_name, message_type='task.result', message_data={ 'status': 'finalMessage', 'query_id': 1 }) break elif 'header' in line: continue else: line = line.strip() if is_select: try: data_line = json.loads(line) except ValueError as e: data_line = json.loads( line[:-1]) # Most probably record is not JSON if data_line.get('@type') == 'statement_error': raise KSqlApiException(data_line['message']) if data_line['row']: # If limit not reached data.append(data_line['row']['columns']) else: data.append([line]) if has_channels( ) and channel_name: # Send results via WS and empty results _send_to_channel(channel_name, message_type='task.result', message_data={ 'data': data, 'meta': metadata, 'query_id': 1 }) data = [] # TODO: special message when end of stream except RuntimeError as e: if 'generator raised StopIteration' in str(e): return data, metadata else: raise e else: data, metadata = self._decode_result(self.ksql(statement)) if has_channels( ) and channel_name: # Send results via WS and empty results _send_to_channel(channel_name, message_type='task.result', message_data={ 'data': data, 'meta': metadata, 'query_id': 1 }) data = [] # TODO: special message when end of stream return data, metadata