def get_topic_data(user, name):
interpreter = _get_interpreter_from_dialect('ksql', user)
api = _get_notebook_api(user, connector_id=interpreter['type'])
data = api.get_sample_data(snippet={})
print(data)
return data
def get_topic_data(user, name):
from desktop.api_public import _get_interpreter_from_dialect # due to a circular import
interpreter = _get_interpreter_from_dialect('ksql', user)
api = _get_notebook_api(user, connector_id=interpreter['type'])
data = api.get_sample_data(snippet={})
print(data)
return data
def create_table_from_kafka(self,
source,
destination,
start_time=-1,
dry_run=False):
if '.' in destination['name']:
database, table_name = destination['name'].split('.', 1)
else:
database = 'default'
table_name = destination['name']
final_table_name = table_name
source_type = source['sourceType']
interpreter = _get_interpreter_from_dialect('flink', self.user)
editor_type = interpreter['type'] # destination['sourceType']
columns = destination['columns']
sql = '''CREATE TABLE %(table_name)s (
%(columns)s
) WITH (
'connector' = 'kafka',
'topic' = '%(topic)s',
'scan.startup.mode' = 'earliest-offset',
'properties.bootstrap.servers' = 'kafka:9094',
'format' = 'json'
);''' % {
'database': database,
'table_name': table_name,
'columns': ',\n'.join(
[' %(name)s %(type)s' % col for col in columns]),
'topic': source.get('kafkaSelectedTopics')
}
if dry_run:
return sql
else:
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
'?source_type=' + source_type
return make_notebook(
name=_('Creating table %(database)s.%(table)s') % {
'database': database,
'table': final_table_name
},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True)
def get_topics(user):
if has_kafka_api():
return KafkaApi().topics()
else:
data = {'snippet': {}, 'database': 'topics'}
interpreter = _get_interpreter_from_dialect('ksql', user)
api = _get_notebook_api(user, connector_id=interpreter['type'])
return [
topic['name'] for topic in api.autocomplete(**data)['tables_meta']
if not topic['name'].startswith('__')
]
def get_topic_data(user, name):
if has_kafka_api():
print(SchemaRegistryApi().subjects())
print(SchemaRegistryApi().subject(name='Kafka-value'))
data = {
'full_headers': [{
'name': 'message',
'type': 'string'
}],
'rows': [['This is rider 894 and I am at 38.1952, -123.1723'],
['This is rider 98 and I am at 39.2531, -121.9547'],
['This is rider 564 and I am at 22.3431, -111.7670']]
}
else:
from desktop.api_public import _get_interpreter_from_dialect # Avoid circular import
interpreter = _get_interpreter_from_dialect('ksql', user)
api = _get_notebook_api(user, connector_id=interpreter['type'])
data = api.get_sample_data(snippet={})
print(data)
return data