def get_topic_data(user, name): interpreter = _get_interpreter_from_dialect('ksql', user) api = _get_notebook_api(user, connector_id=interpreter['type']) data = api.get_sample_data(snippet={}) print(data) return data
def get_topic_data(user, name): from desktop.api_public import _get_interpreter_from_dialect # due to a circular import interpreter = _get_interpreter_from_dialect('ksql', user) api = _get_notebook_api(user, connector_id=interpreter['type']) data = api.get_sample_data(snippet={}) print(data) return data
def create_table_from_kafka(self, source, destination, start_time=-1, dry_run=False): if '.' in destination['name']: database, table_name = destination['name'].split('.', 1) else: database = 'default' table_name = destination['name'] final_table_name = table_name source_type = source['sourceType'] interpreter = _get_interpreter_from_dialect('flink', self.user) editor_type = interpreter['type'] # destination['sourceType'] columns = destination['columns'] sql = '''CREATE TABLE %(table_name)s ( %(columns)s ) WITH ( 'connector' = 'kafka', 'topic' = '%(topic)s', 'scan.startup.mode' = 'earliest-offset', 'properties.bootstrap.servers' = 'kafka:9094', 'format' = 'json' );''' % { 'database': database, 'table_name': table_name, 'columns': ',\n'.join( [' %(name)s %(type)s' % col for col in columns]), 'topic': source.get('kafkaSelectedTopics') } if dry_run: return sql else: on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \ '?source_type=' + source_type return make_notebook( name=_('Creating table %(database)s.%(table)s') % { 'database': database, 'table': final_table_name }, editor_type=editor_type, statement=sql.strip(), status='ready', database=database, on_success_url=on_success_url, last_executed=start_time, is_task=True)
def get_topics(user): if has_kafka_api(): return KafkaApi().topics() else: data = {'snippet': {}, 'database': 'topics'} interpreter = _get_interpreter_from_dialect('ksql', user) api = _get_notebook_api(user, connector_id=interpreter['type']) return [ topic['name'] for topic in api.autocomplete(**data)['tables_meta'] if not topic['name'].startswith('__') ]
def get_topic_data(user, name): if has_kafka_api(): print(SchemaRegistryApi().subjects()) print(SchemaRegistryApi().subject(name='Kafka-value')) data = { 'full_headers': [{ 'name': 'message', 'type': 'string' }], 'rows': [['This is rider 894 and I am at 38.1952, -123.1723'], ['This is rider 98 and I am at 39.2531, -121.9547'], ['This is rider 564 and I am at 22.3431, -111.7670']] } else: from desktop.api_public import _get_interpreter_from_dialect # Avoid circular import interpreter = _get_interpreter_from_dialect('ksql', user) api = _get_notebook_api(user, connector_id=interpreter['type']) data = api.get_sample_data(snippet={}) print(data) return data