def request(self,
command,
sub_url,
port=None,
json=None,
expected_status_code=201):
if port is None:
port = self.kafka_connect_port
kafka_rest_endpoint = self.kafka_rest_url + ":" + str(port)
request = requests.Request(command,
kafka_rest_endpoint + sub_url,
json=json)
prepared_request = request.prepare()
if self.dry or self.debug:
pretty_print(request.__dict__, title="Connector HTTP Request")
if not self.dry:
response = self.requests_session.send(prepared_request)
if self.debug:
pretty_print(response.json(), title="Connector HTTP Response")
if response.status_code != expected_status_code:
raise Exception(
"Error: status code {} != expected status code {}! Run with -g/--debug to see server response"
.format(response.status_code, expected_status_code))
return response
def cassandra_sink(ctx, topic_name, keyspace, table, cassandra_username,
cassandra_password_file, tasks_max, cassandra_hosts,
cassandra_port):
from sonar_driver.kafka_connect.connector import Connector
from sonar_driver.kafka_connect.cassandra_sink_config import CassandraSinkConfig
connector = Connector(
"cassandra_sink-" + topic_name,
CassandraSinkConfig(
topic_name,
keyspace,
table,
cassandra_username,
cassandra_password_file,
cassandra_hosts.split(','),
cassandra_port,
None,
tasks_max,
False # TODO: when delete sunk works, enable this
))
if ctx.obj['DEBUG']:
pretty_print(connector.json(), title="Connector")
pretty_print(connector.json(), colorize=False)
def status(ctx, connector_names, a):
statuses = []
get_connector_args(ctx, connector_names, a)
for connector_name in ctx.obj['KafkaConnectors']:
ret = ctx.obj['KafkaConnectSession'].get_connector_status(
connector_name)
if not ctx.obj['DRY']:
statuses.append(ret.json())
pretty_print(statuses)
def create_table_from_avro_schema(self, keyspace, table, avro_schema, partition_key, cluster_key):
avro_json = avro_schema.to_json()
columns_clause = ', '.join(map(lambda f: "\"" + f['name'] + "\"" + ' ' + SonarCassandraSession.avro2cass(f['type']), avro_json['fields']))
primary_key_clause = SonarCassandraSession.primary_key(partition_key, cluster_key)
create_query = "CREATE TABLE {}.{} ({}, PRIMARY KEY {})".format(keyspace, table, columns_clause, primary_key_clause)
if self.debug or self.dry:
pretty_print(create_query, title="Create table CQL", lexer=lexers.SqlLexer())
if not self.dry:
self.session.execute(create_query, timeout=None)
def cassandra(ctx, cassandra_password_file, cassandra_hosts, cassandra_port):
from sonar_driver.cassandra.session import SonarCassandraSession
ctx.obj['CassandraSession'] = SonarCassandraSession(
password_filename=cassandra_password_file,
hosts=cassandra_hosts.split(','),
port=cassandra_port,
dry=ctx.obj['DRY'],
debug=ctx.obj['DEBUG'])
pretty_print(ctx.obj['CassandraSession'].__dict__,
title='Cassandra Session')
def create_table(ctx, keyspace, table, avro_schema_file, partition_key,
cluster_key):
import avro.schema
cassandra_session = ctx.obj['CassandraSession']
with open(avro_schema_file, 'r') as f:
avro_schema = avro.schema.Parse(f.read())
if ctx.obj['DEBUG']:
pretty_print(avro_schema.to_json(), title="Avro schema")
if not ctx.obj['DRY'] and cassandra_session.table_exists(keyspace, table):
raise Exception("Table {}.{} already exists!".format(keyspace, table))
else:
cassandra_session.create_table_from_avro_schema(
keyspace, table, avro_schema, partition_key, cluster_key)
def uninstall_connectors(connectors,
kafka_rest_url='localhost',
kafka_rest_port=8083,
debug=False,
dry=False):
# Initialize Kafka Connect REST session
kafka_connect_session = KafkaConnectSession(kafka_rest_url,
kafka_rest_port, debug, dry)
# Issue uninstall for each
for connector in connectors:
kafka_connect_session.uninstall_connector(connector)
# Get remaining connectors
connectors_response = kafka_connect_session.get_connectors()
if not dry:
connectors = connectors_response.json()
else:
connectors = []
pretty_print(connectors, title="Connectors remaining")
def sonar_directory_source(ctx, topic_name, ingest_dir, completed_dir,
avro_schema_file, tasks_max, file_format,
format_options, batch_rows, batch_files,
zookeeper_host, zookeeper_port):
import avro.schema
from sonar_driver.kafka_connect.connector import Connector
from sonar_driver.kafka_connect.sonar_directory_source_config import SonarDirectorySourceConfig
with open(avro_schema_file, 'r') as f:
avro_schema = avro.schema.Parse(f.read())
if ctx.obj['DEBUG']:
pretty_print(avro_schema.to_json(), title="Avro schema")
connector = Connector(
"sonar_directory_source-" + topic_name,
SonarDirectorySourceConfig(topic_name, ingest_dir, completed_dir,
avro_schema, tasks_max, file_format,
format_options, batch_rows, batch_files,
zookeeper_host, zookeeper_port))
if ctx.obj['DEBUG']:
pretty_print(connector.json(), title="Connector")
pretty_print(connector.json(), colorize=False)
def config(ctx, connector_name, a, config_cmds):
if a:
ctx.obj['KafkaConnectors'] = ctx.obj[
'KafkaConnectSession'].get_connectors().json()
else:
ctx.obj['KafkaConnectors'] = [connector_name]
set_vars = [s.split('=') for s in config_cmds]
if not all([len(s) == 2 for s in set_vars]):
raise ValueError("Invalid config command(s), must be key=value")
for connector_name in ctx.obj['KafkaConnectors']:
c = ctx.obj['KafkaConnectSession'].get_connector_config(connector_name)
if not ctx.obj['DRY']:
c = c.json()
if len(set_vars) > 0:
for v in set_vars:
c[v[0]] = v[1]
ctx.obj['KafkaConnectSession'].set_connector_config(
connector_name, c)
else:
pretty_print(c)
def table_exists(self, keyspace, table):
exists_query = "SELECT table_name FROM system_schema.tables WHERE keyspace_name='{}' AND table_name='{}'".format(keyspace, table)
if self.debug:
pretty_print(exists_query, title="Check for Cassandra table CQL", lexer=lexers.SqlLexer())
if not self.dry:
try:
results = self.session.execute(exists_query)
except AuthenticationFailed:
raise Exception("Cassandra user '{}' unauthorized to view system_schema.tables on hosts '{}'!".format(self.username, self.hosts))
if self.debug:
pretty_print(results.current_rows, title="Query results")
if results.current_rows:
return True
else:
return False
else:
return True
def list(ctx):
pretty_print(ctx.obj['KafkaConnectSession'].get_schemas().json())