def test_create_or_update() -> None:
connect = Connect(connect_url="http://localhost:8083")
connect_config = InfluxConfig()
connect_config.update_topics(["t1", "t2", "t3"])
result = connect.create_or_update(name="influxdb-sink",
connect_config=connect_config.asjson())
assert "influxdb-sink" in result
def upload(
ctx: click.Context,
configfile: str,
name: str,
dry_run: bool,
) -> int:
"""Upload the connector configuration from a file."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
with open(configfile) as f:
connect_config = json.load(f)
# Ensure connector name is consistent
connect_config["name"] = name
# Validate the connector configuration only.
if dry_run:
validation = connect.validate(
name=connect_config["connector.class"],
connect_config=json.dumps(connect_config),
)
click.echo(validation)
return 0
click.echo(f"Uploading {name} connector configuration...")
click.echo(connect.validate_and_create(name, json.dumps(connect_config)))
return 0
def test_validate() -> None:
connect = Connect(connect_url="http://localhost:8083")
connect_config = InfluxConfig()
connect_config.update_topics(["t1", "t2", "t3"])
result = connect.validate(name="InfluxSinkConnector",
connect_config=connect_config.asjson())
assert "error_count" in result
def test_expected_exception_get() -> None:
"""Test expected exception."""
with pytest.raises(ValueError):
connect = Connect(connect_url="http://some-url")
connect._request(HTTPMethod.GET,
uri="http://some-url",
data='{"foo": "bar"}')
def test_integration_broker_connect(ensure_broker_service: Fixture,
ensure_connect_service: Fixture) -> None:
"""Test kafkaconnect with a Kafka broker and Kafka Connect.
pytest-docker uses the docker-compose.yaml in the test directory.
"""
broker_url = Config.broker_url
admin_client = AdminClient({"bootstrap.servers": broker_url})
t1 = NewTopic(topic="test.t1", num_partitions=1)
t2 = NewTopic(topic="test.t2", num_partitions=1)
t3 = NewTopic(topic="test.t3", num_partitions=1)
# Create test topics in Kafka
try:
admin_client.create_topics([t1, t2, t3])
time.sleep(1)
except KafkaException:
return None
# Test topic discovery
topic = Topic(broker_url=broker_url,
topic_regex="test.*",
excluded_topics="test.t1")
assert "test.t2" in topic.names
assert "test.t3" in topic.names
# Configure the connector
connect = Connect(connect_url=Config.connect_url)
connect_config = InfluxConfig()
connect_config.update_topics(topic.names)
# Create the connector using the Kafka Connect API
connect.create_or_update(name="influxdb-sink",
connect_config=connect_config.asjson())
# List connectors from the Kafka Connect API
list = connect.list()
assert "influxdb-sink" in list
def create_s3_sink(
ctx: click.Context,
configfile: str,
name: str,
aws_access_key_id: str,
aws_secret_access_key: str,
dry_run: bool,
show_status: bool,
show_status_interval: int,
) -> int:
"""Create an instance of the S3 Sink connector.
Use the --show-status option to output status.
"""
# Get configuration from the parent command
if ctx.parent:
parent_config = ctx.parent.obj["config"]
connect = Connect(parent_config.connect_url)
with open(configfile) as f:
config = json.load(f)
if name:
click.echo("Updating connector name.")
config["name"] = name
if None in (aws_access_key_id, aws_secret_access_key):
click.echo(
"Could not get the AWS credentials. "
"Use the --access-key-id and --aws-secret-access-key options "
"or set the AWS credentials using the AWS_ACCESS_KEY_ID and "
"AWS_SECRET_ACCESS_KEY env variables.")
return 1
config["aws.access.key.id"] = aws_access_key_id
config["aws.secret.access.key"] = aws_secret_access_key
# Validate the configuration only.
if dry_run:
validation = connect.validate(
name=config["connector.class"],
connect_config=json.dumps(config),
)
click.echo(validation)
return 0
name = config["name"]
click.echo(f"Creating the {name} connector...")
click.echo(connect.validate_and_create(name, json.dumps(config)))
if show_status:
while True:
time.sleep(int(show_status_interval) / 1000)
try:
click.echo(connect.status(name=name))
except KeyboardInterrupt:
raise click.ClickException("Interruped.")
return 0
def delete(ctx: click.Context, name: str) -> None:
"""Delete a connector.
Halt tasks and remove the connector configuration.
"""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.remove(name))
def test_integration_broker_connect(ensure_broker_service: Fixture,
ensure_connect_service: Fixture) -> None:
"""Test kafkaconnect with a Kafka broker and Kafka Connect.
pytest-docker uses the docker-compose.yaml in the test directory.
"""
admin_client = AdminClient({"bootstrap.servers": BROKER_URL})
t1 = NewTopic(topic="test.t1", num_partitions=1)
t2 = NewTopic(topic="test.t2", num_partitions=1)
t3 = NewTopic(topic="test.t3", num_partitions=1)
# Create test topics in Kafka
try:
admin_client.create_topics([t1, t2, t3])
time.sleep(5)
except KafkaException:
return None
# Test topic discovery
topic = Topic(
broker_url=BROKER_URL,
topic_regex="test.*",
excluded_topic_regex="test.t1",
)
assert "test.t1" not in topic.names
assert "test.t2" in topic.names
assert "test.t3" in topic.names
# Configure the connector
connect = Connect(connect_url=CONNECT_URL)
connect_config = InfluxConfig(
name="influxdb-sink",
connect_influx_url="http://localhost:8086",
connect_influx_db="mydb",
tasks_max=1,
connect_influx_username="******",
connect_influx_password="******",
connect_influx_error_policy="foo",
connect_influx_max_retries="1",
connect_influx_retry_interval="1",
connect_progress_enabled=True,
)
connect_config.update_topics(topic.names)
# Create the connector using the Kafka Connect API
connect.create_or_update(name="influxdb-sink",
connect_config=connect_config.asjson())
# List connectors from the Kafka Connect API
list = connect.list()
assert "influxdb-sink" in list
def create_jdbc_sink(
ctx: click.Context,
configfile: str,
name: str,
dry_run: bool,
show_status: bool,
show_status_interval: int,
) -> int:
"""Create an instance of the JDBC Sink connector.
Use the --show-status option to output status.
"""
# Get configuration from the parent command
if ctx.parent:
parent_config = ctx.parent.obj["config"]
connect = Connect(parent_config.connect_url)
with open(configfile) as f:
config = json.load(f)
# Override connector name in the configuration
if name:
config["name"] = name
# Validate the configuration only.
if dry_run:
validation = connect.validate(
name=config["connector.class"],
connect_config=json.dumps(config),
)
click.echo(validation)
return 0
name = config["name"]
click.echo(f"Creating the {name} connector...")
click.echo(connect.validate_and_create(name, json.dumps(config)))
if show_status:
while True:
time.sleep(int(show_status_interval) / 1000)
try:
click.echo(connect.status(name=name))
except KeyboardInterrupt:
raise click.ClickException("Interruped.")
return 0
def test_create_or_update() -> None:
"""Test create_or_update method."""
connect = Connect(connect_url="http://localhost:8083")
connect_config = InfluxConfig(
name="influxdb-sink",
connect_influx_url="http://localhost:8086",
connect_influx_db="mydb",
tasks_max=1,
connect_influx_username="******",
connect_influx_password="******",
connect_influx_error_policy="foo",
connect_influx_max_retries="1",
connect_influx_retry_interval="1",
connect_progress_enabled=True,
)
connect_config.update_topics(["t1", "t2", "t3"])
result = connect.create_or_update(name="influxdb-sink",
connect_config=connect_config.asjson())
assert "influxdb-sink" in result
def pause(ctx: click.Context, name: str) -> None:
"""Pause the connector and its tasks."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.pause(name))
def restart(ctx: click.Context, name: str) -> None:
"""Restart a connector and its tasks."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.restart(name))
def plugins(ctx: click.Context) -> None:
"""Get a list of connector plugins available in the Connect cluster."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.plugins())
def topics(ctx: click.Context, name: str) -> None:
"""Get the list of topic names used by the connector."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.topics(name))
def test_list() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.list()
assert "influxdb-sink" in result
def config(ctx: click.Context, name: str) -> None:
"""Get the connector configuration."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.config(name))
def info(ctx: click.Context, name: str) -> None:
"""Get information about the connector."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.info(name))
def test_remove() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.remove("influxdb-sink")
assert result == ""
def create_influxdb_sink(
ctx: click.Context,
topiclist: tuple,
name: str,
connect_influx_url: str,
connect_influx_db: str,
tasks_max: str,
connect_influx_username: str,
connect_influx_password: str,
topic_regex: str,
dry_run: bool,
auto_update: bool,
validate: bool,
check_interval: str,
excluded_topic_regex: str,
connect_influx_error_policy: str,
connect_influx_max_retries: str,
connect_influx_retry_interval: str,
connect_progress_enabled: str,
timestamp: str,
) -> int:
"""Create an instance of the InfluxDB Sink connector.
A list of topics can be specified using the TOPICLIST argument.
If not, topics are discovered from Kafka. Use the ``--topic-regex`` and
``--excluded_topics`` options to help in selecting the topics
that you want to write to InfluxDB. To check for new topics and update
the connector configuration use the
``--auto-update`` and ``--check-interval`` options.
"""
# Get configuration from the main command
if ctx.parent:
config = ctx.parent.obj["config"]
# Connector configuration
influx_config = InfluxConfig(
name=name,
connect_influx_url=connect_influx_url,
connect_influx_db=connect_influx_db,
tasks_max=int(tasks_max),
connect_influx_username=connect_influx_username,
connect_influx_password=connect_influx_password,
connect_influx_error_policy=connect_influx_error_policy,
connect_influx_max_retries=connect_influx_max_retries,
connect_influx_retry_interval=connect_influx_retry_interval,
connect_progress_enabled=(connect_progress_enabled == "true"),
)
# The variadic argument is a tuple
topics: List[str] = list(topiclist)
if not topics:
click.echo("Discoverying Kafka topics...")
topics = Topic(config.broker_url, topic_regex,
excluded_topic_regex).names
n = 0 if not topics else len(topics)
click.echo(f"Found {n} topics.")
connect = Connect(connect_url=config.connect_url)
if topics:
influx_config.update_topics(topics, timestamp)
# --validate option
if validate:
click.echo(
connect.validate(
name=influx_config.connector_class,
connect_config=influx_config.asjson(),
))
return 0
# --dry-run option returns the connector configuration
if dry_run:
click.echo(influx_config.asjson())
return 0
# Validate configuration before creating the connector
validation = connect.validate(
name=influx_config.connector_class,
connect_config=influx_config.asjson(),
)
try:
error_count = json.loads(validation)["error_count"]
click.echo(f"Validation returned {error_count} error(s).")
if error_count > 0:
click.echo(
"Use the ``--validate`` option to return the validation "
"results.")
return 1
except Exception:
click.echo(validation)
return 1
click.echo(f"Uploading {name} connector configuration...")
connect.create_or_update(name=name,
connect_config=influx_config.asjson())
if auto_update:
while True:
time.sleep(int(check_interval) / 1000)
try:
# Current list of topics from Kafka
current_topics = Topic(config.broker_url, topic_regex,
excluded_topic_regex).names
new_topics = list(set(current_topics) - set(topics))
if new_topics:
click.echo("Found new topics, updating the connector...")
influx_config.update_topics(current_topics, timestamp)
connect.create_or_update(
name=name, connect_config=influx_config.asjson())
topics = current_topics
except KeyboardInterrupt:
raise click.ClickException("Interruped.")
return 0
def test_topics() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.topics("influxdb-sink")
# Kafka Connect 5.3.1
assert "connectors/influxdb-sink/topics not found." in result
def test_tasks() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.tasks("influxdb-sink")
assert '"task": 0' in result
def test_config() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.config("influxdb-sink")
assert '"name": "influxdb-sink"' in result
def test_status() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.status("influxdb-sink")
assert "RUNNING" in result
def resume(ctx: click.Context, name: str) -> None:
"""Resume a paused connector."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.resume(name))
def list(ctx: click.Context) -> None:
"""Get a list of active connectors."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.list())
def create_mirrormaker2(
ctx: click.Context,
name: str,
heartbeat_configfile: str,
checkpoint_configfile: str,
mirror_source_configfile: str,
dry_run: bool,
show_status: bool,
show_status_interval: int,
) -> int:
"""Create an instance of the MirrorMaker 2 connectors.
Create the heartbeat, checkpoint and mirror-source
connectors. Use the --show-status option to output status.
"""
# Get configuration from the main command
if ctx.parent:
config = ctx.parent.obj["config"]
connect = Connect(config.connect_url)
with open(heartbeat_configfile) as f:
heartbeat_config = json.load(f)
with open(checkpoint_configfile) as f:
checkpoint_config = json.load(f)
with open(mirror_source_configfile) as f:
mirror_source_config = json.load(f)
# Override connector name in the configuration
if name:
heartbeat_config["name"] = f"{name}-heartbeat"
checkpoint_config["name"] = f"{name}-checkpoint"
mirror_source_config["name"] = f"{name}-mirror-source"
# Validate the configuration only.
if dry_run:
heartbeat_validation = connect.validate(
name=heartbeat_config["connector.class"],
connect_config=json.dumps(heartbeat_config),
)
click.echo(heartbeat_validation)
checkpoint_validation = connect.validate(
name=checkpoint_config["connector.class"],
connect_config=json.dumps(checkpoint_config),
)
click.echo(checkpoint_validation)
mirror_source_validation = connect.validate(
name=mirror_source_config["connector.class"],
connect_config=json.dumps(mirror_source_config),
)
click.echo(mirror_source_validation)
return 0
heartbeat_name = heartbeat_config["name"]
click.echo(f"Creating the {heartbeat_name} connector...")
click.echo(
connect.validate_and_create(heartbeat_name,
json.dumps(heartbeat_config)))
checkpoint_name = checkpoint_config["name"]
click.echo(f"Creating the {checkpoint_name} connector...")
click.echo(
connect.validate_and_create(checkpoint_name,
json.dumps(checkpoint_config)))
mirror_source_name = mirror_source_config["name"]
click.echo(f"Creating the {mirror_source_name} connector...")
click.echo(
connect.validate_and_create(mirror_source_name,
json.dumps(mirror_source_config)))
if show_status:
while True:
time.sleep(int(show_status_interval) / 1000)
try:
click.echo(connect.status(name=heartbeat_name))
click.echo(connect.status(name=checkpoint_name))
click.echo(connect.status(name=mirror_source_name))
except KeyboardInterrupt:
raise click.ClickException("Interruped.")
return 0
def status(ctx: click.Context, name: str) -> None:
"""Get the connector status."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.status(name))
def tasks(ctx: click.Context, name: str) -> None:
"""Get a list of tasks currently running for the connector."""
config = ctx.obj["config"]
connect = Connect(config.connect_url)
click.echo(connect.tasks(name))
def test_plugins() -> None:
connect = Connect(connect_url="http://localhost:8083")
result = connect.plugins()
assert "InfluxSinkConnector" in result
def test_expected_exception_delete() -> None:
with pytest.raises(ValueError):
connect = Connect(connect_url="http://some-url")
connect._request(HTTPMethod.DELETE,
uri="http://some-url",
data='{"foo": "bar"}')
