def test_create_or_update() -> None: connect = Connect(connect_url="http://localhost:8083") connect_config = InfluxConfig() connect_config.update_topics(["t1", "t2", "t3"]) result = connect.create_or_update(name="influxdb-sink", connect_config=connect_config.asjson()) assert "influxdb-sink" in result
def upload( ctx: click.Context, configfile: str, name: str, dry_run: bool, ) -> int: """Upload the connector configuration from a file.""" config = ctx.obj["config"] connect = Connect(config.connect_url) with open(configfile) as f: connect_config = json.load(f) # Ensure connector name is consistent connect_config["name"] = name # Validate the connector configuration only. if dry_run: validation = connect.validate( name=connect_config["connector.class"], connect_config=json.dumps(connect_config), ) click.echo(validation) return 0 click.echo(f"Uploading {name} connector configuration...") click.echo(connect.validate_and_create(name, json.dumps(connect_config))) return 0
def test_validate() -> None: connect = Connect(connect_url="http://localhost:8083") connect_config = InfluxConfig() connect_config.update_topics(["t1", "t2", "t3"]) result = connect.validate(name="InfluxSinkConnector", connect_config=connect_config.asjson()) assert "error_count" in result
def test_expected_exception_get() -> None: """Test expected exception.""" with pytest.raises(ValueError): connect = Connect(connect_url="http://some-url") connect._request(HTTPMethod.GET, uri="http://some-url", data='{"foo": "bar"}')
def test_integration_broker_connect(ensure_broker_service: Fixture, ensure_connect_service: Fixture) -> None: """Test kafkaconnect with a Kafka broker and Kafka Connect. pytest-docker uses the docker-compose.yaml in the test directory. """ broker_url = Config.broker_url admin_client = AdminClient({"bootstrap.servers": broker_url}) t1 = NewTopic(topic="test.t1", num_partitions=1) t2 = NewTopic(topic="test.t2", num_partitions=1) t3 = NewTopic(topic="test.t3", num_partitions=1) # Create test topics in Kafka try: admin_client.create_topics([t1, t2, t3]) time.sleep(1) except KafkaException: return None # Test topic discovery topic = Topic(broker_url=broker_url, topic_regex="test.*", excluded_topics="test.t1") assert "test.t2" in topic.names assert "test.t3" in topic.names # Configure the connector connect = Connect(connect_url=Config.connect_url) connect_config = InfluxConfig() connect_config.update_topics(topic.names) # Create the connector using the Kafka Connect API connect.create_or_update(name="influxdb-sink", connect_config=connect_config.asjson()) # List connectors from the Kafka Connect API list = connect.list() assert "influxdb-sink" in list
def create_s3_sink( ctx: click.Context, configfile: str, name: str, aws_access_key_id: str, aws_secret_access_key: str, dry_run: bool, show_status: bool, show_status_interval: int, ) -> int: """Create an instance of the S3 Sink connector. Use the --show-status option to output status. """ # Get configuration from the parent command if ctx.parent: parent_config = ctx.parent.obj["config"] connect = Connect(parent_config.connect_url) with open(configfile) as f: config = json.load(f) if name: click.echo("Updating connector name.") config["name"] = name if None in (aws_access_key_id, aws_secret_access_key): click.echo( "Could not get the AWS credentials. " "Use the --access-key-id and --aws-secret-access-key options " "or set the AWS credentials using the AWS_ACCESS_KEY_ID and " "AWS_SECRET_ACCESS_KEY env variables.") return 1 config["aws.access.key.id"] = aws_access_key_id config["aws.secret.access.key"] = aws_secret_access_key # Validate the configuration only. if dry_run: validation = connect.validate( name=config["connector.class"], connect_config=json.dumps(config), ) click.echo(validation) return 0 name = config["name"] click.echo(f"Creating the {name} connector...") click.echo(connect.validate_and_create(name, json.dumps(config))) if show_status: while True: time.sleep(int(show_status_interval) / 1000) try: click.echo(connect.status(name=name)) except KeyboardInterrupt: raise click.ClickException("Interruped.") return 0
def delete(ctx: click.Context, name: str) -> None: """Delete a connector. Halt tasks and remove the connector configuration. """ config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.remove(name))
def test_integration_broker_connect(ensure_broker_service: Fixture, ensure_connect_service: Fixture) -> None: """Test kafkaconnect with a Kafka broker and Kafka Connect. pytest-docker uses the docker-compose.yaml in the test directory. """ admin_client = AdminClient({"bootstrap.servers": BROKER_URL}) t1 = NewTopic(topic="test.t1", num_partitions=1) t2 = NewTopic(topic="test.t2", num_partitions=1) t3 = NewTopic(topic="test.t3", num_partitions=1) # Create test topics in Kafka try: admin_client.create_topics([t1, t2, t3]) time.sleep(5) except KafkaException: return None # Test topic discovery topic = Topic( broker_url=BROKER_URL, topic_regex="test.*", excluded_topic_regex="test.t1", ) assert "test.t1" not in topic.names assert "test.t2" in topic.names assert "test.t3" in topic.names # Configure the connector connect = Connect(connect_url=CONNECT_URL) connect_config = InfluxConfig( name="influxdb-sink", connect_influx_url="http://localhost:8086", connect_influx_db="mydb", tasks_max=1, connect_influx_username="******", connect_influx_password="******", connect_influx_error_policy="foo", connect_influx_max_retries="1", connect_influx_retry_interval="1", connect_progress_enabled=True, ) connect_config.update_topics(topic.names) # Create the connector using the Kafka Connect API connect.create_or_update(name="influxdb-sink", connect_config=connect_config.asjson()) # List connectors from the Kafka Connect API list = connect.list() assert "influxdb-sink" in list
def create_jdbc_sink( ctx: click.Context, configfile: str, name: str, dry_run: bool, show_status: bool, show_status_interval: int, ) -> int: """Create an instance of the JDBC Sink connector. Use the --show-status option to output status. """ # Get configuration from the parent command if ctx.parent: parent_config = ctx.parent.obj["config"] connect = Connect(parent_config.connect_url) with open(configfile) as f: config = json.load(f) # Override connector name in the configuration if name: config["name"] = name # Validate the configuration only. if dry_run: validation = connect.validate( name=config["connector.class"], connect_config=json.dumps(config), ) click.echo(validation) return 0 name = config["name"] click.echo(f"Creating the {name} connector...") click.echo(connect.validate_and_create(name, json.dumps(config))) if show_status: while True: time.sleep(int(show_status_interval) / 1000) try: click.echo(connect.status(name=name)) except KeyboardInterrupt: raise click.ClickException("Interruped.") return 0
def test_create_or_update() -> None: """Test create_or_update method.""" connect = Connect(connect_url="http://localhost:8083") connect_config = InfluxConfig( name="influxdb-sink", connect_influx_url="http://localhost:8086", connect_influx_db="mydb", tasks_max=1, connect_influx_username="******", connect_influx_password="******", connect_influx_error_policy="foo", connect_influx_max_retries="1", connect_influx_retry_interval="1", connect_progress_enabled=True, ) connect_config.update_topics(["t1", "t2", "t3"]) result = connect.create_or_update(name="influxdb-sink", connect_config=connect_config.asjson()) assert "influxdb-sink" in result
def pause(ctx: click.Context, name: str) -> None: """Pause the connector and its tasks.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.pause(name))
def restart(ctx: click.Context, name: str) -> None: """Restart a connector and its tasks.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.restart(name))
def plugins(ctx: click.Context) -> None: """Get a list of connector plugins available in the Connect cluster.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.plugins())
def topics(ctx: click.Context, name: str) -> None: """Get the list of topic names used by the connector.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.topics(name))
def test_list() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.list() assert "influxdb-sink" in result
def config(ctx: click.Context, name: str) -> None: """Get the connector configuration.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.config(name))
def info(ctx: click.Context, name: str) -> None: """Get information about the connector.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.info(name))
def test_remove() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.remove("influxdb-sink") assert result == ""
def create_influxdb_sink( ctx: click.Context, topiclist: tuple, name: str, connect_influx_url: str, connect_influx_db: str, tasks_max: str, connect_influx_username: str, connect_influx_password: str, topic_regex: str, dry_run: bool, auto_update: bool, validate: bool, check_interval: str, excluded_topic_regex: str, connect_influx_error_policy: str, connect_influx_max_retries: str, connect_influx_retry_interval: str, connect_progress_enabled: str, timestamp: str, ) -> int: """Create an instance of the InfluxDB Sink connector. A list of topics can be specified using the TOPICLIST argument. If not, topics are discovered from Kafka. Use the ``--topic-regex`` and ``--excluded_topics`` options to help in selecting the topics that you want to write to InfluxDB. To check for new topics and update the connector configuration use the ``--auto-update`` and ``--check-interval`` options. """ # Get configuration from the main command if ctx.parent: config = ctx.parent.obj["config"] # Connector configuration influx_config = InfluxConfig( name=name, connect_influx_url=connect_influx_url, connect_influx_db=connect_influx_db, tasks_max=int(tasks_max), connect_influx_username=connect_influx_username, connect_influx_password=connect_influx_password, connect_influx_error_policy=connect_influx_error_policy, connect_influx_max_retries=connect_influx_max_retries, connect_influx_retry_interval=connect_influx_retry_interval, connect_progress_enabled=(connect_progress_enabled == "true"), ) # The variadic argument is a tuple topics: List[str] = list(topiclist) if not topics: click.echo("Discoverying Kafka topics...") topics = Topic(config.broker_url, topic_regex, excluded_topic_regex).names n = 0 if not topics else len(topics) click.echo(f"Found {n} topics.") connect = Connect(connect_url=config.connect_url) if topics: influx_config.update_topics(topics, timestamp) # --validate option if validate: click.echo( connect.validate( name=influx_config.connector_class, connect_config=influx_config.asjson(), )) return 0 # --dry-run option returns the connector configuration if dry_run: click.echo(influx_config.asjson()) return 0 # Validate configuration before creating the connector validation = connect.validate( name=influx_config.connector_class, connect_config=influx_config.asjson(), ) try: error_count = json.loads(validation)["error_count"] click.echo(f"Validation returned {error_count} error(s).") if error_count > 0: click.echo( "Use the ``--validate`` option to return the validation " "results.") return 1 except Exception: click.echo(validation) return 1 click.echo(f"Uploading {name} connector configuration...") connect.create_or_update(name=name, connect_config=influx_config.asjson()) if auto_update: while True: time.sleep(int(check_interval) / 1000) try: # Current list of topics from Kafka current_topics = Topic(config.broker_url, topic_regex, excluded_topic_regex).names new_topics = list(set(current_topics) - set(topics)) if new_topics: click.echo("Found new topics, updating the connector...") influx_config.update_topics(current_topics, timestamp) connect.create_or_update( name=name, connect_config=influx_config.asjson()) topics = current_topics except KeyboardInterrupt: raise click.ClickException("Interruped.") return 0
def test_topics() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.topics("influxdb-sink") # Kafka Connect 5.3.1 assert "connectors/influxdb-sink/topics not found." in result
def test_tasks() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.tasks("influxdb-sink") assert '"task": 0' in result
def test_config() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.config("influxdb-sink") assert '"name": "influxdb-sink"' in result
def test_status() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.status("influxdb-sink") assert "RUNNING" in result
def resume(ctx: click.Context, name: str) -> None: """Resume a paused connector.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.resume(name))
def list(ctx: click.Context) -> None: """Get a list of active connectors.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.list())
def create_mirrormaker2( ctx: click.Context, name: str, heartbeat_configfile: str, checkpoint_configfile: str, mirror_source_configfile: str, dry_run: bool, show_status: bool, show_status_interval: int, ) -> int: """Create an instance of the MirrorMaker 2 connectors. Create the heartbeat, checkpoint and mirror-source connectors. Use the --show-status option to output status. """ # Get configuration from the main command if ctx.parent: config = ctx.parent.obj["config"] connect = Connect(config.connect_url) with open(heartbeat_configfile) as f: heartbeat_config = json.load(f) with open(checkpoint_configfile) as f: checkpoint_config = json.load(f) with open(mirror_source_configfile) as f: mirror_source_config = json.load(f) # Override connector name in the configuration if name: heartbeat_config["name"] = f"{name}-heartbeat" checkpoint_config["name"] = f"{name}-checkpoint" mirror_source_config["name"] = f"{name}-mirror-source" # Validate the configuration only. if dry_run: heartbeat_validation = connect.validate( name=heartbeat_config["connector.class"], connect_config=json.dumps(heartbeat_config), ) click.echo(heartbeat_validation) checkpoint_validation = connect.validate( name=checkpoint_config["connector.class"], connect_config=json.dumps(checkpoint_config), ) click.echo(checkpoint_validation) mirror_source_validation = connect.validate( name=mirror_source_config["connector.class"], connect_config=json.dumps(mirror_source_config), ) click.echo(mirror_source_validation) return 0 heartbeat_name = heartbeat_config["name"] click.echo(f"Creating the {heartbeat_name} connector...") click.echo( connect.validate_and_create(heartbeat_name, json.dumps(heartbeat_config))) checkpoint_name = checkpoint_config["name"] click.echo(f"Creating the {checkpoint_name} connector...") click.echo( connect.validate_and_create(checkpoint_name, json.dumps(checkpoint_config))) mirror_source_name = mirror_source_config["name"] click.echo(f"Creating the {mirror_source_name} connector...") click.echo( connect.validate_and_create(mirror_source_name, json.dumps(mirror_source_config))) if show_status: while True: time.sleep(int(show_status_interval) / 1000) try: click.echo(connect.status(name=heartbeat_name)) click.echo(connect.status(name=checkpoint_name)) click.echo(connect.status(name=mirror_source_name)) except KeyboardInterrupt: raise click.ClickException("Interruped.") return 0
def status(ctx: click.Context, name: str) -> None: """Get the connector status.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.status(name))
def tasks(ctx: click.Context, name: str) -> None: """Get a list of tasks currently running for the connector.""" config = ctx.obj["config"] connect = Connect(config.connect_url) click.echo(connect.tasks(name))
def test_plugins() -> None: connect = Connect(connect_url="http://localhost:8083") result = connect.plugins() assert "InfluxSinkConnector" in result
def test_expected_exception_delete() -> None: with pytest.raises(ValueError): connect = Connect(connect_url="http://some-url") connect._request(HTTPMethod.DELETE, uri="http://some-url", data='{"foo": "bar"}')