def _create_config(self):
try:
self._config = Config.get_instance()
except ConfigNotExistsException:
click.echo(f"No config provided in {config_dir()}")
if ensure_approval(
f"Should a sample file be created in {config_dir()}"):
config_dir().mkdir(exist_ok=True)
copyfile(sample_config_path().as_posix(), config_path())
else:
raise
if ensure_approval("Do you want to modify the config file now?"):
click.edit(filename=config_path().as_posix())
self._config = Config.get_instance()
def __init__(
self,
group_id: str,
topic_name: str,
output_directory: pathlib.Path,
last: bool,
match: str = None,
initialize_default_output_directory: bool = False,
enable_auto_commit: bool = True,
):
super().__init__(
group_id,
topic_name,
output_directory,
last,
match,
initialize_default_output_directory,
enable_auto_commit,
)
self.schema_registry_client = SchemaRegistryClient(
Config.get_instance().schema_registry)
self.writers[-1] = (StdOutAvroWriter(
schema_registry_client=self.schema_registry_client)
if output_directory is None else AvroFileWriter(
self.output_directory /
"partition_any", self.schema_registry_client))
if self._initialize_default_output_directory and self.output_directory is not None:
self.writers[-1].init_destination_directory()
def unittest_config(request: FixtureRequest,
load_config: config_loader) -> Config:
conffile, _ = load_config(LOAD_INTEGRATION_TEST_CONFIG)
esque_config = Config.get_instance()
if request.config.getoption("--local"):
esque_config.context_switch("local")
return esque_config
def __init__(self):
self._config = Config.get_instance()
self.confluent_client = AdminClient(
self._config.create_confluent_config())
self.pykafka_client = pykafka.client.KafkaClient(
**self._config.create_pykafka_config(), broker_version="1.0.0")
self.confluent_client.poll(timeout=1)
self.__topic_controller = None
def commit_offsets(self, consumer_id: str, offsets: List[TopicPartition]):
config = Config.get_instance()
consumer = Consumer({
"group.id": consumer_id,
**config.create_confluent_config()
})
consumer.commit(offsets=offsets, asynchronous=False)
consumer.close()
def _get_partitions(
self,
topic: Topic,
retrieve_last_timestamp: bool,
get_partition_watermarks: bool = True) -> List[Partition]:
assert not (
retrieve_last_timestamp and not get_partition_watermarks
), "Can not retrieve timestamp without partition watermarks"
config = Config.get_instance().create_confluent_config()
config.update({
"group.id": ESQUE_GROUP_ID,
"topic.metadata.refresh.interval.ms": "250"
})
with closing(confluent_kafka.Consumer(config)) as consumer:
confluent_topic = consumer.list_topics(
topic=topic.name).topics[topic.name]
partitions: List[Partition] = []
if not get_partition_watermarks:
return [
Partition(partition_id, -1, -1, meta.isrs, meta.leader,
meta.replicas, None) for partition_id, meta in
confluent_topic.partitions.items()
]
for partition_id, meta in confluent_topic.partitions.items():
try:
low, high = consumer.get_watermark_offsets(
TopicPartition(topic=topic.name,
partition=partition_id))
except KafkaException:
# retry after metadata should be refreshed (also consider small network delays)
# unfortunately we cannot explicitly cause and wait for a metadata refresh
time.sleep(1)
low, high = consumer.get_watermark_offsets(
TopicPartition(topic=topic.name,
partition=partition_id))
latest_timestamp = None
if high > low and retrieve_last_timestamp:
assignment = [
TopicPartition(topic=topic.name,
partition=partition_id,
offset=high - 1)
]
consumer.assign(assignment)
msg = consumer.poll(timeout=10)
if msg is None:
logger.warning(
f"Due to timeout latest timestamp for topic `{topic.name}` "
f"and partition `{partition_id}` is missing.")
else:
latest_timestamp = float(msg.timestamp()[1]) / 1000
partition = Partition(partition_id, low, high, meta.isrs,
meta.leader, meta.replicas,
latest_timestamp)
partitions.append(partition)
return partitions
def test_fix_missing_context_config(interactive_cli_runner: CliRunner,
load_config: config_loader):
load_config(LOAD_BROKEN_CONFIG)
_cfg = Config(disable_validation=True)
assert _cfg.current_context not in _cfg.available_contexts
interactive_cli_runner.invoke(esque,
args=["config", "fix"],
catch_exceptions=False)
_cfg = Config.get_instance()
assert _cfg.current_context in _cfg.available_contexts
def __init__(self, topic_name: str, match: str = None):
self.queue_length = 100000
self.internal_queue_length_limit = self.queue_length / 0.5
self._config = Config.get_instance().create_confluent_config()
self._setup_config()
self.logger = logging.getLogger(__name__)
self._topic_name = topic_name
self._match = match
self._producer = None
if self._match is not None:
self._rule_tree = RuleTree(match)
else:
self._rule_tree = None
self.create_internal_producer()
def _setup_config(self):
offset_reset = "earliest"
if self._last:
offset_reset = "latest"
self._config = Config.get_instance().create_confluent_config()
self._config.update({
"group.id": self._group_id,
"error_cb": log_error,
# We need to commit offsets manually once we"re sure it got saved
# to the sink
"enable.auto.commit": self._enable_auto_commit,
"enable.partition.eof": True,
# We need this to start at the last committed offset instead of the
# latest when subscribing for the first time
"default.topic.config": {
"auto.offset.reset": offset_reset
},
})
def config(config_version: int, load_config: config_loader):
old_conf, _ = load_config(config_version)
new_path, _ = migrate(Path(old_conf))
Config.set_instance(Config())
return Config.get_instance()
def __init__(self):
self._config = Config.get_instance()
self.__topic_controller = None
def reload_config(load_config: config_loader, *, config: int = LOAD_INTEGRATION_TEST_CONFIG):
conffile, _ = load_config(config)
Config.set_instance(Config())
return Config.get_instance()
def broken_test_config(load_config: config_loader) -> Config:
conffile, _ = load_config(LOAD_BROKEN_CONFIG)
esque_config = Config.get_instance()
return esque_config
def _setup_config(self):
super()._setup_config()
self._config.update(
{"schema.registry.url": Config.get_instance().schema_registry})
def __init__(self, cluster: "Cluster", config: Optional[Config] = None):
self.cluster: "Cluster" = cluster
if config is None:
config = Config.get_instance()
self.config = config