def _create_config(self): try: self._config = Config.get_instance() except ConfigNotExistsException: click.echo(f"No config provided in {config_dir()}") if ensure_approval( f"Should a sample file be created in {config_dir()}"): config_dir().mkdir(exist_ok=True) copyfile(sample_config_path().as_posix(), config_path()) else: raise if ensure_approval("Do you want to modify the config file now?"): click.edit(filename=config_path().as_posix()) self._config = Config.get_instance()
def __init__( self, group_id: str, topic_name: str, output_directory: pathlib.Path, last: bool, match: str = None, initialize_default_output_directory: bool = False, enable_auto_commit: bool = True, ): super().__init__( group_id, topic_name, output_directory, last, match, initialize_default_output_directory, enable_auto_commit, ) self.schema_registry_client = SchemaRegistryClient( Config.get_instance().schema_registry) self.writers[-1] = (StdOutAvroWriter( schema_registry_client=self.schema_registry_client) if output_directory is None else AvroFileWriter( self.output_directory / "partition_any", self.schema_registry_client)) if self._initialize_default_output_directory and self.output_directory is not None: self.writers[-1].init_destination_directory()
def unittest_config(request: FixtureRequest, load_config: config_loader) -> Config: conffile, _ = load_config(LOAD_INTEGRATION_TEST_CONFIG) esque_config = Config.get_instance() if request.config.getoption("--local"): esque_config.context_switch("local") return esque_config
def __init__(self): self._config = Config.get_instance() self.confluent_client = AdminClient( self._config.create_confluent_config()) self.pykafka_client = pykafka.client.KafkaClient( **self._config.create_pykafka_config(), broker_version="1.0.0") self.confluent_client.poll(timeout=1) self.__topic_controller = None
def commit_offsets(self, consumer_id: str, offsets: List[TopicPartition]): config = Config.get_instance() consumer = Consumer({ "group.id": consumer_id, **config.create_confluent_config() }) consumer.commit(offsets=offsets, asynchronous=False) consumer.close()
def _get_partitions( self, topic: Topic, retrieve_last_timestamp: bool, get_partition_watermarks: bool = True) -> List[Partition]: assert not ( retrieve_last_timestamp and not get_partition_watermarks ), "Can not retrieve timestamp without partition watermarks" config = Config.get_instance().create_confluent_config() config.update({ "group.id": ESQUE_GROUP_ID, "topic.metadata.refresh.interval.ms": "250" }) with closing(confluent_kafka.Consumer(config)) as consumer: confluent_topic = consumer.list_topics( topic=topic.name).topics[topic.name] partitions: List[Partition] = [] if not get_partition_watermarks: return [ Partition(partition_id, -1, -1, meta.isrs, meta.leader, meta.replicas, None) for partition_id, meta in confluent_topic.partitions.items() ] for partition_id, meta in confluent_topic.partitions.items(): try: low, high = consumer.get_watermark_offsets( TopicPartition(topic=topic.name, partition=partition_id)) except KafkaException: # retry after metadata should be refreshed (also consider small network delays) # unfortunately we cannot explicitly cause and wait for a metadata refresh time.sleep(1) low, high = consumer.get_watermark_offsets( TopicPartition(topic=topic.name, partition=partition_id)) latest_timestamp = None if high > low and retrieve_last_timestamp: assignment = [ TopicPartition(topic=topic.name, partition=partition_id, offset=high - 1) ] consumer.assign(assignment) msg = consumer.poll(timeout=10) if msg is None: logger.warning( f"Due to timeout latest timestamp for topic `{topic.name}` " f"and partition `{partition_id}` is missing.") else: latest_timestamp = float(msg.timestamp()[1]) / 1000 partition = Partition(partition_id, low, high, meta.isrs, meta.leader, meta.replicas, latest_timestamp) partitions.append(partition) return partitions
def test_fix_missing_context_config(interactive_cli_runner: CliRunner, load_config: config_loader): load_config(LOAD_BROKEN_CONFIG) _cfg = Config(disable_validation=True) assert _cfg.current_context not in _cfg.available_contexts interactive_cli_runner.invoke(esque, args=["config", "fix"], catch_exceptions=False) _cfg = Config.get_instance() assert _cfg.current_context in _cfg.available_contexts
def __init__(self, topic_name: str, match: str = None): self.queue_length = 100000 self.internal_queue_length_limit = self.queue_length / 0.5 self._config = Config.get_instance().create_confluent_config() self._setup_config() self.logger = logging.getLogger(__name__) self._topic_name = topic_name self._match = match self._producer = None if self._match is not None: self._rule_tree = RuleTree(match) else: self._rule_tree = None self.create_internal_producer()
def _setup_config(self): offset_reset = "earliest" if self._last: offset_reset = "latest" self._config = Config.get_instance().create_confluent_config() self._config.update({ "group.id": self._group_id, "error_cb": log_error, # We need to commit offsets manually once we"re sure it got saved # to the sink "enable.auto.commit": self._enable_auto_commit, "enable.partition.eof": True, # We need this to start at the last committed offset instead of the # latest when subscribing for the first time "default.topic.config": { "auto.offset.reset": offset_reset }, })
def config(config_version: int, load_config: config_loader): old_conf, _ = load_config(config_version) new_path, _ = migrate(Path(old_conf)) Config.set_instance(Config()) return Config.get_instance()
def __init__(self): self._config = Config.get_instance() self.__topic_controller = None
def reload_config(load_config: config_loader, *, config: int = LOAD_INTEGRATION_TEST_CONFIG): conffile, _ = load_config(config) Config.set_instance(Config()) return Config.get_instance()
def broken_test_config(load_config: config_loader) -> Config: conffile, _ = load_config(LOAD_BROKEN_CONFIG) esque_config = Config.get_instance() return esque_config
def _setup_config(self): super()._setup_config() self._config.update( {"schema.registry.url": Config.get_instance().schema_registry})
def __init__(self, cluster: "Cluster", config: Optional[Config] = None): self.cluster: "Cluster" = cluster if config is None: config = Config.get_instance() self.config = config