def __init__(self, command_topic_url: str):
"""
Constructor.
:param command_topic_url: The url of the Kafka topic to where the file-writer status/command messages are published.
"""
kafka_address = KafkaTopicUrl(command_topic_url)
self.status_queue = Queue()
self.to_thread_queue = Queue()
thread_kwargs = {
"host_port": kafka_address.host_port,
"topic": kafka_address.topic,
"in_queue": self.to_thread_queue,
"out_queue": self.status_queue,
}
self.map_of_workers: Dict[str, WorkerStatus] = {}
self.map_of_jobs: Dict[str, JobStatus] = {}
self.map_of_commands: Dict[str, CommandStatus] = {}
self.run_thread = True
self.thread = threading.Thread(target=thread_function,
daemon=True,
kwargs=thread_kwargs)
self.thread.start()
def do_exit():
self.stop_thread()
atexit.register(do_exit)
def __init__(self, command_topic_url: str):
temp_cmd_ch = CommandChannel(command_topic_url)
command_url = KafkaTopicUrl(command_topic_url)
try:
temp_producer = KafkaProducer(
bootstrap_servers=[command_url.host_port])
except NoBrokersAvailable as e:
raise NoBrokersAvailable(
f'Unable to find brokers (or connect to brokers) on address: "{command_url.host_port}"'
) from e
super().__init__(command_url.topic, temp_cmd_ch, temp_producer)
def __init__(self, job_topic_url: str, command_topic_url: str):
"""
:param job_topic_url: The Kafka topic that the available file-writers are listening to for write jobs.
:param command_topic_url: The Kafka topic that a file-writer uses to send status updates to and receive direct
commands from.
"""
super().__init__(command_topic_url)
self._job_pool = KafkaTopicUrl(job_topic_url)
try:
self._pool_producer = KafkaProducer(
bootstrap_servers=[self._job_pool.host_port])
except NoBrokersAvailable as e:
raise NoBrokersAvailable(
f'Unable to find brokers (or connect to brokers) on address: "{self._job_pool.host_port}"'
) from e
def test_failure_8():
with pytest.raises(RuntimeError):
KafkaTopicUrl("kafka://what.com:1234/hej/again")
def test_failure_7():
with pytest.raises(RuntimeError):
KafkaTopicUrl("kafka://what.com:sfa/hej")
def test_failure_6():
with pytest.raises(RuntimeError):
KafkaTopicUrl("//what.com:124/hej")
def test_success_1():
url_obj = KafkaTopicUrl("kafka://addr:9012/hello_12")
assert url_obj.host_port == "addr:9012"
assert url_obj.topic == "hello_12"
def test_failure_2():
with pytest.raises(RuntimeError):
KafkaTopicUrl("kafka://what.com/")
def test_failure_1():
with pytest.raises(RuntimeError):
KafkaTopicUrl("kafka://:9012/hello_12")
def test_success_5():
url_obj = KafkaTopicUrl("192.168.1.21:9012/hello_12")
assert url_obj.host_port == "192.168.1.21:9012"
assert url_obj.topic == "hello_12"
def test_success_4():
url_obj = KafkaTopicUrl("addr.se:9012/hello_12")
assert url_obj.host_port == "addr.se:9012"
assert url_obj.topic == "hello_12"