def post_messages(self):
messages = [str(x) for x in range(100)]
redis_instance = RedisInterface("Producer")
for message in messages:
status = self.producer_instance.enqueue(message)
while not status:
status = self.producer_instance.enqueue(message)
event = "Producer: Successfully posted a message = {} into Kafka.".format(message)
redis_instance.write_an_event_in_redis_db(event)
redis_instance.increment_enqueue_count()
self.producer_instance.cleanup()
return True
class DirectoryWatch:
def __init__(self):
self.before={}
self.after={}
self.video_file_path = None
self.producer_consumer_type = None
self.redis_instance = RedisInterface("Producer")
self.load_environment_variables()
self.producer_instance = ProducerConsumerAPI(is_producer=True,
thread_identifier="Producer",
type_of_messaging_queue=self.producer_consumer_type)
def load_environment_variables(self):
while self.video_file_path is None or \
self.producer_consumer_type is None:
time.sleep(1)
self.video_file_path = os.getenv("video_file_path_key", default=None)
self.producer_consumer_type = os.getenv("producer_consumer_queue_type_key", default=None)
logging_to_console_and_syslog(("video_file_path={}".format(self.video_file_path)))
logging_to_console_and_syslog(("producer_consumer_type={}".format(self.producer_consumer_type)))
def cleanup(self):
self.producer_instance.cleanup()
def process_new_file(self,file_name):
# post the file_name into the producer queue.
self.producer_instance.enqueue(file_name)
event = "Producer: Successfully posted a message = {} into msgQ.".format(file_name)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
def watch_a_directory(self):
self.before = {}
while True:
time.sleep(1)
self.after = dict([(f, None) for f in os.listdir(self.video_file_path)])
added = [f for f in self.after if not f in self.before]
removed = [f for f in self.before if not f in self.after]
if added:
logging_to_console_and_syslog("Added: " + str(added))
for filename in added:
self.process_new_file(filename)
if removed:
logging_to_console_and_syslog("Removed: " + str(removed))
self.before = self.after
class ProducerConsumerAPI:
"""
This is a factory design pattern.
This class produces messages into
1. Kafka Queue.
2. Rabbit Message Queue.
"""
rabbitMsgQType = "Rabbit"
kafkaMsgQType = "Kafka"
confluentKafkaMsgQType = "ConfluentKafka"
def __init__(self,
is_producer=False,
is_consumer=False,
perform_subscription=False,
type_of_messaging_queue=None,
thread_identifier=None):
self.message_queue_instance = None
self.redis_instance = None
self.is_producer = is_producer
self.is_consumer = is_consumer
self.perform_subscription = perform_subscription
self.type_of_messaging_queue = type_of_messaging_queue
self.thread_identifier = thread_identifier
self.read_environment_variables()
#self.__connect()
def read_environment_variables(self):
"""
This method is used to read the environment variables defined in the OS.
:return:
"""
while self.type_of_messaging_queue is None:
time.sleep(2)
logging_to_console_and_syslog(
"ProducerConsumerAPI: "
"Trying to read the environment variables...")
self.type_of_messaging_queue = os.getenv(
"type_of_messaging_queue_key", default=None)
logging_to_console_and_syslog("ProducerConsumerAPI:"
"type_of_messaging_queue={}".format(
self.type_of_messaging_queue))
def __connect(self):
"""
This method tries to connect to the messaging queue.
:return:
"""
if self.message_queue_instance is None:
try:
if self.type_of_messaging_queue == ProducerConsumerAPI.kafkaMsgQType:
self.message_queue_instance = KafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.rabbitMsgQType:
self.message_queue_instance = RabbitMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.confluentKafkaMsgQType:
self.message_queue_instance = ConfluentKafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
if not self.redis_instance:
if self.is_producer:
self.redis_instance = RedisInterface(
"Producer{}".format(self.thread_identifier))
elif self.is_consumer:
self.redis_instance = RedisInterface(
"Consumer{}".format(self.thread_identifier))
except:
print("Exception in user code:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
time.sleep(5)
else:
logging_to_console_and_syslog(
"ProducerConsumerAPI: Successfully "
"created producer instance for messageQ type ={}".format(
self.type_of_messaging_queue))
def enqueue(self, filename):
"""
This method tries to post a message.
:param filename:
:return True or False:
"""
status = False
if filename is None or len(filename) == 0:
logging_to_console_and_syslog("filename is None or invalid")
return status
if self.message_queue_instance is None:
self.__connect()
if hasattr(self.message_queue_instance, 'enqueue'):
status = self.message_queue_instance.enqueue(filename)
event = "Producer: Successfully posted a message = {} into msgQ. Status={}".format(
filename, status)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
return status
def dequeue(self):
"""
This method tries to post a message.
:return Freezes the current context and yeilds a message:
Please make sure to iterate this over to unfreeze the context.
"""
if self.message_queue_instance is None:
self.__connect()
msg = None
if hasattr(self.message_queue_instance, 'dequeue'):
msg = self.message_queue_instance.dequeue()
if msg:
self.redis_instance.increment_dequeue_count()
self.redis_instance.write_an_event_in_redis_db(
"Consumer {}: Dequeued Message = {}".format(
self.thread_identifier, msg))
self.cleanup()
return msg
def cleanup(self):
if self.message_queue_instance:
self.message_queue_instance.cleanup()
self.message_queue_instance = None
class TestMachineLearningWorkers(unittest.TestCase):
max_number_of_jobs = 100
directory_name = 'test_files'
def setUp(self):
os.environ["broker_name_key"] = "{}:9094".format("mec-poc")
os.environ["topic_key"] = "video-file-name"
os.environ["redis_log_keyname_key"] = "briefcam"
os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue"
os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue"
os.environ["redis_server_hostname_key"] = "mec-poc"
os.environ["redis_server_port_key"] = "6379"
os.environ[
"producer_consumer_queue_type_key"] = ProducerConsumerAPI.kafkaMsgQType
os.environ["data_parser_type_key"] = DataParserInterface.TensorFlow
self.dirname = os.path.dirname(os.path.realpath(__file__))
current_file_path_list = os.path.realpath(__file__).split('/')
data_path_directory = '/'.join(current_file_path_list[:-1])
os.environ["data_file_path_key"] = "{}/{}".format(
data_path_directory, TestMachineLearningWorkers.directory_name)
self.create_test_docker_container()
self.machine_learning_worker_thread = None
self.producer_instance = None
@staticmethod
def start_machine_learning_workers():
logging_to_console_and_syslog("Starting {}".format(
threading.current_thread().getName()))
t = threading.currentThread()
worker = MachineLearningWorker()
while getattr(t, "do_run", True):
#logging_to_console_and_syslog("***dequeue_and_process_jobs***")
worker.dequeue_and_process_jobs()
logging_to_console_and_syslog("Consumer {}: Exiting".format(
threading.current_thread().getName()))
def create_machine_learning_worker_thread(self):
self.machine_learning_worker_thread = \
threading.Thread(name="{}{}".format("thread", 1),
target=TestMachineLearningWorkers.start_machine_learning_workers
)
self.machine_learning_worker_thread.do_run = True
self.machine_learning_worker_thread.name = "{}_{}".format(
"test_machine_learning_workers", 1)
self.machine_learning_worker_thread.start()
def post_messages(self):
self.redis_instance = RedisInterface("Producer")
messages = [
str(x)
for x in range(TestMachineLearningWorkers.max_number_of_jobs)
]
for message in messages:
self.producer_instance.enqueue(message)
event = "Producer: Successfully posted a message = {} into msgQ.".format(
message)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
return True
def create_producer_and_produce_jobs(self, msgq_type):
self.producer_instance = ProducerConsumerAPI(
is_producer=True,
thread_identifier="Producer",
type_of_messaging_queue=msgq_type)
logging_to_console_and_syslog("Posting messages.")
self.assertTrue(self.post_messages())
def validate_machine_learning_workers(self):
self.create_machine_learning_worker_thread()
time.sleep(30)
self.create_producer_and_produce_jobs(
ProducerConsumerAPI.kafkaMsgQType)
time.sleep(5)
logging_to_console_and_syslog(
"Validating if the machine learning workers"
" successfully enqueued the messages.")
redis_instance = RedisInterface("Consumer")
self.assertEqual(
redis_instance.get_current_dequeue_count().decode('utf8'),
str(TestMachineLearningWorkers.max_number_of_jobs))
logging_to_console_and_syslog(
"dequeue_count={},max_number_of_jobs={}".format(
redis_instance.get_current_dequeue_count(),
TestMachineLearningWorkers.max_number_of_jobs))
def test_run(self):
logging_to_console_and_syslog(
"Validating **************** Machine Learning workers *****************."
)
self.validate_machine_learning_workers()
def create_test_docker_container(self):
completedProcess = subprocess.run([
"sudo", "docker-compose", "-f",
"{}/docker-compose_wurstmeister_kafka.yml".format(
self.dirname), "up", "-d"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
def delete_test_docker_container(self):
completedProcess = subprocess.run([
"sudo", "docker-compose", "-f",
"{}/docker-compose_wurstmeister_kafka.yml".format(
self.dirname), "down"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
def tearDown(self):
self.delete_test_docker_container()
subprocess.run(
['rm', '-rf', TestMachineLearningWorkers.directory_name],
stdout=subprocess.PIPE)
self.machine_learning_worker_thread.do_run = False
time.sleep(1)
logging_to_console_and_syslog("Trying to join thread.")
self.machine_learning_worker_thread.join(1.0)
time.sleep(1)
if self.machine_learning_worker_thread.isAlive():
try:
logging_to_console_and_syslog("Trying to __stop thread.")
self.machine_learning_worker_thread._stop()
except:
logging_to_console_and_syslog(
"Caught an exception while stopping thread.")
docker_api_interface_instance = DockerAPIInterface(
image_name=self.dirname.split('/')[-2],
dockerfile_directory_name=self.dirname)
docker_api_interface_instance.stop_docker_container_by_name()