def run_consumer_instance(): logging_to_console_and_syslog("Starting {}".format( threading.current_thread().getName())) t = threading.currentThread() redis_instance = RedisInterface("Consumer{}".format( threading.current_thread().getName())) consumer_instance = ConfluentKafkaMsgQAPI( is_consumer=True, thread_identifier=threading.current_thread().getName(), perform_subscription=True) while getattr(t, "do_run", True): t = threading.currentThread() message = consumer_instance.dequeue() if message: logging_to_console_and_syslog( "Consumer {}: Dequeued Message = {}".format( threading.current_thread().getName(), message)) redis_instance.increment_dequeue_count() redis_instance.write_an_event_in_redis_db( "Consumer {}: Dequeued Message = {}".format( threading.current_thread().getName(), message)) time.sleep(5) consumer_instance.cleanup() logging_to_console_and_syslog("Consumer {}: Exiting".format( threading.current_thread().getName()))
class MachineLearningWorker: def __init__(self): self.hostname = os.popen("cat /etc/hostname").read() self.cont_id = os.popen( "cat /proc/self/cgroup | head -n 1 | cut -d '/' -f3").read() self.producer_consumer_queue_type = None self.load_environment_variables() self.consumer_instance = None self.data_parser_instance = None self.redis_instance = None self.instantiate_objects() def load_environment_variables(self): while self.producer_consumer_queue_type is None: time.sleep(1) self.producer_consumer_queue_type = os.getenv( "producer_consumer_queue_type_key", default=None) logging_to_console_and_syslog( ("producer_consumer_queue_type={}".format( self.producer_consumer_queue_type))) def instantiate_objects(self): self.consumer_instance = ProducerConsumerAPI( is_consumer=True, thread_identifier="Consumer_{}".format(self.cont_id), type_of_messaging_queue=self.producer_consumer_queue_type) self.data_parser_instance = DataParserInterface() self.redis_instance = RedisInterface("Consumer_{}".format( self.cont_id)) def cleanup(self): self.consumer_instance.cleanup() def process_job(self, message): self.data_parser_instance.process_job(message) def dequeue_and_process_jobs(self): message = self.consumer_instance.dequeue() if message: try: event = "Consumer: Successfully dequeued a message = {} from msgQ.".format( message) self.redis_instance.write_an_event_in_redis_db(event) self.redis_instance.increment_dequeue_count() start_time = datetime.now() self.process_job(message) time_elapsed = datetime.now() - start_time event = 'Time taken to process {} = (hh:mm:ss.ms) {}'.format( message, time_elapsed) self.redis_instance.write_an_event_in_redis_db(event) except: print("Exception in dequeue_and_process_jobs:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) self.cleanup() self.instantiate_objects()
def post_messages(self): messages = [str(x) for x in range(100)] redis_instance = RedisInterface("Producer") for message in messages: status = self.producer_instance.enqueue(message) while not status: status = self.producer_instance.enqueue(message) event = "Producer: Successfully posted a message = {} into Kafka.".format(message) redis_instance.write_an_event_in_redis_db(event) redis_instance.increment_enqueue_count() self.producer_instance.cleanup() return True
class DirectoryWatch: def __init__(self): self.before={} self.after={} self.video_file_path = None self.producer_consumer_type = None self.redis_instance = RedisInterface("Producer") self.load_environment_variables() self.producer_instance = ProducerConsumerAPI(is_producer=True, thread_identifier="Producer", type_of_messaging_queue=self.producer_consumer_type) def load_environment_variables(self): while self.video_file_path is None or \ self.producer_consumer_type is None: time.sleep(1) self.video_file_path = os.getenv("video_file_path_key", default=None) self.producer_consumer_type = os.getenv("producer_consumer_queue_type_key", default=None) logging_to_console_and_syslog(("video_file_path={}".format(self.video_file_path))) logging_to_console_and_syslog(("producer_consumer_type={}".format(self.producer_consumer_type))) def cleanup(self): self.producer_instance.cleanup() def process_new_file(self,file_name): # post the file_name into the producer queue. self.producer_instance.enqueue(file_name) event = "Producer: Successfully posted a message = {} into msgQ.".format(file_name) self.redis_instance.write_an_event_in_redis_db(event) self.redis_instance.increment_enqueue_count() def watch_a_directory(self): self.before = {} while True: time.sleep(1) self.after = dict([(f, None) for f in os.listdir(self.video_file_path)]) added = [f for f in self.after if not f in self.before] removed = [f for f in self.before if not f in self.after] if added: logging_to_console_and_syslog("Added: " + str(added)) for filename in added: self.process_new_file(filename) if removed: logging_to_console_and_syslog("Removed: " + str(removed)) self.before = self.after
def create_local_consumer(self): redis_instance = RedisInterface(threading.current_thread().getName()) consumer_instance = ConfluentKafkaMsgQAPI( is_consumer=True, thread_identifier=threading.current_thread().getName(), perform_subscription=True) while redis_instance.get_current_enqueue_count() != \ redis_instance.get_current_dequeue_count(): message = consumer_instance.dequeue() if message is None or message.error(): continue else: logging_to_console_and_syslog( "Consumer {}: Dequeued Message = {}".format( threading.current_thread().getName(), message)) redis_instance.increment_dequeue_count() redis_instance.write_an_event_in_redis_db( "Consumer {}: Dequeued Message = {}".format( threading.current_thread().getName(), message)) consumer_instance.cleanup()
class ProducerConsumerAPI: """ This is a factory design pattern. This class produces messages into 1. Kafka Queue. 2. Rabbit Message Queue. """ rabbitMsgQType = "Rabbit" kafkaMsgQType = "Kafka" confluentKafkaMsgQType = "ConfluentKafka" def __init__(self, is_producer=False, is_consumer=False, perform_subscription=False, type_of_messaging_queue=None, thread_identifier=None): self.message_queue_instance = None self.redis_instance = None self.is_producer = is_producer self.is_consumer = is_consumer self.perform_subscription = perform_subscription self.type_of_messaging_queue = type_of_messaging_queue self.thread_identifier = thread_identifier self.read_environment_variables() #self.__connect() def read_environment_variables(self): """ This method is used to read the environment variables defined in the OS. :return: """ while self.type_of_messaging_queue is None: time.sleep(2) logging_to_console_and_syslog( "ProducerConsumerAPI: " "Trying to read the environment variables...") self.type_of_messaging_queue = os.getenv( "type_of_messaging_queue_key", default=None) logging_to_console_and_syslog("ProducerConsumerAPI:" "type_of_messaging_queue={}".format( self.type_of_messaging_queue)) def __connect(self): """ This method tries to connect to the messaging queue. :return: """ if self.message_queue_instance is None: try: if self.type_of_messaging_queue == ProducerConsumerAPI.kafkaMsgQType: self.message_queue_instance = KafkaMsgQAPI( is_producer=self.is_producer, is_consumer=self.is_consumer, perform_subscription=self.perform_subscription, thread_identifier=self.thread_identifier) elif self.type_of_messaging_queue == ProducerConsumerAPI.rabbitMsgQType: self.message_queue_instance = RabbitMsgQAPI( is_producer=self.is_producer, is_consumer=self.is_consumer, perform_subscription=self.perform_subscription, thread_identifier=self.thread_identifier) elif self.type_of_messaging_queue == ProducerConsumerAPI.confluentKafkaMsgQType: self.message_queue_instance = ConfluentKafkaMsgQAPI( is_producer=self.is_producer, is_consumer=self.is_consumer, perform_subscription=self.perform_subscription, thread_identifier=self.thread_identifier) if not self.redis_instance: if self.is_producer: self.redis_instance = RedisInterface( "Producer{}".format(self.thread_identifier)) elif self.is_consumer: self.redis_instance = RedisInterface( "Consumer{}".format(self.thread_identifier)) except: print("Exception in user code:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) time.sleep(5) else: logging_to_console_and_syslog( "ProducerConsumerAPI: Successfully " "created producer instance for messageQ type ={}".format( self.type_of_messaging_queue)) def enqueue(self, filename): """ This method tries to post a message. :param filename: :return True or False: """ status = False if filename is None or len(filename) == 0: logging_to_console_and_syslog("filename is None or invalid") return status if self.message_queue_instance is None: self.__connect() if hasattr(self.message_queue_instance, 'enqueue'): status = self.message_queue_instance.enqueue(filename) event = "Producer: Successfully posted a message = {} into msgQ. Status={}".format( filename, status) self.redis_instance.write_an_event_in_redis_db(event) self.redis_instance.increment_enqueue_count() return status def dequeue(self): """ This method tries to post a message. :return Freezes the current context and yeilds a message: Please make sure to iterate this over to unfreeze the context. """ if self.message_queue_instance is None: self.__connect() msg = None if hasattr(self.message_queue_instance, 'dequeue'): msg = self.message_queue_instance.dequeue() if msg: self.redis_instance.increment_dequeue_count() self.redis_instance.write_an_event_in_redis_db( "Consumer {}: Dequeued Message = {}".format( self.thread_identifier, msg)) self.cleanup() return msg def cleanup(self): if self.message_queue_instance: self.message_queue_instance.cleanup() self.message_queue_instance = None
class TestMachineLearningWorkers(unittest.TestCase): max_number_of_jobs = 100 directory_name = 'test_files' def setUp(self): os.environ["broker_name_key"] = "{}:9094".format("mec-poc") os.environ["topic_key"] = "video-file-name" os.environ["redis_log_keyname_key"] = "briefcam" os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue" os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue" os.environ["redis_server_hostname_key"] = "mec-poc" os.environ["redis_server_port_key"] = "6379" os.environ[ "producer_consumer_queue_type_key"] = ProducerConsumerAPI.kafkaMsgQType os.environ["data_parser_type_key"] = DataParserInterface.TensorFlow self.dirname = os.path.dirname(os.path.realpath(__file__)) current_file_path_list = os.path.realpath(__file__).split('/') data_path_directory = '/'.join(current_file_path_list[:-1]) os.environ["data_file_path_key"] = "{}/{}".format( data_path_directory, TestMachineLearningWorkers.directory_name) self.create_test_docker_container() self.machine_learning_worker_thread = None self.producer_instance = None @staticmethod def start_machine_learning_workers(): logging_to_console_and_syslog("Starting {}".format( threading.current_thread().getName())) t = threading.currentThread() worker = MachineLearningWorker() while getattr(t, "do_run", True): #logging_to_console_and_syslog("***dequeue_and_process_jobs***") worker.dequeue_and_process_jobs() logging_to_console_and_syslog("Consumer {}: Exiting".format( threading.current_thread().getName())) def create_machine_learning_worker_thread(self): self.machine_learning_worker_thread = \ threading.Thread(name="{}{}".format("thread", 1), target=TestMachineLearningWorkers.start_machine_learning_workers ) self.machine_learning_worker_thread.do_run = True self.machine_learning_worker_thread.name = "{}_{}".format( "test_machine_learning_workers", 1) self.machine_learning_worker_thread.start() def post_messages(self): self.redis_instance = RedisInterface("Producer") messages = [ str(x) for x in range(TestMachineLearningWorkers.max_number_of_jobs) ] for message in messages: self.producer_instance.enqueue(message) event = "Producer: Successfully posted a message = {} into msgQ.".format( message) self.redis_instance.write_an_event_in_redis_db(event) self.redis_instance.increment_enqueue_count() return True def create_producer_and_produce_jobs(self, msgq_type): self.producer_instance = ProducerConsumerAPI( is_producer=True, thread_identifier="Producer", type_of_messaging_queue=msgq_type) logging_to_console_and_syslog("Posting messages.") self.assertTrue(self.post_messages()) def validate_machine_learning_workers(self): self.create_machine_learning_worker_thread() time.sleep(30) self.create_producer_and_produce_jobs( ProducerConsumerAPI.kafkaMsgQType) time.sleep(5) logging_to_console_and_syslog( "Validating if the machine learning workers" " successfully enqueued the messages.") redis_instance = RedisInterface("Consumer") self.assertEqual( redis_instance.get_current_dequeue_count().decode('utf8'), str(TestMachineLearningWorkers.max_number_of_jobs)) logging_to_console_and_syslog( "dequeue_count={},max_number_of_jobs={}".format( redis_instance.get_current_dequeue_count(), TestMachineLearningWorkers.max_number_of_jobs)) def test_run(self): logging_to_console_and_syslog( "Validating **************** Machine Learning workers *****************." ) self.validate_machine_learning_workers() def create_test_docker_container(self): completedProcess = subprocess.run([ "sudo", "docker-compose", "-f", "{}/docker-compose_wurstmeister_kafka.yml".format( self.dirname), "up", "-d" ], stdout=subprocess.PIPE) self.assertIsNotNone(completedProcess) self.assertIsNotNone(completedProcess.stdout) def delete_test_docker_container(self): completedProcess = subprocess.run([ "sudo", "docker-compose", "-f", "{}/docker-compose_wurstmeister_kafka.yml".format( self.dirname), "down" ], stdout=subprocess.PIPE) self.assertIsNotNone(completedProcess) self.assertIsNotNone(completedProcess.stdout) def tearDown(self): self.delete_test_docker_container() subprocess.run( ['rm', '-rf', TestMachineLearningWorkers.directory_name], stdout=subprocess.PIPE) self.machine_learning_worker_thread.do_run = False time.sleep(1) logging_to_console_and_syslog("Trying to join thread.") self.machine_learning_worker_thread.join(1.0) time.sleep(1) if self.machine_learning_worker_thread.isAlive(): try: logging_to_console_and_syslog("Trying to __stop thread.") self.machine_learning_worker_thread._stop() except: logging_to_console_and_syslog( "Caught an exception while stopping thread.") docker_api_interface_instance = DockerAPIInterface( image_name=self.dirname.split('/')[-2], dockerfile_directory_name=self.dirname) docker_api_interface_instance.stop_docker_container_by_name()