def instantiate_objects(self):
self.consumer_instance = ProducerConsumerAPI(
is_consumer=True,
thread_identifier="Consumer_{}".format(self.cont_id),
type_of_messaging_queue=self.producer_consumer_queue_type)
self.data_parser_instance = DataParserInterface()
self.redis_instance = RedisInterface("Consumer_{}".format(
self.cont_id))
def __init__(self):
self.before={}
self.after={}
self.video_file_path = None
self.producer_consumer_type = None
self.redis_instance = RedisInterface("Producer")
self.load_environment_variables()
self.producer_instance = ProducerConsumerAPI(is_producer=True,
thread_identifier="Producer",
type_of_messaging_queue=self.producer_consumer_type)
def __init__(self):
self.docker_instance = None
self.min_threshold = -1
self.max_threshold = -1
self.auto_scale_service_name = None
self.auto_scale_time_interval = 10
self.scale_down_count = 0
self.scale_down_count_max_threshold = 0
self.redis_instance = RedisInterface("AutoScaler")
self.__load_environment_variables()
self.__perform_auto_scaling()
def post_messages(self):
self.redis_instance = RedisInterface("Producer")
messages = [
str(x)
for x in range(TestMachineLearningWorkers.max_number_of_jobs)
]
for message in messages:
self.producer_instance.enqueue(message)
event = "Producer: Successfully posted a message = {} into msgQ.".format(
message)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
return True
def perform_enqueue_dequeue(self):
logging_to_console_and_syslog(
"Validating producer instance to be not null.")
self.create_producer_thread()
time.sleep(30)
logging_to_console_and_syslog(
"Validating if the Producer successfully enqueued the messages.")
redis_instance = RedisInterface("Producer")
self.assertEqual(
redis_instance.get_current_enqueue_count().decode('utf8'),
str(TestJobDispatcher.max_number_of_jobs))
logging_to_console_and_syslog(
"enqueue_count={},max_number_of_jobs={}".format(
redis_instance.get_current_enqueue_count(),
TestJobDispatcher.max_number_of_jobs))
def __init__(self):
logging_to_console_and_syslog(
"**********Initializing PyTorch Parser ***********")
self.hostname = os.popen("cat /etc/hostname").read()
self.cont_id = os.popen(
"cat /proc/self/cgroup | head -n 1 | cut -d '/' -f3").read()
PyTorchParser.redis_instance = RedisInterface("BriefCam+{}".format(
self.cont_id))
def validate_machine_learning_workers(self):
self.create_machine_learning_worker_thread()
time.sleep(30)
self.create_producer_and_produce_jobs(
ProducerConsumerAPI.kafkaMsgQType)
time.sleep(5)
logging_to_console_and_syslog(
"Validating if the machine learning workers"
" successfully enqueued the messages.")
redis_instance = RedisInterface("Consumer")
self.assertEqual(
redis_instance.get_current_dequeue_count().decode('utf8'),
str(TestMachineLearningWorkers.max_number_of_jobs))
logging_to_console_and_syslog(
"dequeue_count={},max_number_of_jobs={}".format(
redis_instance.get_current_dequeue_count(),
TestMachineLearningWorkers.max_number_of_jobs))
class DirectoryWatch:
def __init__(self):
self.before={}
self.after={}
self.video_file_path = None
self.producer_consumer_type = None
self.redis_instance = RedisInterface("Producer")
self.load_environment_variables()
self.producer_instance = ProducerConsumerAPI(is_producer=True,
thread_identifier="Producer",
type_of_messaging_queue=self.producer_consumer_type)
def load_environment_variables(self):
while self.video_file_path is None or \
self.producer_consumer_type is None:
time.sleep(1)
self.video_file_path = os.getenv("video_file_path_key", default=None)
self.producer_consumer_type = os.getenv("producer_consumer_queue_type_key", default=None)
logging_to_console_and_syslog(("video_file_path={}".format(self.video_file_path)))
logging_to_console_and_syslog(("producer_consumer_type={}".format(self.producer_consumer_type)))
def cleanup(self):
self.producer_instance.cleanup()
def process_new_file(self,file_name):
# post the file_name into the producer queue.
self.producer_instance.enqueue(file_name)
event = "Producer: Successfully posted a message = {} into msgQ.".format(file_name)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
def watch_a_directory(self):
self.before = {}
while True:
time.sleep(1)
self.after = dict([(f, None) for f in os.listdir(self.video_file_path)])
added = [f for f in self.after if not f in self.before]
removed = [f for f in self.before if not f in self.after]
if added:
logging_to_console_and_syslog("Added: " + str(added))
for filename in added:
self.process_new_file(filename)
if removed:
logging_to_console_and_syslog("Removed: " + str(removed))
self.before = self.after
def __connect(self):
"""
This method tries to connect to the messaging queue.
:return:
"""
if self.message_queue_instance is None:
try:
if self.type_of_messaging_queue == ProducerConsumerAPI.kafkaMsgQType:
self.message_queue_instance = KafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.rabbitMsgQType:
self.message_queue_instance = RabbitMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.confluentKafkaMsgQType:
self.message_queue_instance = ConfluentKafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
if not self.redis_instance:
if self.is_producer:
self.redis_instance = RedisInterface(
"Producer{}".format(self.thread_identifier))
elif self.is_consumer:
self.redis_instance = RedisInterface(
"Consumer{}".format(self.thread_identifier))
except:
print("Exception in user code:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
time.sleep(5)
else:
logging_to_console_and_syslog(
"ProducerConsumerAPI: Successfully "
"created producer instance for messageQ type ={}".format(
self.type_of_messaging_queue))
def run_consumer_instance():
logging_to_console_and_syslog("Starting {}".format(
threading.current_thread().getName()))
t = threading.currentThread()
redis_instance = RedisInterface("Consumer{}".format(
threading.current_thread().getName()))
consumer_instance = ConfluentKafkaMsgQAPI(
is_consumer=True,
thread_identifier=threading.current_thread().getName(),
perform_subscription=True)
while getattr(t, "do_run", True):
t = threading.currentThread()
message = consumer_instance.dequeue()
if message:
logging_to_console_and_syslog(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
redis_instance.increment_dequeue_count()
redis_instance.write_an_event_in_redis_db(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
time.sleep(5)
consumer_instance.cleanup()
logging_to_console_and_syslog("Consumer {}: Exiting".format(
threading.current_thread().getName()))
def create_local_consumer2(self):
c = None
redis_instance = RedisInterface(threading.current_thread().getName())
conf = {
'bootstrap.servers': "localhost:9092",
'group.id': "video-file-name",
'session.timeout.ms': 6000,
'auto.offset.reset': 'earliest'
}
while redis_instance.get_current_enqueue_count() != \
redis_instance.get_current_dequeue_count():
if not c:
c = Consumer(conf)
c.subscribe(["video-file-name"], on_assign=print_assignment)
msg = c.poll(timeout=1.0)
if msg is None or msg.error():
continue
else:
logging_to_console_and_syslog(
'%% %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(), msg.offset(), str(
msg.key())))
logging_to_console_and_syslog("msg.value()={}".format(
msg.value()))
redis_instance.increment_dequeue_count()
c.close()
c = None
time.sleep(5)
class MachineLearningWorker:
def __init__(self):
self.hostname = os.popen("cat /etc/hostname").read()
self.cont_id = os.popen(
"cat /proc/self/cgroup | head -n 1 | cut -d '/' -f3").read()
self.producer_consumer_queue_type = None
self.load_environment_variables()
self.consumer_instance = None
self.data_parser_instance = None
self.redis_instance = None
self.instantiate_objects()
def load_environment_variables(self):
while self.producer_consumer_queue_type is None:
time.sleep(1)
self.producer_consumer_queue_type = os.getenv(
"producer_consumer_queue_type_key", default=None)
logging_to_console_and_syslog(
("producer_consumer_queue_type={}".format(
self.producer_consumer_queue_type)))
def instantiate_objects(self):
self.consumer_instance = ProducerConsumerAPI(
is_consumer=True,
thread_identifier="Consumer_{}".format(self.cont_id),
type_of_messaging_queue=self.producer_consumer_queue_type)
self.data_parser_instance = DataParserInterface()
self.redis_instance = RedisInterface("Consumer_{}".format(
self.cont_id))
def cleanup(self):
self.consumer_instance.cleanup()
def process_job(self, message):
self.data_parser_instance.process_job(message)
def dequeue_and_process_jobs(self):
message = self.consumer_instance.dequeue()
if message:
try:
event = "Consumer: Successfully dequeued a message = {} from msgQ.".format(
message)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_dequeue_count()
start_time = datetime.now()
self.process_job(message)
time_elapsed = datetime.now() - start_time
event = 'Time taken to process {} = (hh:mm:ss.ms) {}'.format(
message, time_elapsed)
self.redis_instance.write_an_event_in_redis_db(event)
except:
print("Exception in dequeue_and_process_jobs:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
self.cleanup()
self.instantiate_objects()
def post_messages(self):
messages = [str(x) for x in range(100)]
redis_instance = RedisInterface("Producer")
for message in messages:
status = self.producer_instance.enqueue(message)
while not status:
status = self.producer_instance.enqueue(message)
event = "Producer: Successfully posted a message = {} into Kafka.".format(message)
redis_instance.write_an_event_in_redis_db(event)
redis_instance.increment_enqueue_count()
self.producer_instance.cleanup()
return True
def create_local_consumer(self):
redis_instance = RedisInterface(threading.current_thread().getName())
consumer_instance = ConfluentKafkaMsgQAPI(
is_consumer=True,
thread_identifier=threading.current_thread().getName(),
perform_subscription=True)
while redis_instance.get_current_enqueue_count() != \
redis_instance.get_current_dequeue_count():
message = consumer_instance.dequeue()
if message is None or message.error():
continue
else:
logging_to_console_and_syslog(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
redis_instance.increment_dequeue_count()
redis_instance.write_an_event_in_redis_db(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
consumer_instance.cleanup()
def test_run(self):
logging_to_console_and_syslog("*********Creating consumer instances.*******************")
self.create_consumers()
logging_to_console_and_syslog("***************Creating producer instance.************")
time.sleep(10)
self.create_producer_and_produce_jobs()
time.sleep(120)
logging_to_console_and_syslog("Validating if the consumer successfully dequeued messages.")
redis_instance = RedisInterface(threading.current_thread().getName())
self.assertEqual(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count())
logging_to_console_and_syslog("enqueue_count={},dequeue_count={}"
.format(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count()))
def __init__(self):
logging_to_console_and_syslog("**********Initializing Briefcam Parser ***********")
self.case_name = None
self.case_url = None
self.browser_loc = None
self.username = None
self.password = None
self.image_directory = None
self.process = None
self.browser_ready = False
self.browser_name = None
self.max_retry_attempts = 0
self.sleep_time = 1
self.time_for_browser_to_open = 60
self.time_between_input_character = 0
self.redis_log_keyname = None
self.video_file_path = None
self.total_job_done_count_redis_name = None
self.hostname = os.popen("cat /etc/hostname").read()
self.cont_id = os.popen("cat /proc/self/cgroup | head -n 1 | cut -d '/' -f3").read()
BriefCamParser.redis_instance = RedisInterface("BriefCam+{}".format(self.cont_id))
self.import_environment_variables()
self.prepare_browser()
def perform_enqueue_dequeue(self, msgq_type, perform_subscription=False):
logging_to_console_and_syslog(
"Creating producer instance and producing jobs.")
self.create_producer_and_produce_jobs(msgq_type)
time.sleep(10)
logging_to_console_and_syslog(
"Creating consumer threads to consume jobs.")
self.create_consumers(msgq_type, perform_subscription)
time.sleep(120)
logging_to_console_and_syslog(
"Validating if the consumer successfully dequeued messages.")
redis_instance = RedisInterface(threading.current_thread().getName())
self.assertEqual(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count())
logging_to_console_and_syslog(
"enqueue_count={},dequeue_count={}".format(
redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count()))
def test_auto_scaler(self):
logging_to_console_and_syslog(
"******************test_auto_scaler******************")
self.redis_instance = RedisInterface(TestAutoScaler.service_name)
self.redis_instance.set_the_key_in_redis_db("enqueue", 1)
self.redis_instance.set_the_key_in_redis_db("dequeue", 1)
self.create_auto_scaler_thread()
#SCALE UP
jobs_in_pipe = 10
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 20
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 51
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 101
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
class TestAutoScaler(unittest.TestCase):
service_to_be_tested = 'machine_learning_workers'
service_name = 'test_auto_scaler'
sleep_time = 5
min_threshold = 1
max_threshold = 100
retry_attempts = 3
total_number_of_iterations = 3
def setUp(self):
os.environ["redis_log_keyname_key"] = "briefcam"
os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue"
os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue"
os.environ["redis_server_hostname_key"] = "localhost"
os.environ["redis_server_port_key"] = "6379"
os.environ["min_threshold_key"] = str(TestAutoScaler.min_threshold)
os.environ["max_threshold_key"] = str(TestAutoScaler.max_threshold)
os.environ["auto_scale_time_interval_key"] = str(
TestAutoScaler.sleep_time)
os.environ[
"auto_scale_service_name_key"] = TestAutoScaler.service_to_be_tested
self.__create_docker_stack()
self.auto_scaler = None
@staticmethod
def run_auto_scaler():
logging_to_console_and_syslog("Instantiating AutoScaler...")
auto_scaler = AutoScaler()
logging_to_console_and_syslog("Shutting down AutoScaler...")
def __create_docker_stack(self):
completedProcess = subprocess.run([
"docker", "stack", "deploy", "-c", "docker-compose.yml",
TestAutoScaler.service_name
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
logging_to_console_and_syslog(completedProcess.stdout.decode('utf8'))
time.sleep(30)
def create_auto_scaler_thread(self):
self.auto_scaler_thread = threading.Thread(
name="{}{}".format("thread", 1),
target=TestAutoScaler.run_auto_scaler)
self.auto_scaler_thread.do_run = True
self.auto_scaler_thread.name = "{}_{}".format("auto_scaler_thread", 1)
self.auto_scaler_thread.start()
time.sleep(30)
def get_current_count(self, service_name):
docker_svc_instance = DockerService(service_name)
self.assertIsNotNone(docker_svc_instance)
service_id = docker_svc_instance.get_service_id_from_service_name()
self.assertIsNotNone(service_id)
logging_to_console_and_syslog("get_service_id_from_service_name:"
"service_name={},service_id={}".format(
service_name, service_id))
current_container_count = docker_svc_instance.get_current_number_of_containers_per_service(
)
return current_container_count
def validate_auto_scaler(self, jobs_in_pipe, scale_up):
self.redis_instance.set_the_key_in_redis_db("enqueue", jobs_in_pipe)
service_name = '{}_{}'.format(TestAutoScaler.service_name,
TestAutoScaler.service_to_be_tested)
expected_container_count = 0
for index in range(TestAutoScaler.total_number_of_iterations):
current_container_count = self.get_current_count(service_name)
if index == 0:
# Start off from the current value of the total number of containers.
expected_container_count = current_container_count
if current_container_count != expected_container_count:
for count in range(TestAutoScaler.retry_attempts):
time.sleep(1)
current_container_count = self.get_current_count(
service_name)
logging_to_console_and_syslog(
"Iteration={},"
"Retrying after a second."
"current_container_count={},"
"expected_container_count={}".format(
index, current_container_count,
expected_container_count))
if current_container_count == expected_container_count:
break
logging_to_console_and_syslog(
"Iteration={},"
"get_current_number_of_containers_per_service:"
"current_container_count={},"
"expected_container_count={}".format(index,
current_container_count,
expected_container_count))
self.assertEqual(current_container_count, expected_container_count)
if scale_up:
if 0 < jobs_in_pipe <= 10 and \
current_container_count + 1 < TestAutoScaler.max_threshold:
expected_container_count += 1
elif 11 < jobs_in_pipe <= 50:
if current_container_count + 10 < TestAutoScaler.max_threshold:
expected_container_count += 10
else:
expected_container_count += TestAutoScaler.max_threshold - current_container_count
elif 50 < jobs_in_pipe <= 100:
if current_container_count + 20 < TestAutoScaler.max_threshold:
expected_container_count += 20
else:
expected_container_count += TestAutoScaler.max_threshold - current_container_count
elif 100 < jobs_in_pipe <= 200:
if current_container_count + 30 < TestAutoScaler.max_threshold:
expected_container_count += 30
else:
expected_container_count += TestAutoScaler.max_threshold - current_container_count
else:
if current_container_count == TestAutoScaler.max_threshold and \
current_container_count - 30 >= TestAutoScaler.min_threshold:
expected_container_count -= 30
elif current_container_count <= TestAutoScaler.max_threshold // 2 and \
current_container_count - 20 >= TestAutoScaler.min_threshold:
expected_container_count -= 20
elif current_container_count <= TestAutoScaler.max_threshold // 4 and \
current_container_count - 10 >= TestAutoScaler.min_threshold:
expected_container_count -= 10
elif current_container_count - 1 >= TestAutoScaler.min_threshold:
expected_container_count -= 1
logging_to_console_and_syslog("Sleeping for {} seconds.".format(
TestAutoScaler.sleep_time))
time.sleep(TestAutoScaler.sleep_time)
logging_to_console_and_syslog("Waking up after {} seconds.".format(
TestAutoScaler.sleep_time))
def test_auto_scaler(self):
logging_to_console_and_syslog(
"******************test_auto_scaler******************")
self.redis_instance = RedisInterface(TestAutoScaler.service_name)
self.redis_instance.set_the_key_in_redis_db("enqueue", 1)
self.redis_instance.set_the_key_in_redis_db("dequeue", 1)
self.create_auto_scaler_thread()
#SCALE UP
jobs_in_pipe = 10
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 20
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 51
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
#SCALE UP
jobs_in_pipe = 101
logging_to_console_and_syslog(
"************Testing SCALE UP Jobs in pipe = {} ****************".
format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=True)
#SCALE DOWN
jobs_in_pipe = 0
logging_to_console_and_syslog(
"************Testing SCALE DOWN Jobs in pipe = {} ****************"
.format(jobs_in_pipe))
self.validate_auto_scaler(jobs_in_pipe, scale_up=False)
def __tear_down_docker_stack(self):
completedProcess = subprocess.run(
["docker", "stack", "rm", TestAutoScaler.service_name],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
logging_to_console_and_syslog(completedProcess.stdout.decode('utf8'))
def tearDown(self):
self.__tear_down_docker_stack()
self.auto_scaler_thread.join(1.0)
try:
docker_api_interface_instance = DockerAPIInterface(
image_name=TestAutoScaler.service_to_be_tested)
docker_api_interface_instance.stop_docker_container_by_name()
except:
logging_to_console_and_syslog(
"Caught an exception while stopping {}".format(
TestAutoScaler.service_to_be_tested))
print("Exception in user code:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
class ProducerConsumerAPI:
"""
This is a factory design pattern.
This class produces messages into
1. Kafka Queue.
2. Rabbit Message Queue.
"""
rabbitMsgQType = "Rabbit"
kafkaMsgQType = "Kafka"
confluentKafkaMsgQType = "ConfluentKafka"
def __init__(self,
is_producer=False,
is_consumer=False,
perform_subscription=False,
type_of_messaging_queue=None,
thread_identifier=None):
self.message_queue_instance = None
self.redis_instance = None
self.is_producer = is_producer
self.is_consumer = is_consumer
self.perform_subscription = perform_subscription
self.type_of_messaging_queue = type_of_messaging_queue
self.thread_identifier = thread_identifier
self.read_environment_variables()
#self.__connect()
def read_environment_variables(self):
"""
This method is used to read the environment variables defined in the OS.
:return:
"""
while self.type_of_messaging_queue is None:
time.sleep(2)
logging_to_console_and_syslog(
"ProducerConsumerAPI: "
"Trying to read the environment variables...")
self.type_of_messaging_queue = os.getenv(
"type_of_messaging_queue_key", default=None)
logging_to_console_and_syslog("ProducerConsumerAPI:"
"type_of_messaging_queue={}".format(
self.type_of_messaging_queue))
def __connect(self):
"""
This method tries to connect to the messaging queue.
:return:
"""
if self.message_queue_instance is None:
try:
if self.type_of_messaging_queue == ProducerConsumerAPI.kafkaMsgQType:
self.message_queue_instance = KafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.rabbitMsgQType:
self.message_queue_instance = RabbitMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
elif self.type_of_messaging_queue == ProducerConsumerAPI.confluentKafkaMsgQType:
self.message_queue_instance = ConfluentKafkaMsgQAPI(
is_producer=self.is_producer,
is_consumer=self.is_consumer,
perform_subscription=self.perform_subscription,
thread_identifier=self.thread_identifier)
if not self.redis_instance:
if self.is_producer:
self.redis_instance = RedisInterface(
"Producer{}".format(self.thread_identifier))
elif self.is_consumer:
self.redis_instance = RedisInterface(
"Consumer{}".format(self.thread_identifier))
except:
print("Exception in user code:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
time.sleep(5)
else:
logging_to_console_and_syslog(
"ProducerConsumerAPI: Successfully "
"created producer instance for messageQ type ={}".format(
self.type_of_messaging_queue))
def enqueue(self, filename):
"""
This method tries to post a message.
:param filename:
:return True or False:
"""
status = False
if filename is None or len(filename) == 0:
logging_to_console_and_syslog("filename is None or invalid")
return status
if self.message_queue_instance is None:
self.__connect()
if hasattr(self.message_queue_instance, 'enqueue'):
status = self.message_queue_instance.enqueue(filename)
event = "Producer: Successfully posted a message = {} into msgQ. Status={}".format(
filename, status)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
return status
def dequeue(self):
"""
This method tries to post a message.
:return Freezes the current context and yeilds a message:
Please make sure to iterate this over to unfreeze the context.
"""
if self.message_queue_instance is None:
self.__connect()
msg = None
if hasattr(self.message_queue_instance, 'dequeue'):
msg = self.message_queue_instance.dequeue()
if msg:
self.redis_instance.increment_dequeue_count()
self.redis_instance.write_an_event_in_redis_db(
"Consumer {}: Dequeued Message = {}".format(
self.thread_identifier, msg))
self.cleanup()
return msg
def cleanup(self):
if self.message_queue_instance:
self.message_queue_instance.cleanup()
self.message_queue_instance = None
class TestMachineLearningWorkers(unittest.TestCase):
max_number_of_jobs = 100
directory_name = 'test_files'
def setUp(self):
os.environ["broker_name_key"] = "{}:9094".format("mec-poc")
os.environ["topic_key"] = "video-file-name"
os.environ["redis_log_keyname_key"] = "briefcam"
os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue"
os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue"
os.environ["redis_server_hostname_key"] = "mec-poc"
os.environ["redis_server_port_key"] = "6379"
os.environ[
"producer_consumer_queue_type_key"] = ProducerConsumerAPI.kafkaMsgQType
os.environ["data_parser_type_key"] = DataParserInterface.TensorFlow
self.dirname = os.path.dirname(os.path.realpath(__file__))
current_file_path_list = os.path.realpath(__file__).split('/')
data_path_directory = '/'.join(current_file_path_list[:-1])
os.environ["data_file_path_key"] = "{}/{}".format(
data_path_directory, TestMachineLearningWorkers.directory_name)
self.create_test_docker_container()
self.machine_learning_worker_thread = None
self.producer_instance = None
@staticmethod
def start_machine_learning_workers():
logging_to_console_and_syslog("Starting {}".format(
threading.current_thread().getName()))
t = threading.currentThread()
worker = MachineLearningWorker()
while getattr(t, "do_run", True):
#logging_to_console_and_syslog("***dequeue_and_process_jobs***")
worker.dequeue_and_process_jobs()
logging_to_console_and_syslog("Consumer {}: Exiting".format(
threading.current_thread().getName()))
def create_machine_learning_worker_thread(self):
self.machine_learning_worker_thread = \
threading.Thread(name="{}{}".format("thread", 1),
target=TestMachineLearningWorkers.start_machine_learning_workers
)
self.machine_learning_worker_thread.do_run = True
self.machine_learning_worker_thread.name = "{}_{}".format(
"test_machine_learning_workers", 1)
self.machine_learning_worker_thread.start()
def post_messages(self):
self.redis_instance = RedisInterface("Producer")
messages = [
str(x)
for x in range(TestMachineLearningWorkers.max_number_of_jobs)
]
for message in messages:
self.producer_instance.enqueue(message)
event = "Producer: Successfully posted a message = {} into msgQ.".format(
message)
self.redis_instance.write_an_event_in_redis_db(event)
self.redis_instance.increment_enqueue_count()
return True
def create_producer_and_produce_jobs(self, msgq_type):
self.producer_instance = ProducerConsumerAPI(
is_producer=True,
thread_identifier="Producer",
type_of_messaging_queue=msgq_type)
logging_to_console_and_syslog("Posting messages.")
self.assertTrue(self.post_messages())
def validate_machine_learning_workers(self):
self.create_machine_learning_worker_thread()
time.sleep(30)
self.create_producer_and_produce_jobs(
ProducerConsumerAPI.kafkaMsgQType)
time.sleep(5)
logging_to_console_and_syslog(
"Validating if the machine learning workers"
" successfully enqueued the messages.")
redis_instance = RedisInterface("Consumer")
self.assertEqual(
redis_instance.get_current_dequeue_count().decode('utf8'),
str(TestMachineLearningWorkers.max_number_of_jobs))
logging_to_console_and_syslog(
"dequeue_count={},max_number_of_jobs={}".format(
redis_instance.get_current_dequeue_count(),
TestMachineLearningWorkers.max_number_of_jobs))
def test_run(self):
logging_to_console_and_syslog(
"Validating **************** Machine Learning workers *****************."
)
self.validate_machine_learning_workers()
def create_test_docker_container(self):
completedProcess = subprocess.run([
"sudo", "docker-compose", "-f",
"{}/docker-compose_wurstmeister_kafka.yml".format(
self.dirname), "up", "-d"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
def delete_test_docker_container(self):
completedProcess = subprocess.run([
"sudo", "docker-compose", "-f",
"{}/docker-compose_wurstmeister_kafka.yml".format(
self.dirname), "down"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
def tearDown(self):
self.delete_test_docker_container()
subprocess.run(
['rm', '-rf', TestMachineLearningWorkers.directory_name],
stdout=subprocess.PIPE)
self.machine_learning_worker_thread.do_run = False
time.sleep(1)
logging_to_console_and_syslog("Trying to join thread.")
self.machine_learning_worker_thread.join(1.0)
time.sleep(1)
if self.machine_learning_worker_thread.isAlive():
try:
logging_to_console_and_syslog("Trying to __stop thread.")
self.machine_learning_worker_thread._stop()
except:
logging_to_console_and_syslog(
"Caught an exception while stopping thread.")
docker_api_interface_instance = DockerAPIInterface(
image_name=self.dirname.split('/')[-2],
dockerfile_directory_name=self.dirname)
docker_api_interface_instance.stop_docker_container_by_name()
class AutoScaler:
def __init__(self):
self.docker_instance = None
self.min_threshold = -1
self.max_threshold = -1
self.auto_scale_service_name = None
self.auto_scale_time_interval = 10
self.scale_down_count = 0
self.scale_down_count_max_threshold = 0
self.redis_instance = RedisInterface("AutoScaler")
self.__load_environment_variables()
self.__perform_auto_scaling()
def __load_environment_variables(self):
while self.min_threshold is -1 or \
self.max_threshold is -1 or \
not self.auto_scale_service_name:
time.sleep(1)
self.min_threshold = int(os.getenv("min_threshold_key",
default=-1))
self.max_threshold = int(os.getenv("max_threshold_key",
default=-1))
self.scale_down_count_max_threshold = int(
os.getenv("scale_down_count_max_threshold_key", default=60))
self.auto_scale_time_interval = int(
os.getenv("auto_scale_time_interval_key", default=10))
self.auto_scale_service_name = os.getenv(
"auto_scale_service_name_key", default=None)
logging_to_console_and_syslog(
("min_threshold={}".format(self.min_threshold)))
logging_to_console_and_syslog(
("max_threshold={}".format(self.max_threshold)))
logging_to_console_and_syslog(("auto_scale_service_name={}".format(
self.auto_scale_service_name)))
logging_to_console_and_syslog(("auto_scale_time_interval={}".format(
self.auto_scale_time_interval)))
def __perform_scale_down_operation(self):
current_number_of_docker_instances = \
self.docker_instance.get_current_number_of_containers_per_service()
if current_number_of_docker_instances == self.max_threshold and \
current_number_of_docker_instances - 30 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 30)
elif current_number_of_docker_instances <= self.max_threshold // 2 and \
current_number_of_docker_instances - 20 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 20)
elif current_number_of_docker_instances <= self.max_threshold // 4 and \
current_number_of_docker_instances - 10 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 10)
elif current_number_of_docker_instances - 1 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 1)
def __perform_scale_up_operation(self, jobs_in_pipe):
current_number_of_docker_instances = \
self.docker_instance.get_current_number_of_containers_per_service()
if 0 < jobs_in_pipe <= 10 and \
current_number_of_docker_instances + 1 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances + 1)
elif 11 < jobs_in_pipe <= 50:
if current_number_of_docker_instances + 10 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
10)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
elif 50 < jobs_in_pipe <= 100:
if current_number_of_docker_instances + 20 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
20)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
else:
if current_number_of_docker_instances + 30 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
30)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
def __perform_auto_scaling(self):
"""
Wake up every pre-specified time interval and do the following:
Read the current total_job_done_count from Redis.
Read the current total_job_to_be_done_count from Redis.
Compute the difference between total_job_to_be_done_count and total_job_done_count.
This gives the total number of jobs in the pipe.
Now, count the current number of consumer instances (containers) for the specified docker service.
if the current count of the number of instances is greater than max_threshold, then,
return False
if the total number of jobs in the pipe is 0:
if current count of the number of instance is equal to max_threshold:
scale down by 30
elif current count of the number of instance is above max_threshold//2:
scale down by 20
elif current count of the number of instance is above max_threshold//4:
scale down by 10
else:
scale down by 1
elif the total number of jobs in the pipe is between 1-10:
scale up by 1
elif total number of jobs in the pipe is between 11-50:
scale up by 10
elif total number of jobs in the pipe is between 51-100:
scale up by 20
elif total number of jobs in the pipe is between 101-200:
scale up by 30
:return:
"""
self.docker_instance = DockerService(self.auto_scale_service_name)
while True:
time.sleep(self.auto_scale_time_interval)
current_job_to_be_done_count = int(
self.redis_instance.get_current_enqueue_count())
current_job_done_count = int(
self.redis_instance.get_current_dequeue_count())
jobs_in_pipe = current_job_to_be_done_count - current_job_done_count
logging_to_console_and_syslog("current_job_to_be_done_count={},"
"current_job_done_count={},"
"jobs_in_pipe={}.".format(
current_job_to_be_done_count,
current_job_done_count,
jobs_in_pipe))
if jobs_in_pipe <= 0:
if self.scale_down_count == self.scale_down_count_max_threshold:
logging_to_console_and_syslog(
"Performing scale down operation.")
self.__perform_scale_down_operation()
self.scale_down_count = 0
else:
self.scale_down_count += 1
logging_to_console_and_syslog(
"Bumping up self.scale_down_count to {}.".format(
self.scale_down_count))
else:
logging_to_console_and_syslog("Performing scale up operation.")
self.__perform_scale_up_operation(jobs_in_pipe)
self.scale_down_count = 0
def cleanup(self):
pass