def test_run(self):
logging_to_console_and_syslog("*********Creating consumer instances.*******************")
self.create_consumers()
logging_to_console_and_syslog("***************Creating producer instance.************")
time.sleep(10)
self.create_producer_and_produce_jobs()
time.sleep(120)
logging_to_console_and_syslog("Validating if the consumer successfully dequeued messages.")
redis_instance = RedisInterface(threading.current_thread().getName())
self.assertEqual(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count())
logging_to_console_and_syslog("enqueue_count={},dequeue_count={}"
.format(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count()))
def perform_enqueue_dequeue(self):
logging_to_console_and_syslog(
"Validating producer instance to be not null.")
self.create_producer_thread()
time.sleep(30)
logging_to_console_and_syslog(
"Validating if the Producer successfully enqueued the messages.")
redis_instance = RedisInterface("Producer")
self.assertEqual(
redis_instance.get_current_enqueue_count().decode('utf8'),
str(TestJobDispatcher.max_number_of_jobs))
logging_to_console_and_syslog(
"enqueue_count={},max_number_of_jobs={}".format(
redis_instance.get_current_enqueue_count(),
TestJobDispatcher.max_number_of_jobs))
def create_local_consumer2(self):
c = None
redis_instance = RedisInterface(threading.current_thread().getName())
conf = {
'bootstrap.servers': "localhost:9092",
'group.id': "video-file-name",
'session.timeout.ms': 6000,
'auto.offset.reset': 'earliest'
}
while redis_instance.get_current_enqueue_count() != \
redis_instance.get_current_dequeue_count():
if not c:
c = Consumer(conf)
c.subscribe(["video-file-name"], on_assign=print_assignment)
msg = c.poll(timeout=1.0)
if msg is None or msg.error():
continue
else:
logging_to_console_and_syslog(
'%% %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(), msg.offset(), str(
msg.key())))
logging_to_console_and_syslog("msg.value()={}".format(
msg.value()))
redis_instance.increment_dequeue_count()
c.close()
c = None
time.sleep(5)
def perform_enqueue_dequeue(self, msgq_type, perform_subscription=False):
logging_to_console_and_syslog(
"Creating producer instance and producing jobs.")
self.create_producer_and_produce_jobs(msgq_type)
time.sleep(10)
logging_to_console_and_syslog(
"Creating consumer threads to consume jobs.")
self.create_consumers(msgq_type, perform_subscription)
time.sleep(120)
logging_to_console_and_syslog(
"Validating if the consumer successfully dequeued messages.")
redis_instance = RedisInterface(threading.current_thread().getName())
self.assertEqual(redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count())
logging_to_console_and_syslog(
"enqueue_count={},dequeue_count={}".format(
redis_instance.get_current_enqueue_count(),
redis_instance.get_current_dequeue_count()))
def create_local_consumer(self):
redis_instance = RedisInterface(threading.current_thread().getName())
consumer_instance = ConfluentKafkaMsgQAPI(
is_consumer=True,
thread_identifier=threading.current_thread().getName(),
perform_subscription=True)
while redis_instance.get_current_enqueue_count() != \
redis_instance.get_current_dequeue_count():
message = consumer_instance.dequeue()
if message is None or message.error():
continue
else:
logging_to_console_and_syslog(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
redis_instance.increment_dequeue_count()
redis_instance.write_an_event_in_redis_db(
"Consumer {}: Dequeued Message = {}".format(
threading.current_thread().getName(), message))
consumer_instance.cleanup()
class AutoScaler:
def __init__(self):
self.docker_instance = None
self.min_threshold = -1
self.max_threshold = -1
self.auto_scale_service_name = None
self.auto_scale_time_interval = 10
self.scale_down_count = 0
self.scale_down_count_max_threshold = 0
self.redis_instance = RedisInterface("AutoScaler")
self.__load_environment_variables()
self.__perform_auto_scaling()
def __load_environment_variables(self):
while self.min_threshold is -1 or \
self.max_threshold is -1 or \
not self.auto_scale_service_name:
time.sleep(1)
self.min_threshold = int(os.getenv("min_threshold_key",
default=-1))
self.max_threshold = int(os.getenv("max_threshold_key",
default=-1))
self.scale_down_count_max_threshold = int(
os.getenv("scale_down_count_max_threshold_key", default=60))
self.auto_scale_time_interval = int(
os.getenv("auto_scale_time_interval_key", default=10))
self.auto_scale_service_name = os.getenv(
"auto_scale_service_name_key", default=None)
logging_to_console_and_syslog(
("min_threshold={}".format(self.min_threshold)))
logging_to_console_and_syslog(
("max_threshold={}".format(self.max_threshold)))
logging_to_console_and_syslog(("auto_scale_service_name={}".format(
self.auto_scale_service_name)))
logging_to_console_and_syslog(("auto_scale_time_interval={}".format(
self.auto_scale_time_interval)))
def __perform_scale_down_operation(self):
current_number_of_docker_instances = \
self.docker_instance.get_current_number_of_containers_per_service()
if current_number_of_docker_instances == self.max_threshold and \
current_number_of_docker_instances - 30 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 30)
elif current_number_of_docker_instances <= self.max_threshold // 2 and \
current_number_of_docker_instances - 20 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 20)
elif current_number_of_docker_instances <= self.max_threshold // 4 and \
current_number_of_docker_instances - 10 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 10)
elif current_number_of_docker_instances - 1 >= self.min_threshold:
self.docker_instance.scale(current_number_of_docker_instances - 1)
def __perform_scale_up_operation(self, jobs_in_pipe):
current_number_of_docker_instances = \
self.docker_instance.get_current_number_of_containers_per_service()
if 0 < jobs_in_pipe <= 10 and \
current_number_of_docker_instances + 1 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances + 1)
elif 11 < jobs_in_pipe <= 50:
if current_number_of_docker_instances + 10 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
10)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
elif 50 < jobs_in_pipe <= 100:
if current_number_of_docker_instances + 20 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
20)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
else:
if current_number_of_docker_instances + 30 < self.max_threshold:
self.docker_instance.scale(current_number_of_docker_instances +
30)
else:
self.docker_instance.scale(current_number_of_docker_instances +
self.max_threshold -
current_number_of_docker_instances)
def __perform_auto_scaling(self):
"""
Wake up every pre-specified time interval and do the following:
Read the current total_job_done_count from Redis.
Read the current total_job_to_be_done_count from Redis.
Compute the difference between total_job_to_be_done_count and total_job_done_count.
This gives the total number of jobs in the pipe.
Now, count the current number of consumer instances (containers) for the specified docker service.
if the current count of the number of instances is greater than max_threshold, then,
return False
if the total number of jobs in the pipe is 0:
if current count of the number of instance is equal to max_threshold:
scale down by 30
elif current count of the number of instance is above max_threshold//2:
scale down by 20
elif current count of the number of instance is above max_threshold//4:
scale down by 10
else:
scale down by 1
elif the total number of jobs in the pipe is between 1-10:
scale up by 1
elif total number of jobs in the pipe is between 11-50:
scale up by 10
elif total number of jobs in the pipe is between 51-100:
scale up by 20
elif total number of jobs in the pipe is between 101-200:
scale up by 30
:return:
"""
self.docker_instance = DockerService(self.auto_scale_service_name)
while True:
time.sleep(self.auto_scale_time_interval)
current_job_to_be_done_count = int(
self.redis_instance.get_current_enqueue_count())
current_job_done_count = int(
self.redis_instance.get_current_dequeue_count())
jobs_in_pipe = current_job_to_be_done_count - current_job_done_count
logging_to_console_and_syslog("current_job_to_be_done_count={},"
"current_job_done_count={},"
"jobs_in_pipe={}.".format(
current_job_to_be_done_count,
current_job_done_count,
jobs_in_pipe))
if jobs_in_pipe <= 0:
if self.scale_down_count == self.scale_down_count_max_threshold:
logging_to_console_and_syslog(
"Performing scale down operation.")
self.__perform_scale_down_operation()
self.scale_down_count = 0
else:
self.scale_down_count += 1
logging_to_console_and_syslog(
"Bumping up self.scale_down_count to {}.".format(
self.scale_down_count))
else:
logging_to_console_and_syslog("Performing scale up operation.")
self.__perform_scale_up_operation(jobs_in_pipe)
self.scale_down_count = 0
def cleanup(self):
pass