def __write_log_to_redis_and_logging_framework(self, event, write_to_redis_event_summary=False): logging_to_console_and_syslog(event) BriefCamParser.redis_instance.write_an_event_in_redis_db(event) if write_to_redis_event_summary: BriefCamParser.redis_instance.write_an_event_in_redis_db(event)
def create_local_consumer2(self): c = None redis_instance = RedisInterface(threading.current_thread().getName()) conf = { 'bootstrap.servers': "localhost:9092", 'group.id': "video-file-name", 'session.timeout.ms': 6000, 'auto.offset.reset': 'earliest' } while redis_instance.get_current_enqueue_count() != \ redis_instance.get_current_dequeue_count(): if not c: c = Consumer(conf) c.subscribe(["video-file-name"], on_assign=print_assignment) msg = c.poll(timeout=1.0) if msg is None or msg.error(): continue else: logging_to_console_and_syslog( '%% %s [%d] at offset %d with key %s:\n' % (msg.topic(), msg.partition(), msg.offset(), str( msg.key()))) logging_to_console_and_syslog("msg.value()={}".format( msg.value())) redis_instance.increment_dequeue_count() c.close() c = None time.sleep(5)
def process_job(self, file_name): logging_to_console_and_syslog("**********Processing Briefcam Job {} ***********" .format(file_name)) job_done = False while not job_done: try: if not self.browser_alive(): self.__write_log_to_redis_and_logging_framework( "Browser is not alive. Reopening the browser.", True) self.clean_up() self.__search_and_leftclick_case(file_name) if not self.__add_video("{}/{}".format(self.video_file_path, file_name)): self.__write_log_to_redis_and_logging_framework( "Add video failed. Closing and reopening the browser.", True) self.clean_up() continue self.__make_sure_video_is_added_successfully() self.__delete_video_clip_from_shared_volume(file_name) self.go_to_main_screen() job_done = True except BriefCamServerException: self.clean_up()
def __init__(self, docker_tag=None, image_name=None, dockerfile_directory_name=None): if docker_tag: self.docker_tag = docker_tag else: self.docker_tag = "ssriram1978" if image_name: self.image_name = image_name else: self.image_name = "unittest" self.docker_image_name = "{}/{}:latest".format(self.docker_tag, self.image_name) if dockerfile_directory_name: self.dirname = dockerfile_directory_name else: logging_to_console_and_syslog("Cannot proceed without knowing the path to dockerfile.") raise BaseException self.docker_instance = docker.from_env() self.container = None
def __init__(self): logging_to_console_and_syslog("Initializing RTSPDockerOrchestrator instance.") self.image_name = None self.environment = None self.bind_mount = None self.read_environment_variables() self.docker_instance = docker.from_env()
def tearDown(self): for index in range(self.max_consumer_threads): self.consumer_threads[index].do_run = False time.sleep(5) self.delete_test_docker_container() for index in range(self.max_consumer_threads): logging_to_console_and_syslog("Trying to join thread {}." .format(self.consumer_threads[index].getName())) self.consumer_threads[index].join(1.0) if self.consumer_threads[index].is_alive(): try: logging_to_console_and_syslog("Trying to __stop thread {}." .format(self.consumer_threads[index].getName())) self.consumer_threads[index]._stop() except: logging_to_console_and_syslog("Caught an exception while stopping thread {}" .format(self.consumer_threads[index].getName())) docker_image_name = self.dirname.split('/')[-2] logging_to_console_and_syslog("Trying to stop docker image{}." .format(docker_image_name)) try: docker_api_interface_instance = DockerAPIInterface(image_name=docker_image_name, dockerfile_directory_name=self.dirname) docker_api_interface_instance.stop_docker_container_by_name() except: logging_to_console_and_syslog("Caught an exception while stopping {}" .format(docker_image_name)) print("Exception in user code:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60)
def process_file_name(self, filename): """ Use the filename as the key in the dictionary video_file_name_size to fetch the inner dictionary value. Create a new dictionary value if the key is not found. Update the existing dictionary value if a key/value already exists in the dictionary video_file_name_size. :param filename: :return status: """ status = False if not filename or not type(filename) == str: return status if filename.endswith('.mp4'): filesize = self.__getSize(filename) logging_to_console_and_syslog("Found a file name that ends with .mp4 {}, " "filesize={}" .format(filename, filesize)) if filesize == 0: return status dict_obj = self.video_file_name_size[filename] if not dict_obj: # No dictionary object found for key filename. # Create a new dictionary and set it as the value in dict video_file_name_size. status = self.create_new_dict_obj(filename, filesize) else: status = self.update_existing_dict_obj(filename, filesize) return status
def read_environment_variables(self): """ This method is used to read the environment variables defined in the OS. :return: """ while self.redis_log_keyname is None or \ self.total_job_dequeued_count_redis_name is None: time.sleep(2) logging_to_console_and_syslog("RedisInterface:{} " "Trying to read the " "environment variables...".format( self.thread_identifer)) self.redis_log_keyname = os.getenv("redis_log_keyname_key", default=None) self.total_job_enqueued_count_redis_name = os.getenv( "total_job_enqueued_count_redis_name_key", default="total_job_enqueued_count") self.total_job_dequeued_count_redis_name = os.getenv( "total_job_dequeued_count_redis_name_key", default=None) logging_to_console_and_syslog( "RedisInterface:{} " "redis_log_keyname={}, " "total_job_enqueued_count_redis_name={}, " "total_job_dequeued_count_redis_name={}. ".format( self.thread_identifer, self.redis_log_keyname, self.total_job_enqueued_count_redis_name, self.total_job_dequeued_count_redis_name))
def validate_master_db_entries(self): """ This is a book keeping function. This function does the following: 1. It walks through each and every document in the couch DB and gets the document Identifier. 2. For each document identifier, it checks if there is a corresponding container identifier in the database. a. For each docker container identifier in the database, it checks if there is a corresponding docker container that is alive. 1. If the docker container is alive, then it just logs a message. 2. If the docker container is not alive, then it deletes the docker identifier from database and start a new docker container with the passed in data as the container argument. """ for key, data in self.couchdb_instance.yield_database_handle_entries(): logging_to_console_and_syslog( "Found a key {} and data {} in the briefcam database.".format( key, data)) container_id_dict = self.couchdb_instance.fetch_data_from_id_to_container_entry( key) if container_id_dict: container_id = container_id_dict[key] logging_to_console_and_syslog( "There is a valid container {} " "for this id {} in id to container db.".format( container_id, key)) self.validate_and_restart_container_if_needed( key, data, container_id) else: self.create_a_new_container_and_assign_a_job(key, data)
def __init__(self, docker_tag=None, image_name=None, dockerfile_path=None): if not dockerfile_path: logging_to_console_and_syslog("You need to specify the path to Dockerfile.") raise BaseException if image_name: self.image_name = image_name else: self.image_name = dockerfile_path.split('/')[-1] self.dirname = dockerfile_path self.docker_tag = docker_tag logging_to_console_and_syslog("docker_tag={}," "image_name={}," "dirname={}" .format(docker_tag, self.image_name, self.dirname)) super().__init__(docker_tag, self.image_name, self.dirname) self.docker_image = None
def create_producer_and_produce_jobs(self, msgq_type): self.producer_instance = ProducerConsumerAPI( is_producer=True, thread_identifier="Producer", type_of_messaging_queue=msgq_type) logging_to_console_and_syslog("Posting messages.") self.assertTrue(self.post_messages())
def __consumer_connect_to_broker(self): """ This method tries to connect to the kafka broker. :return: """ is_connected = False if self.consumer_instance is None: try: logging_to_console_and_syslog( "Consumer:{}:Trying to connect to broker_name={}".format( self.thread_identifier, self.broker_name)) self.consumer_instance = KafkaConsumer( bootstrap_servers=self.broker_name, group_id="kafka-consumer") is_connected = True except: logging_to_console_and_syslog( "Consumer:{}:Exception in user code:".format( self.thread_identifier)) logging_to_console_and_syslog("-" * 60) traceback.print_exc(file=sys.stdout) logging_to_console_and_syslog("-" * 60) time.sleep(5) logging_to_console_and_syslog("Consumer:{}:Consumer Successfully " "connected to broker_name={}".format( self.thread_identifier, self.broker_name)) return is_connected
def create_test_delete_test(self, parser_type, docker_compose_yml_file): self.create_docker_container(docker_compose_yml_file) time.sleep(10) os.environ["data_parser_type_key"] = parser_type if parser_type is DataParserInterface.BriefCam: os.environ["case_name_key"] = "MEC-POC" os.environ["case_url_key"] = "http://mec-demo/synopsis/" os.environ["browser_name_key"] = "/opt/google/chrome/chrome" os.environ["browser_loc_key"] = "google-chrome" os.environ["login_username_key"] = "Brief" os.environ["login_password_key"] = "Cam" os.environ["image_directory"] = "image_cont" os.environ["max_retry_attempts_key"] = "8" os.environ["sleep_time_key"] = "1" os.environ["time_between_input_character_key"] = "0.1" os.environ["time_for_browser_to_open_key"] = "60" os.environ["total_job_done_count_redis_name_key"] = "total_job_done_count" os.environ["video_file_path_key"] = self.dirname self.data_parser_instance = DataParserInterface() logging_to_console_and_syslog("Validating Data Parser Instance to be not null.") self.assertIsNotNone(self.data_parser_instance) self.assertTrue(self.data_parser_instance.process_job(self.filename)) self.delete_docker_container(docker_compose_yml_file)
def build(self, dockerfile_path): logging_to_console_and_syslog("Building docker image" " found in directory {}." .format(dockerfile_path)) self.docker_instance = DockerBuildUTPublish(dockerfile_path=dockerfile_path) self.docker_instance.create_docker_container()
def get_current_dequeue_count(self): count = self.redis_instance.read_key_value_from_redis_db( self.total_job_dequeued_count_redis_name) logging_to_console_and_syslog( "RedisInterface:{}." "{}={}".format(self.thread_identifer, self.total_job_dequeued_count_redis_name, count)) return count
def __init__(self, thread_identifer=None): logging_to_console_and_syslog("Instantiating RedisInterface.") self.total_job_enqueued_count_redis_name = None self.total_job_dequeued_count_redis_name = None self.redis_log_keyname = None self.thread_identifer = thread_identifer self.read_environment_variables() self.redis_instance = RedisClient()
def write_an_event_in_redis_db(self, event): logging_to_console_and_syslog("RedisInterface:{}." "Writing at key={}" " event={}".format( self.thread_identifer, self.redis_log_keyname, event)) self.redis_instance.write_an_event_on_redis_db(event, self.redis_log_keyname)
def load_environment_variables(self): while self.data_parser_type is None: time.sleep(1) self.data_parser_type = os.getenv("data_parser_type_key", default=None) logging_to_console_and_syslog( ("DataParserInterface: data_parser_type={}".format( self.data_parser_type)))
def __init__(self): logging_to_console_and_syslog( "**********Initializing PyTorch Parser ***********") self.hostname = os.popen("cat /etc/hostname").read() self.cont_id = os.popen( "cat /proc/self/cgroup | head -n 1 | cut -d '/' -f3").read() PyTorchParser.redis_instance = RedisInterface("BriefCam+{}".format( self.cont_id))
def increment_dequeue_count(self): logging_to_console_and_syslog( "RedisInterface:{}." "Incrementing total_job_dequeued_count={}".format( self.thread_identifer, self.total_job_dequeued_count_redis_name)) self.redis_instance.increment_key_in_redis_db( self.total_job_dequeued_count_redis_name)
def deploy(self): dirname = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[:-1]) deployment_file = dirname + "/" + DockerBuildUTDeploy.deployment_file logging_to_console_and_syslog("Deploying {}." .format(deployment_file)) docker_instance = DockerBuildUTPublish(dockerfile_path='/') docker_instance.deploy(deployment_file, DockerBuildUTDeploy.stack_name)
def load_environment_variables(self): while self.video_file_path is None or \ self.producer_consumer_type is None: time.sleep(1) self.video_file_path = os.getenv("video_file_path_key", default=None) self.producer_consumer_type = os.getenv("producer_consumer_queue_type_key", default=None) logging_to_console_and_syslog(("video_file_path={}".format(self.video_file_path))) logging_to_console_and_syslog(("producer_consumer_type={}".format(self.producer_consumer_type)))
def delivery_callback(err, msg): if err: logging_to_console_and_syslog('%% Message failed delivery: %s\n' % err) else: logging_to_console_and_syslog( '%% Message delivered to %s [%d] @ %s\n' % (msg.topic(), msg.partition(), str(msg.offset())))
def __init__(self): event = "BriefCam Server click timeout occurred." logging_to_console_and_syslog(event) BriefCamParser.redis_instance.write_an_event_on_redis_db(event) BriefCamParser.redis_instance.write_an_event_on_redis_db(event) pyautogui.press('esc') pyautogui.press('esc') raise BriefCamServerException
def __tear_down_docker_stack(self): completedProcess = subprocess.run(["docker", "stack", "rm", TestDockerService.service_name], stdout=subprocess.PIPE) self.assertIsNotNone(completedProcess) logging_to_console_and_syslog(completedProcess.stdout.decode('utf8'))
def create_rows_in_master_db(self): logging_to_console_and_syslog("Unit testing function add_entry_into_master_database()") self.document_id1 = self.couchdb_instance.add_entry_into_master_database(self.document1) self.assertIsNotNone(self.document_id1) logging_to_console_and_syslog("Unit testing function add_entry_into_master_database()") self.document_id2 = self.couchdb_instance.add_entry_into_master_database(self.document2) self.assertIsNotNone(self.document_id2)
def test_run(self): logging_to_console_and_syslog("Validating couchdb_instance to be not null.") self.assertIsNotNone(self.couchdb_instance) self.create_rows_in_master_db() self.create_rows_in_id_to_container_db() self.validate_data_from_master_db() self.fetch_rows_from_master_db() self.fetch_rows_from_id_to_container_db()
def __create_docker_stack(self): completedProcess = subprocess.run([ "docker", "stack", "deploy", "-c", "docker-compose.yml", TestAutoScaler.service_name ], stdout=subprocess.PIPE) self.assertIsNotNone(completedProcess) logging_to_console_and_syslog(completedProcess.stdout.decode('utf8')) time.sleep(30)
def create_docker_image(self): logging_to_console_and_syslog("Creating DockerAPIInterface instance()") self.docker_image = DockerAPIInterface(docker_tag=self.docker_tag, image_name=self.image_name, dockerfile_directory_name=self.dirname) logging_to_console_and_syslog("Invoking create_docker_container") self.docker_image.create_docker_container()
def test_run(self): #logging_to_console_and_syslog("Validating **************** KAFKA MSGQ *****************.") #self.perform_enqueue_dequeue(ProducerConsumerAPI.kafkaMsgQType) #self.cleanup_test_environment() #logging_to_console_and_syslog("Validating **************** KAFKA MSGQ + SUBSCRIPTION *****************.") self.perform_enqueue_dequeue(ProducerConsumerAPI.kafkaMsgQType, perform_subscription=True) logging_to_console_and_syslog( "Validating **************** RABBIT MSGQ *****************.")