def test_piper_worker_cache_reuse(): if sys.version_info >= (3, 8): # Initiate cache smm, cache_process, cache_queue, cache_ids = start_cache( logging, "true:100000", False, "") load_shared_memory_manager() # Create multiple objects and store with the same name: amount = 10 np_objs = [np.random.rand(4) for _ in range(amount)] obj_name = "name" np_objs_names = [obj_name for _ in range(amount)] # Check insertions for i in range(amount): insert_object_into_cache_wrapper(logging, cache_queue, np_objs[i], np_objs_names[i], np_objs_names[i], None) if obj_name not in cache_ids: raise Exception("Object " + obj_name + " not found in cache_ids.") else: if cache_ids[obj_name][4] != 9: raise Exception("Wrong number of hits!!!") # Stop cache stop_cache(smm, cache_queue, False, cache_process) else: print(NOT_PYTHON_3_8)
def test_piper_worker_cache_reuse(): if sys.version_info >= (3, 8): # Initiate cache smm, cache_process, cache_queue, cache_ids = start_cache( logging, "true:100") load_shared_memory_manager() # Create multiple objects and store with the same name: amount = 10 np_objs = [np.random.rand(4) for _ in range(amount)] np_objs_names = ["name" for _ in range(amount)] # Check insertions for i in range(amount): insert_object_into_cache_wrapper(logging, cache_queue, np_objs[i], np_objs_names[i]) # Stop cache stop_cache(smm, cache_queue, cache_process) else: print(NOT_PYTHON_3_8)
def test_piper_worker_cache(): if sys.version_info >= (3, 8): # Initiate cache smm, cache_process, cache_queue, cache_ids = start_cache( logging, "Default") load_shared_memory_manager() # Supported types: np_obj = np.random.rand(4) np_obj_name = "np_obj" list_obj = [1, 2, 3, 4, 5, "hi"] list_obj_name = "list_obj_name" tuple_obj = ("1", 2, 3, "4", "hi") tuple_obj_name = "tuple_obj_name" # Check insertions insert_object_into_cache_wrapper(logging, cache_queue, np_obj, np_obj_name) # noqa: E501 insert_object_into_cache_wrapper(logging, cache_queue, list_obj, list_obj_name) # noqa: E501 insert_object_into_cache_wrapper(logging, cache_queue, tuple_obj, tuple_obj_name) # noqa: E501 # Check retrieves np_obj_new, np_obj_shm = retrieve_object_from_cache( logging, cache_ids, np_obj_name) # noqa: E501 list_obj_new, list_obj_shm = retrieve_object_from_cache( logging, cache_ids, list_obj_name) # noqa: E501 tuple_obj_new, tuple_obj_shm = retrieve_object_from_cache( logging, cache_ids, tuple_obj_name) # noqa: E501 assert ( set(np_obj_new).intersection(np_obj) ), "ERROR: Numpy object retrieved from cache differs from inserted" assert (set(list_obj_new).intersection(list_obj) ), "ERROR: List retrieved from cache differs from inserted" assert (set(tuple_obj_new).intersection(tuple_obj) ), "ERROR: Tuple retrieved from cache differs from inserted" # Check replace new_list_obj = ["hello", "world", 6] replace_object_into_cache(logging, cache_queue, new_list_obj, list_obj_name) # noqa: E501 time.sleep(0.5) list_obj_new2, list_obj_shm2 = retrieve_object_from_cache( logging, cache_ids, list_obj_name) # noqa: E501 assert (set(list_obj_new2).intersection(new_list_obj) ), "ERROR: List retrieved from cache differs from inserted" # Remove object remove_object_from_cache(logging, cache_queue, list_obj_name) is_ok = False try: _, _ = retrieve_object_from_cache(logging, cache_ids, list_obj_name) # noqa: E501 except Exception: # NOSONAR is_ok = True assert (is_ok), "ERROR: List has not been removed." # Check if in cache assert (in_cache( np_obj_name, cache_ids)), "ERROR: numpy object not in cache. And it should be." assert (not in_cache(list_obj_name, cache_ids) ), "ERROR: list object should not be in cache." assert (not in_cache(list_obj_name, {}) ), "ERROR: in cache should return False if dict is empty." # Stop cache stop_cache(smm, cache_queue, cache_process) else: print(NOT_PYTHON_3_8)
def executor(queue, process_name, pipe, conf): # type: (typing.Union[None, Queue], str, Pipe, typing.Any) -> None """Thread main body - Overrides Threading run method. Iterates over the input pipe in order to receive tasks (with their parameters) and process them. Notifies the runtime when each task has finished with the corresponding output value. Finishes when the "quit" message is received. :param queue: Queue where to put exception messages. :param process_name: Process name (Thread-X, where X is the thread id). :param pipe: Pipe to receive and send messages from/to the runtime. :param conf: configuration of the executor. :return: None """ try: # Replace Python Worker's SIGTERM handler. signal.signal(signal.SIGTERM, shutdown_handler) if len(conf.logger.handlers) == 0: # Logger has not been inherited correctly. Happens in MacOS. set_temporary_directory(conf.tmp_dir, create_tmpdir=False) # Reload logger conf.logger, conf.logger_cfg, conf.storage_loggers, _ = \ load_loggers(conf.debug, conf.persistent_storage) # Set the binding in worker mode too context.set_pycompss_context(context.WORKER) logger = conf.logger tracing = conf.tracing storage_conf = conf.storage_conf storage_loggers = conf.storage_loggers # Get a copy of the necessary information from the logger to # re-establish after each task logger_handlers = copy.copy(logger.handlers) logger_level = logger.getEffectiveLevel() logger_formatter = logging.Formatter( logger_handlers[0].formatter._fmt) # noqa storage_loggers_handlers = [] for storage_logger in storage_loggers: storage_loggers_handlers.append(copy.copy(storage_logger.handlers)) # Establish link with the binding-commons to enable task nesting if __debug__: logger.debug(HEADER + "Establishing link with runtime in process " + str(process_name)) # noqa: E501 COMPSs.load_runtime(external_process=False, _logger=logger) COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe) if storage_conf != "null": try: from storage.api import initWorkerPostFork # noqa with event_worker(INIT_WORKER_POSTFORK_EVENT): initWorkerPostFork() except (ImportError, AttributeError): if __debug__: logger.info( HEADER + "[%s] Could not find initWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Start the streaming backend if necessary streaming = False if conf.stream_backend not in [None, "null", "NONE"]: streaming = True if streaming: # Initialize streaming logger.debug(HEADER + "Starting streaming for process " + str(process_name)) try: DistroStreamClientHandler.init_and_start( master_ip=conf.stream_master_ip, master_port=conf.stream_master_port) except Exception as e: logger.error(e) raise e # Connect to Shared memory manager if conf.cache_queue: load_shared_memory_manager() # Process properties alive = True if __debug__: logger.debug(HEADER + "[%s] Starting process" % str(process_name)) # MAIN EXECUTOR LOOP while alive: # Runtime -> pipe - Read command from pipe command = COMPSs.read_pipes() if command != "": if __debug__: logger.debug(HEADER + "[%s] Received command %s" % (str(process_name), str(command))) # Process the command alive = process_message( command, process_name, pipe, queue, tracing, logger, conf.logger_cfg, logger_handlers, logger_level, logger_formatter, storage_conf, storage_loggers, storage_loggers_handlers, conf.cache_queue, conf.cache_ids, conf.cache_profiler) # Stop storage if storage_conf != "null": try: from storage.api import finishWorkerPostFork # noqa with event_worker(FINISH_WORKER_POSTFORK_EVENT): finishWorkerPostFork() except (ImportError, AttributeError): if __debug__: logger.info( HEADER + "[%s] Could not find finishWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Stop streaming if streaming: logger.debug(HEADER + "Stopping streaming for process " + str(process_name)) DistroStreamClientHandler.set_stop() sys.stdout.flush() sys.stderr.flush() if __debug__: logger.debug(HEADER + "[%s] Exiting process " % str(process_name)) pipe.write(QUIT_TAG) pipe.close() except Exception as e: logger.error(e) raise e