def run_job(self, tmp_task): """ Parameters ---------- tmp_task : pynpoint.util.multiproc.TaskInput Task input with the subsets of lines and the job parameters. Returns ------- pynpoint.util.multiproc.TaskResult Task result. """ result_arr = np.zeros( (tmp_task.m_job_parameter[0], tmp_task.m_input_data.shape[1], tmp_task.m_input_data.shape[2])) for i in range(tmp_task.m_input_data.shape[1]): for j in range(tmp_task.m_input_data.shape[2]): result_arr[:, i, j] = apply_function( tmp_data=tmp_task.m_input_data[:, i, j], func=self.m_function, func_args=self.m_function_args) return TaskResult(result_arr, tmp_task.m_job_parameter[1])
def run_job(self, tmp_task): """ Parameters ---------- tmp_task : pynpoint.util.multiproc.TaskInput Input task. Returns ------- pynpoint.util.multiproc.TaskResult Task result. """ result_arr = np.zeros( (tmp_task.m_job_parameter[0], tmp_task.m_input_data.shape[1], tmp_task.m_input_data.shape[2])) for i in six.moves.range(tmp_task.m_input_data.shape[1]): for j in six.moves.range(tmp_task.m_input_data.shape[2]): tmp_line = tmp_task.m_input_data[:, i, j] result_arr[:, i, j] = apply_function(tmp_line, self.m_function, self.m_function_args) return TaskResult(result_arr, tmp_task.m_job_parameter[1])
def apply_function_in_time(self, func, image_in_port, image_out_port, func_args=None): """ Applies a function to all pixel lines in time. Parameters ---------- func : function The input function. image_in_port : pynpoint.core.dataio.InputPort Input port which is linked to the input data. image_out_port : pynpoint.core.dataio.OutputPort Output port which is linked to the results. func_args : tuple, None Additional arguments which are required by the input function. Not used if set to None. Returns ------- NoneType None """ cpu = self._m_config_port.get_attribute('CPU') init_line = image_in_port[:, 0, 0] im_shape = image_in_port.get_shape() size = apply_function(init_line, func, func_args).shape[0] image_out_port.set_all(data=np.zeros((size, im_shape[1], im_shape[2])), data_dim=3, keep_attributes=False) image_in_port.close_port() image_out_port.close_port() capsule = LineProcessingCapsule(image_in_port=image_in_port, image_out_port=image_out_port, num_proc=cpu, function=func, function_args=func_args, data_length=size) capsule.run()
def apply_function_in_time(self, func, image_in_port, image_out_port, func_args=None): """ Applies a function to all pixel lines in time. :param func: The input function. :type func: function :param image_in_port: InputPort which is linked to the input data. :type image_in_port: InputPort :param image_out_port: OutputPort which is linked to the result place. :type image_out_port: OutputPort :param func_args: Additional arguments which are needed by the function *func*. :type func_args: tuple :return: None """ init_line = image_in_port[:, 0, 0] size = apply_function(init_line, func, func_args).shape[0] # if image_out_port.tag == image_in_port.tag and size != image_in_port.get_shape()[0]: # raise ValueError("Input and output port have the same tag while %s is changing " \ # "the length of the signal. Use different input and output ports instead." % func) image_out_port.set_all(np.zeros((size, image_in_port.get_shape()[1], image_in_port.get_shape()[2])), data_dim=3, keep_attributes=False) cpu = self._m_config_port.get_attribute("CPU") line_processor = LineProcessingCapsule(image_in_port, image_out_port, cpu, func, func_args, size) line_processor.run()
def run_job(self, tmp_task: TaskInput) -> TaskResult: """ Parameters ---------- tmp_task : pynpoint.util.multiproc.TaskInput Task input with the subsets of images and the job parameters. Returns ------- pynpoint.util.multiproc.TaskResult Task result. """ result_nimages = tmp_task.m_input_data.shape[0] result_shape = tmp_task.m_job_parameter[0] # first dimension full_shape = [result_nimages] # additional dimensions for item in result_shape: full_shape.append(item) result_arr = np.zeros(full_shape) for i in range(result_nimages): # job parameter contains (result_shape, tuple(stack_slice)) index = tmp_task.m_job_parameter[1][0][0] + i args = update_arguments(index, self.m_nimages, self.m_function_args) result_arr[i, ] = apply_function( tmp_data=tmp_task.m_input_data[i, ], func=self.m_function, func_args=args) sys.stdout.write('.') sys.stdout.flush() return TaskResult(result_arr, tmp_task.m_job_parameter[1])
def apply_function_to_images(self, func, image_in_port, image_out_port, message, func_args=None): """ Function which applies a function to all images of an input port. Stacks of images are processed in parallel if the CPU and MEMORY attribute are set in the central configuration. The number of images per process is equal to the value of MEMORY divided by the value of CPU. Note that the function *func* is not allowed to change the shape of the images if the input and output port have the same tag and ``MEMORY`` is not set to None. Parameters ---------- func : function The function which is applied to all images. Its definitions should be similar to:: def function(image_in, parameter1, parameter2, parameter3) image_in_port : pynpoint.core.dataio.InputPort Input port which is linked to the input data. image_out_port : pynpoint.core.dataio.OutputPort Output port which is linked to the results. message : str Progress message. func_args : tuple Additional arguments that are required by the input function. Returns ------- NoneType None """ memory = self._m_config_port.get_attribute('MEMORY') cpu = self._m_config_port.get_attribute('CPU') nimages = image_in_port.get_shape()[0] if memory == 0 or image_out_port.tag == image_in_port.tag: # load all images in the memory at once if the input and output tag are the # same or if the MEMORY attribute is set to None in the configuration file images = image_in_port.get_all() result = [] start_time = time.time() for i in range(nimages): progress(i, nimages, message+'...', start_time) args = update_arguments(i, nimages, func_args) if args is None: result.append(func(images[i, ])) else: result.append(func(images[i, ], *args)) image_out_port.set_all(np.asarray(result), keep_attributes=True) elif cpu == 1: # process images one-by-one with a single process if CPU is set to 1 image_out_port.del_all_attributes() image_out_port.del_all_data() start_time = time.time() for i in range(nimages): progress(i, nimages, message+'...', start_time) args = update_arguments(i, nimages, func_args) if args is None: result = func(image_in_port[i, ]) else: result = func(image_in_port[i, ], *args) if result.ndim == 1: image_out_port.append(result, data_dim=2) elif result.ndim == 2: image_out_port.append(result, data_dim=3) else: print(message, end='') # process images in parallel in stacks of MEMORY/CPU images image_out_port.del_all_attributes() image_out_port.del_all_data() result = apply_function(tmp_data=image_in_port[0, :, :], func=func, func_args=update_arguments(0, nimages, func_args)) result_shape = result.shape out_shape = [nimages] for item in result_shape: out_shape.append(item) image_out_port.set_all(data=np.zeros(out_shape), data_dim=len(result_shape)+1, keep_attributes=False) image_in_port.close_port() image_out_port.close_port() capsule = StackProcessingCapsule(image_in_port=image_in_port, image_out_port=image_out_port, num_proc=cpu, function=func, function_args=func_args, stack_size=int(memory/cpu), result_shape=result_shape, nimages=nimages) capsule.run() print(' [DONE]')