def __init__(self, batch_loader, augseq, queue_size=50, nb_workers="auto"): ia.do_assert(queue_size > 0) self.augseq = augseq self.queue_source = (batch_loader if isinstance( batch_loader, multiprocessing.queues.Queue) else batch_loader.queue) self.queue_result = multiprocessing.Queue(queue_size) if nb_workers == "auto": try: nb_workers = multiprocessing.cpu_count() except (ImportError, NotImplementedError): nb_workers = 1 # try to reserve at least one core for the main process nb_workers = max(1, nb_workers - 1) else: ia.do_assert(nb_workers >= 1) self.nb_workers = nb_workers self.workers = [] self.nb_workers_finished = 0 seeds = iarandom.get_global_rng().generate_seeds_(nb_workers) for i in range(nb_workers): worker = multiprocessing.Process( target=self._augment_images_worker, args=(augseq, self.queue_source, self.queue_result, seeds[i])) worker.daemon = True worker.start() self.workers.append(worker)
def _load_batches(cls, load_batch_func, queue_internal, join_signal, seedval): if seedval is not None: random.seed(seedval) np.random.seed(seedval) iarandom.seed(seedval) try: gen = (load_batch_func() if not ia.is_generator(load_batch_func) else load_batch_func) for batch in gen: ia.do_assert( isinstance(batch, Batch), "Expected batch returned by load_batch_func to " "be of class imgaug.Batch, got %s." % (type(batch), )) batch_pickled = pickle.dumps(batch, protocol=-1) while not join_signal.is_set(): try: queue_internal.put(batch_pickled, timeout=0.005) break except QueueFull: pass if join_signal.is_set(): break except Exception: traceback.print_exc() finally: queue_internal.put("") time.sleep(0.01)
def __init__(self, batch_loader, augseq, queue_size=50, nb_workers="auto"): ia.do_assert(queue_size > 0) self.augseq = augseq self.queue_source = batch_loader if isinstance(batch_loader, multiprocessing.queues.Queue) else batch_loader.queue self.queue_result = multiprocessing.Queue(queue_size) if nb_workers == "auto": try: nb_workers = multiprocessing.cpu_count() except (ImportError, NotImplementedError): nb_workers = 1 # try to reserve at least one core for the main process nb_workers = max(1, nb_workers - 1) else: ia.do_assert(nb_workers >= 1) self.nb_workers = nb_workers self.workers = [] self.nb_workers_finished = 0 seeds = ia.current_random_state().randint(0, 10**6, size=(nb_workers,)) for i in range(nb_workers): worker = multiprocessing.Process( target=self._augment_images_worker, args=(augseq, self.queue_source, self.queue_result, seeds[i]) ) worker.daemon = True worker.start() self.workers.append(worker)
def _load_batches(self, load_batch_func, queue_internal, join_signal, seedval): if seedval is not None: random.seed(seedval) np.random.seed(seedval) ia.seed(seedval) try: gen = load_batch_func() if not ia.is_generator(load_batch_func) else load_batch_func for batch in gen: ia.do_assert(isinstance(batch, Batch), "Expected batch returned by load_batch_func to be of class imgaug.Batch, got %s." % ( type(batch),)) batch_pickled = pickle.dumps(batch, protocol=-1) while not join_signal.is_set(): try: queue_internal.put(batch_pickled, timeout=0.005) break except QueueFull: pass if join_signal.is_set(): break except Exception: traceback.print_exc() finally: queue_internal.put("") time.sleep(0.01)
def __init__(self, load_batch_func, queue_size=50, nb_workers=1, threaded=True): ia.do_assert(queue_size >= 2, "Queue size for BatchLoader must be at least 2, got %d." % (queue_size,)) ia.do_assert(nb_workers >= 1, "Number of workers for BatchLoader must be at least 1, got %d" % (nb_workers,)) self._queue_internal = multiprocessing.Queue(queue_size//2) self.queue = multiprocessing.Queue(queue_size//2) self.join_signal = multiprocessing.Event() self.workers = [] self.threaded = threaded seeds = ia.current_random_state().randint(0, 10**6, size=(nb_workers,)) for i in range(nb_workers): if threaded: worker = threading.Thread( target=self._load_batches, args=(load_batch_func, self._queue_internal, self.join_signal, None) ) else: worker = multiprocessing.Process( target=self._load_batches, args=(load_batch_func, self._queue_internal, self.join_signal, seeds[i]) ) worker.daemon = True worker.start() self.workers.append(worker) self.main_worker_thread = threading.Thread( target=self._main_worker, args=() ) self.main_worker_thread.daemon = True self.main_worker_thread.start()
def __init__(self, shear=(-40, 41), cval=255, vertical=False, name=None, deterministic=False, random_state=None): """Initialize the augmentator # Arguments shear [float or tuple of 2 floats]: if it is a single number, then image will be sheared in that degree. If it is a tuple of 2 numbers, then the shear value will be chosen randomly cval [int]: fill-in value to new pixels """ super(ItalicizeLine, self).__init__(name=name, deterministic=deterministic, random_state=random_state) if isinstance(shear, StochasticParameter): self.shear = shear elif ia.is_single_number(shear): self.shear = Deterministic(shear) elif ia.is_iterable(shear): ia.do_assert( len(shear) == 2, "Expected rotate tuple/list with 2 entries, got {} entries.". format((len(shear)))) ia.do_assert(all([ia.is_single_number(val) for val in shear]), "Expected floats/ints in shear tuple/list.") self.shear = Uniform(shear[0], shear[1]) else: raise Exception( "Expected float, int, tuple/list with 2 entries or " "StochasticParameter. Got {}.".format(type(shear))) self.cval = cval self.vertical = vertical
def __init__(self, angle=(-10, 10), cval=255, name=None, deterministic=False, random_state=None): """Initialize the augmentator # Arguments angle [float or tuple of 2 floats]: if it is a single number, then image will be rotated in that degree. If it is a tuple of 2 numbers, then the angle value will be chosen randomly cval [int]: fill-in value to new pixels """ super(RotateLine, self).__init__(name=name, deterministic=deterministic, random_state=random_state) if isinstance(angle, StochasticParameter): self.angle = angle elif ia.is_single_number(angle): self.angle = Deterministic(angle) elif ia.is_iterable(angle): ia.do_assert( len(angle) == 2, "Expected rotate tuple/list with 2 entries, got {} entries.". format((len(angle)))) ia.do_assert(all([ia.is_single_number(val) for val in angle]), "Expected floats/ints in angle tuple/list.") self.angle = Uniform(angle[0], angle[1]) else: raise Exception( "Expected float, int, tuple/list with 2 entries or " "StochasticParameter. Got {}.".format(type(angle))) self.cval = cval