def feeder(dictionary: list, queue: multiprocessing.JoinableQueue): """ Feed your words into a queue :param dictionary: :param queue: :return: """ print("Feeding words into the pipeline") count = 0 for word in dictionary: count += 1 while queue.full(): # print("Sleeping .0005 seconds to allow queue to empty.") time.sleep(.0005) queue.put(word) if count % 100000 == 0: print(f"Placed the {count}th word into the queue.")
class RelayAgent(object): """Dummy relay agent. This agent subscribe to the '/decision_result' topic from environment and put the received data into a queue. Then it retrieves and replay the queued data when the its `step()` method is callled. """ def __init__(self, queue_len): self.q = Queue(queue_len) rospy.Subscriber('/decision_result', Int16, self.__mover) def __mover(self, data): try: key = data.data # print "[__mover]: {}".format(key) if self.q.full(): self.q.get_nowait() self.q.task_done() self.q.put(key, timeout=0.1) except Exception as e: print "[__mover]: action enque failed. {}".format(e.message) return def step(self, *args, **kwargs): while True: try: action = self.q.get(timeout=0.1) self.q.task_done() break except: print "[step]: get action failed" time.sleep(0.1) # print "[step()]: action = {}".format(action) return action, {} def set_session(self, *args): return
class CAN: def __init__(self): # Distance Buffer self.distance_buffer = JoinableQueue(100) self.last_received_range_frame = RangeCANFrame(0, 0) # RFID Buffer self.RFID_buffer = JoinableQueue(100) # Bluetooth Remote Control Command Buffer self.btrc_buffer = JoinableQueue(100) # WiFi Control Center Message Buffer self.wfcc_buffer = JoinableQueue(100) # WiFi Train State Message Buffer self.wfts_buffer = JoinableQueue(100) def update_distance_buffer(self, distance_to_obstacle, timestamp): if self.distance_buffer.full(): self.distance_buffer.get() self.distance_buffer.put(RangeCANFrame(distance_to_obstacle, timestamp)) def update_RFID_buffer(self, RFID, timestamp): if self.RFID_buffer.full(): self.RFID_buffer.get() self.RFID_buffer.put(RFIDCANFrame(RFID, timestamp)) def update_btrc_buffer(self, btrc_command, timestamp): if self.btrc_buffer.full(): self.btrc_buffer.get() self.btrc_buffer.put( BluetoothRemoteControlCANFrame(btrc_command, timestamp)) def update_wfcc_buffer(self, wfcc_message, timestamp): if self.wfcc_buffer.full(): self.wfcc_buffer.get() self.wfcc_buffer.put(WiFiControlCenterCANFrame(wfcc_message, timestamp)) def update_wfts_buffer(self, most_recent_position, mode, state, decision, timestamp): if self.wfts_buffer.full(): self.wfts_buffer.get() self.wfts_buffer.put( WiFiTrainStateCANFrame(most_recent_position, mode, state, decision, timestamp)) def get_range_frame(self): if not self.distance_buffer.empty(): self.last_received_range_frame = self.distance_buffer.get_nowait() return self.last_received_range_frame def get_RFID_frame(self): if not self.RFID_buffer.empty(): return self.RFID_buffer.get_nowait() else: return RFIDCANFrame("NaRFID", 0) def get_btrc_frame(self): if not self.btrc_buffer.empty(): return self.btrc_buffer.get_nowait() else: return BluetoothRemoteControlCANFrame("None", 0) def get_wfcc_frame(self): if not self.wfcc_buffer.empty(): wfcc_frame = self.wfcc_buffer.get() return wfcc_frame else: return WiFiControlCenterCANFrame("None", 0) def get_wfts_frame(self): if not self.wfts_buffer.empty(): wfts_frame = self.wfts_buffer.get() return wfts_frame else: return WiFiTrainStateCANFrame(-1, "UNKNOWN", "UNKNOWN", "NO", 0)
pair_buffer = {} scaffold_count = {} # while (not inq.empty()) or sum( [reader.is_alive() for reader in readers] )>0: while True: if args.debug: print("get") try: procid, scaffold, pairs = inq.get() # procid,scaffold,pairs = inq.get(True,10) #print("#got data:",procid,scaffold,len(pairs)) print("#got data from inq:", procid, scaffold, len(pairs), inq.empty(), inq.qsize(), inq.full(), strftime("%Y-%m-%d %H:%M:%S"), sum([reader.is_alive() for reader in readers]), "q.size():", q.qsize(), file=sys.stderr, sep="\t") sys.stderr.flush() sys.stdout.flush() except Exception as e: print(e, file=sys.stderr) if args.top: print("queue get timed out", [reader.cpu_percent() for reader in reader_procs], [worker.cpu_percent() for worker in worker_procs]) #print("#timed out",inq.empty())
class AnalysisManager(): """Manage all analysis' process.""" def __init__(self): # Processing pool. logger.debug("Using pool on %i core" % self.get_parallelism()) # Load modules. self.modules = [] self.load_modules() self.check_module_deps() # Starting worker pool. self.workers = [] self.tasks = JoinableQueue(self.get_parallelism()) self.workers_start() def workers_start(self): """Start workers pool.""" for _ in range(self.get_parallelism()): runner = AnalysisRunner(self.tasks, self.modules) runner.start() self.workers.append(runner) def workers_stop(self): """Stop workers pool.""" # Wait for. for sex_worker in self.workers: sex_worker.join() def get_parallelism(self): """Get the ghiro parallelism level for analysis processing.""" # Check database type. If we detect SQLite we slow down processing to # only one process. SQLite does not support parallelism. if settings.DATABASES["default"]["ENGINE"].endswith("sqlite3"): logger.warning("Detected SQLite database, decreased parallelism to 1. SQLite doesn't support parallelism.") return 1 elif cpu_count() > 1: # Set it to total CPU minus one let or db and other use. return cpu_count() - 1 else: return 1 def load_modules(self): """Load modules.""" # Search for analysis modules, it need to import module directory as package named "modules". for loader_instance, module_name, is_pkg in pkgutil.iter_modules(modules.__path__, modules.__name__ + "."): # Skip packages. if is_pkg: continue # Load module. # NOTE: This code is inspired to Cuckoo Sandbox module loading system. try: module = __import__(module_name, globals(), locals(), ["dummy"], -1) except ImportError as e: logger.error("Unable to import module: %s" % module) else: for class_name, class_pkg in inspect.getmembers(module): if inspect.isclass(class_pkg): # Load only modules which inherits BaseModule. if issubclass(class_pkg, BaseProcessingModule) and class_pkg is not BaseProcessingModule: self.modules.append(class_pkg) logger.debug("Found module: %s" % class_name) # Sort modules by execution order. self.modules.sort(key=lambda x: x.order) def check_module_deps(self): """Check modules for requested deps, if not found removes the module from the list.""" for plugin in self.modules: # NOTE: create the module class instance. if not plugin().check_deps(): self.modules.remove(plugin) logger.warning("Kicked module, requirements not found: %s" % plugin.__name__) def run(self): """Start all analyses.""" # Clean up tasks remaining stale from old runs. if Analysis.objects.filter(state="P").exists(): logger.info("Found %i stale analysis, putting them in queue." % Analysis.objects.filter(state="P").count()) Analysis.objects.filter(state="P").update(state="W") # Infinite finite loop. try: while True: # Fetch tasks waiting processing. tasks = Analysis.objects.filter(state="W").order_by("id") if tasks.exists() and not self.tasks.full(): # Using iterator() to avoid caching. for task in Analysis.objects.filter(state="W").order_by("id").iterator(): self.tasks.put(task) logger.debug("Processing task %s" % task.id) task.state = "P" task.save() elif self.tasks.full(): logger.debug("Queue full. Waiting...") sleep(1) else: logger.debug("No tasks. Waiting...") sleep(1) except KeyboardInterrupt: print("Exiting... (requested by user)") finally: print("Waiting tasks to accomplish...") self.workers_stop() print("Processing done. Have a nice day in the real world.")
class ImagesBatcher(AbstractDataBatcher): def __init__( self, queue_size, batch_size, data_sampler, image_processor=None, audio_processor=None, single_epoch=False, cache_data=False, # TODO: implement me! disk_reader_process_num=1): """ Class for creating sequence of data batches for training or validation. :param queue_size: queue size for Batch readers :param batch_size: size of batches generated :param dataset_parser: dataset structure-related parser with all images and labels :param image_processor: image reading and preprocessing routine :param data_sampler: knows how to sample batches from dataset :param single_epoch: if enabled, image batcher finish one epoch with None batch :param cache_data: do we need to store all data in batcher memory? :param disk_reader_process_num: how many disk readers do we need? """ super(AbstractDataBatcher, self).__init__() # set parameters self.batch_size = batch_size self.epoch_is_finished = False self.batch_queue_balance = 0 if single_epoch: self.sampler_external_info = type('sampler_external_info', (object, ), dict(single_epoch=True)) else: self.sampler_external_info = None # parse given dataset and init data sampler self.data_sampler = data_sampler # set queues if queue_size == -1: queue_size = self.data_sampler.dataset_size() / self.batch_size + 1 self.task_queue = JoinableQueue(queue_size) self.batch_queue = JoinableQueue(queue_size) # init batch disk readers and start they self.data_readers = [] print('disk_reader_process_num:', disk_reader_process_num) for i in range(disk_reader_process_num): self.data_readers.append( (BatchDiskReader(self.task_queue, self.batch_queue, image_processor, audio_processor))) def start(self): self.epoch_is_finished = False # start batch disk readers for reader in self.data_readers: reader.start() # fill task queue with batches to start async reading from disk self.fill_task_queue() def fill_task_queue(self): try: while True: if not self.task_queue.full(): batch = self.data_sampler.sampling( self.batch_size, self.sampler_external_info) if batch is not None: self.task_queue.put_nowait(batch) self.batch_queue_balance += 1 else: self.epoch_is_finished = True break else: break except Exception as e: #Queue.Full: logger.error("ImagesBatcher: ", e) def next_batch(self): """ Returns next batch from data """ if self.epoch_is_finished and self.batch_queue_balance == 0: self.epoch_is_finished = False self.fill_task_queue() return None batch = self.batch_queue.get(block=True) self.batch_queue.task_done() self.batch_queue_balance -= 1 if not self.epoch_is_finished: # fill task queue self.fill_task_queue() return batch def update_sampler(self, target, logits, step, summary_writer): if hasattr(self.data_sampler, 'update'): labels = target.cpu().data.numpy() is_update_sampler = self.data_sampler.update( labels, logits, step, summary_writer) #if is_update_sampler: # self.clear_queue() def clear_queue(self): try: while True: self.task_queue.get_nowait() self.task_queue.task_done() except Exception as e: pass try: while True: self.batch_queue.get_nowait() self.batch_queue.task_done() except Exception as e: pass self.fill_task_queue() def finish(self): for data_reader in self.data_readers: data_reader.deactivate() while not self.task_queue.empty(): self.task_queue.get() self.task_queue.task_done() is_anybody_alive = [ data_reader.is_alive() for data_reader in self.data_readers ].count(True) > 0 while not self.batch_queue.empty() or is_anybody_alive: try: self.batch_queue.get(timeout=1) self.batch_queue.task_done() is_anybody_alive = [ data_reader.is_alive() for data_reader in self.data_readers ].count(True) > 0 except Exception as e: pass self.task_queue.join() self.batch_queue.join() for data_reader in self.data_readers: data_reader.join()
class AnalysisManager(): """Manage all analysis' process.""" def __init__(self): # Processing pool. logger.debug("Using pool on %i core" % self.get_parallelism()) # Load modules. self.modules = [] self.load_modules() self.check_module_deps() # Starting worker pool. self.workers = [] self.tasks = JoinableQueue(self.get_parallelism()) def workers_start(self): """Start workers pool.""" for _ in range(self.get_parallelism()): runner = AnalysisRunner(self.tasks, self.modules) runner.start() self.workers.append(runner) def workers_stop(self): """Stop workers pool.""" # Wait for. for sex_worker in self.workers: sex_worker.join() def get_parallelism(self): """Get the ghiro parallelism level for analysis processing.""" # Check database type. If we detect SQLite we slow down processing to # only one process. SQLite does not support parallelism. if settings.DATABASES["default"]["ENGINE"].endswith("sqlite3"): logger.warning("Detected SQLite database, decreased parallelism to 1. SQLite doesn't support parallelism.") return 1 elif cpu_count() > 1: # Set it to total CPU minus one let or db and other use. return cpu_count() - 1 else: return 1 def load_modules(self): """Load modules.""" # Search for analysis modules, it need to import module directory as package named "modules". for loader_instance, module_name, is_pkg in pkgutil.iter_modules(modules.__path__, modules.__name__ + "."): # Skip packages. if is_pkg: continue # Load module. # NOTE: This code is inspired to Cuckoo Sandbox module loading system. try: module = __import__(module_name, globals(), locals(), ["dummy"], -1) except ImportError as e: logger.error("Unable to import module: %s" % module) else: for class_name, class_pkg in inspect.getmembers(module): if inspect.isclass(class_pkg): # Load only modules which inherits BaseModule. if issubclass(class_pkg, BaseProcessingModule) and class_pkg is not BaseProcessingModule: self.modules.append(class_pkg) logger.debug("Found module: %s" % class_name) # Sort modules by execution order. self.modules.sort(key=lambda x: x.order) def check_module_deps(self): """Check modules for requested deps, if not found removes the module from the list.""" for plugin in self.modules: # NOTE: create the module class instance. if not plugin().check_deps(): self.modules.remove(plugin) logger.warning("Kicked module, requirements not found: %s" % plugin.__name__) def run(self): """Start all analyses.""" # Start workers. self.workers_start() # Clean up tasks remaining stale from old runs. if Analysis.objects.filter(state="P").exists(): logger.info("Found %i stale analysis, putting them in queue." % Analysis.objects.filter(state="P").count()) Analysis.objects.filter(state="P").update(state="W") # Infinite finite loop. try: while True: # Fetch tasks waiting processing. tasks = Analysis.objects.filter(state="W").order_by("id") if tasks.exists() and not self.tasks.full(): # Using iterator() to avoid caching. for task in Analysis.objects.filter(state="W").order_by("id").iterator(): self.tasks.put(task) logger.debug("Processing task %s" % task.id) task.state = "P" task.save() elif self.tasks.full(): logger.debug("Queue full. Waiting...") sleep(1) else: logger.debug("No tasks. Waiting...") sleep(1) except KeyboardInterrupt: print("Exiting... (requested by user)") finally: print("Waiting tasks to accomplish...") self.workers_stop() print("Processing done. Have a nice day in the real world.") def stop(self): """Stops the analysis manager.""" if self.workers: self.workers_stop()
class Multiprocess_Data(object): def __init__(self, data_path, batch_size, capacity, num_threads=4, all_samples=None): self.jobs = JoinableQueue(capacity) self.results = Queue(capacity) self.path = data_path self.threads = [] self.num_theads = num_threads self.batch_size = batch_size self.all_samples = all_samples self.create_processes() # 读取数据路径, 根据自己的数据可以重载该函数 def read_path(self, path): datas = np.loadtxt(path, str) if self.all_samples == None: self.all_samples = datas.shape[0] print('train sets of numbers is: %d' % self.all_samples) return datas #处理数据函数,如旋转,缩放大小等, 根据自己的需求重载此函数 def process_data(self, each_data_path): image_path, maskee_path = each_data_path imge = misc.imread(image_path) masker = misc.imread(maskee_path) img_re = misc.imresize(imge, (256, 256)) masker_re = misc.imresize(masker, (256, 256)) return (img_re, masker_re) def worker(self): while True: try: data_path = self.jobs.get() result = self.process_data(data_path) self.results.put(result) except Exception as err: report(err, True) finally: self.jobs.task_done() def add_jobs(self): temp = 0 datas = self.read_path(self.path) while (True): temp += 1 indices = np.arange(len(datas)) np.random.shuffle(indices) datas = datas[indices] print('temp: %d' % temp) if not self.jobs.full(): for item in datas: self.jobs.put(item) def create_processes(self): add_job_process = Process(target=self.add_jobs) add_job_process.daemon = True add_job_process.start() self.threads.append(add_job_process) for _ in range(self.num_theads): worker_process = Process(target=self.worker) worker_process.daemon = True worker_process.start() self.threads.append(worker_process) print('进程数目: %s' % len(self.threads)) def shuffle_batch(self): image_batch = [] masker_batch = [] for _ in range(self.batch_size): image, masker = self.results.get() masker = np.expand_dims(masker, axis=2) image_batch.append(image) masker_batch.append(masker) return np.stack(image_batch), np.stack(masker_batch) def end_processers(self): for process in self.threads: process.terminate() process.join()
class QueuedMultiProcessor: STOP_SIGNAL = "STOP" QUEUE_SIZE = 1500 SLEEP_IF_FULL = 0.01 def __init__( self, stream, worker, writer, worker_setup=None, writer_setup=None, writer_teardown=None, logger=None, chunksize=1, ): self.worker = worker self.worker_initializer = worker_setup self.writer = writer self.writer_initializer = writer_setup self.writer_teardown = writer_teardown self.chunksize = chunksize self.stream = stream self.logger = logger self.processes = [] self.name = "Main Process" def writer_wrapper(self): self.name = "py_WRITER" setproctitle.setproctitle(self.name) if self.writer_initializer: self.writer_initializer() while True: chunk = self.out_queue.get() if chunk == self.STOP_SIGNAL: break for item in chunk: self.writer(item) if self.writer_teardown: if self.logger: self.logger.info("Tearing down Writer") self.writer_teardown() def worker_wrapper(self, i): self.name = f"py_WORKER_{i}" setproctitle.setproctitle(self.name) if self.worker_initializer: self.worker_initializer() while True: cache = [] chunk = self.in_queue.get() if chunk == self.STOP_SIGNAL: self.in_queue.task_done() break for item in chunk: result = self.worker(item) cache.append(result) self.out_queue.put(cache) self.in_queue.task_done() def kill_all_workers(self, *args): if self.logger: self.logger.critical("KILLING %s", self.name) os.kill(os.getpid(), signal.SIGKILL) def _run(self, n_workers): self.in_queue = JoinableQueue(maxsize=self.QUEUE_SIZE) self.out_queue = Queue(maxsize=self.QUEUE_SIZE) signal.signal(signal.SIGINT, self.kill_all_workers) self.write_process = Process(target=self.writer_wrapper) self.write_process.start() for i, _ in enumerate(range(n_workers), 1): proc = Process(target=self.worker_wrapper, args=(i, )) proc.start() self.processes.append(proc) cache = [] for item in self.stream: cache.append(item) while self.in_queue.full(): time.sleep(self.SLEEP_IF_FULL) if len(cache) > self.chunksize: self.in_queue.put(cache) cache = [] self.in_queue.put(cache) for _ in self.processes: self.in_queue.put(self.STOP_SIGNAL) if self.logger: self.logger.info("joining in_queue") self.in_queue.join() for worker in self.processes: worker.join() if self.logger: self.logger.info("killing processes") self.out_queue.put(self.STOP_SIGNAL) self.write_process.join() def run(self, n_workers=2): setproctitle.setproctitle("py_MAIN") self._run(n_workers)
class ProcessExecutor(Process, metaclass=ABCMeta): def __init__(self, name, shared_memory_handler, CORE_POOL_SIZE: int, MAX_POOL_SIZE: int, KEEP_ALIVE_TIME: float, RETYR_NUM: int, worker_class, TPS=-1): super().__init__() self.__stop_signal = False self.__pause_signal = False self.name = name self.shared_memory_handler = shared_memory_handler self.TPS = TPS self.RETRY_NUM = RETYR_NUM self.CORE_POOL_SIZE = CORE_POOL_SIZE self.MAX_POOL_SIZE = MAX_POOL_SIZE self.KEEP_ALIVE_TIME = KEEP_ALIVE_TIME self.job_queue = JoinableQueue(self.MAX_POOL_SIZE * 2) self.worker_class = worker_class self.workers = {} self.worker_states = {} self.__worker_count = 0 self.__init_worker_states() def __init_worker_states(self): for i in range(self.MAX_POOL_SIZE): self.worker_states[str(i)] = const.WORKER_EMPTY def __get_first_empty_worker_id(self) -> int: for i in range(self.MAX_POOL_SIZE): if self.worker_states[str(i)] == const.WORKER_EMPTY: return i @property def pause_signal(self): return self.__pause_signal @property def stop_signal(self): return self.__stop_signal @property def worker_count(self): count = 0 for k, v in self.worker_states.items(): if v != const.WORKER_EMPTY: count += 1 return count @stop_signal.setter def stop_signal(self, signal: bool): self.__stop_signal = signal @pause_signal.setter def pause_signal(self, signal: bool): self.__pause_signal = signal @abstractmethod def get_job(self) -> object: raise NotImplementedError @abstractmethod def create_worker(self, worker_class, is_core=True, init_job=None) -> object: raise NotImplementedError def update_worker_state(self, id_, state): if not self.worker_states.get(id_): return else: self.worker_states[id_] = state if state == const.WORKER_EMPTY: self.workers[id_] = None def __receiver_process(self): while not self.stop_signal: if self.pause_signal: time.sleep(1) continue job = self.get_job() # self.logger.info(self.worker_states) if self.worker_count < self.CORE_POOL_SIZE: core_worker = self.create_worker(self.worker_class, str(self.__get_first_empty_worker_id()), is_core=True, init_job=job) self.workers[str(core_worker.id)] = core_worker self.update_worker_state(core_worker.id, const.WORKER_INIT) core_worker.start() elif self.job_queue.full(): if self.worker_count < self.MAX_POOL_SIZE: non_core_worker = self.create_worker(self.worker_class, str(self.__get_first_empty_worker_id()), is_core=False, init_job=job) self.workers[str(non_core_worker.id)]= non_core_worker self.update_worker_state(non_core_worker.id, const.WORKER_INIT) non_core_worker.start() else: self.job_queue.put(job) else: self.job_queue.put(job) def __init_inner_log(self): MultiprocessLog.worker_configurer(self.shared_memory_handler.log_message_queue) self.logger = logging.getLogger(self.name) def ran(self): pass def __init_receiver(self): self.receiver = threading.Thread(target=self.__receiver_process) self.receiver.setDaemon(True) self.receiver.start() def before_waiting(self): pass def __wait_for_thread(self): self.receiver.join() def run(self): self.__init_inner_log() self.ran() self.__init_receiver() self.before_waiting() self.__wait_for_thread()
consumers_events = map(lambda x: Event(), xrange(0, CPU_COUNT)) consumers = map( lambda x: Process(target=utils.consume, args=(candidates, consumers_events[x], ( samples, otsu, TARGET, name), check_candidate)), xrange(0, CPU_COUNT)) for p in consumers: p.daemon = True map(lambda proc: proc.start(), consumers) producers_events = map(lambda x: Event(), xrange(0, CPU_COUNT)) producers = map( lambda x: Process(target=sample_get, args=(candidates, producers_events[x], (size, FILE, MASK, (Xpdf, Ypdf)))), xrange(0, CPU_COUNT)) for p in producers: p.daemon = True map(lambda proc: proc.start(), producers) while not samples.full(): if samples.qsize() > 0: print samples.qsize() time.sleep(10) map(lambda event: event.set(), producers_events) candidates.join() map(lambda event: event.set(), consumers_events) time.sleep(1) end = datetime.now() print end - start
histogram_merge_worker.start() if args.top: reader_procs = [ psutil.Process(reader.pid) for reader in readers ] worker_procs = [ psutil.Process(worker.pid) for worker in workers ] pair_buffer={} scaffold_count={} # while (not inq.empty()) or sum( [reader.is_alive() for reader in readers] )>0: while True: if args.debug: print("get") try: procid,scaffold,pairs = inq.get() # procid,scaffold,pairs = inq.get(True,10) #print("#got data:",procid,scaffold,len(pairs)) print("#got data from inq:",procid,scaffold,len(pairs),inq.empty(),inq.qsize(),inq.full(),strftime("%Y-%m-%d %H:%M:%S"),sum( [reader.is_alive() for reader in readers] ),"q.size():",q.qsize(),file=sys.stderr,sep="\t") sys.stderr.flush() sys.stdout.flush() except Exception as e: print(e,file=sys.stderr) if args.top: print("queue get timed out",[reader.cpu_percent() for reader in reader_procs],[worker.cpu_percent() for worker in worker_procs]) #print("#timed out",inq.empty()) print("#read from queue timed out:",inq.empty(),inq.qsize(),inq.full(),strftime("%Y-%m-%d %H:%M:%S"),sum( [reader.is_alive() for reader in readers] ),file=sys.stderr,sep="\t") sys.stderr.flush() continue if args.debug: print("got") if not scaffold in pair_buffer: pair_buffer[scaffold]=[] pair_buffer[scaffold] += pairs scaffold_count[scaffold] = scaffold_count.get(scaffold,0)+1