def get_novel_dict(self, chapter_url_list: list) -> dict: result = self.downloader.get_result(chapter_url_list) result.show_time_cost() result.show_urls_status() print(" 重试失败章节 ".center(shutil.get_terminal_size().columns - 7, '*')) result.retry_failed_urls() result.show_urls_status() print(" 分离章节内容 ".center(shutil.get_terminal_size().columns - 7, '*')) process_number = self.downloader.config.get_config( "multi", "process_number") process_number = int(process_number // 1.5) if process_number > 2 else process_number queue = SimpleQueue() for i in range(process_number): Process(target=self.fill_novel_dict, args=(chapter_url_list[i::process_number], result.get_urls_detail_dict(), queue)).start() for i in tqdm(range(len(chapter_url_list)), total=len(chapter_url_list), desc="分离章节内容", unit="章节", postfix={"process": process_number}): queue.get() return result.get_urls_detail_dict()
def merge_db(db_folder, new_db_name, db_to_merge): assert path.exists( db_folder ), '`{}` is a wrong path to db folder, please correct it.'.format( db_folder) shutdown = Event() writer_queue = SimpleQueue() writer = Writer(db_folder=db_folder, db_name=new_db_name, queue=writer_queue, shutdown=shutdown) reader = Reader(db_folder=db_folder, db_to_merge=db_to_merge, queue=writer_queue, shutdown=shutdown) reader.start() writer.start() pbar = tqdm(total=len(db_to_merge)) c = 0 while not shutdown.is_set(): try: new_c = writer.counter.value progress = new_c - c if progress > 0: pbar.update(progress) c = new_c Event().wait(2) except KeyboardInterrupt: print() print("Main thread grab the keyboard interrupt") break shutdown.set() pbar.close() if writer.is_alive(): print("Waiting writer...") writer.join() print("WRITER EXECUTED.") if reader.is_alive(): writer_queue.get() print("Waiting reader...") reader.join() print("READER EXECUTED.") print() print("Done.")
class Communicator: def __init__(self): self.transmit_queue = SimpleQueue() self.operations_queue = SimpleQueue() self.client_queue = SimpleQueue() self.emitter_queue = SimpleQueue() def read_operations(self): return self.operations_queue.get() def add_operation(self, operation): self.operations_queue.put([NEW_DATA, IN_PROCESS, operation.data()]) def finish_operation(self, operation): self.operations_queue.put([NEW_DATA, FINISHED, operation.data()]) def finish_task(self, operation, task): self.operations_queue.put( [NEW_DATA, TASK_FINISHED, operation.data(), task.data()]) def sent_task(self, operation, task): self.operations_queue.put( [NEW_DATA, TASK_SENT, operation.data(), task.data()]) def stop_operations(self): self.operations_queue.put([STOP]) def read_transmit(self): return self.transmit_queue.get() def stop_transmit(self): self.transmit_queue.put([STOP]) def notify_end_task(self, operation, task): self.transmit_queue.put([NEW_DATA, operation.data(), task.data()]) def get_current_credits(self): self.operations_queue.put([NEW_DATA, CREDITS]) return self.client_queue.get() def notify_current_credits(self, credits_): self.client_queue.put(credits_) def read_emitter(self): return self.emitter_queue.get() def emit(self, to, data, callback=None): self.emitter_queue.put([NEW_DATA, to, data, callback]) def stop_emitter(self): self.emitter_queue.put([STOP])
def merge_db(db_folder, new_db_name, db_to_merge): assert path.exists(db_folder), '`{}` is a wrong path to db folder, please correct it.'.format(db_folder) shutdown = Event() writer_queue = SimpleQueue() writer = Writer(db_folder=db_folder, db_name=new_db_name, queue=writer_queue, shutdown=shutdown) reader = Reader(db_folder=db_folder, db_to_merge=db_to_merge, queue=writer_queue, shutdown=shutdown) reader.start() writer.start() pbar = tqdm(total=len(db_to_merge)) c = 0 while not shutdown.is_set(): try: new_c = writer.counter.value progress = new_c - c if progress > 0: pbar.update(progress) c = new_c Event().wait(2) except KeyboardInterrupt: print() print("Main thread grab the keyboard interrupt") break shutdown.set() pbar.close() # writer.join() # reader.join() print("writer alive", writer.is_alive()) print("reader alive", reader.is_alive()) if writer.is_alive(): print("Waiting writer...") writer.join() print("WRITER EXECUTED") if reader.is_alive(): print("Waiting reader...") writer_queue.get() print("Waiting reader 2...") reader.join() print("READER EXECUTED") print("Done.")
def learning_loop(exit_flag: mp.Value, gvfs: Sequence[Sequence[Learner]], behaviour_gvf: SARSA, main2gvf: mp.SimpleQueue, gvf2main: mp.SimpleQueue, gvf2plot: mp.SimpleQueue): action, action_prob, obs, x = None, None, None, None # get first state while exit_flag.value == 0 and obs is None: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value == 0: obs, x = main2gvf.get() action, action_prob = behaviour_gvf.policy(obs=obs, x=x) gvf2main.put(action) # main loop while exit_flag.value == 0: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value: break # get data from servos obsp, xp = main2gvf.get() actionp, action_probp = behaviour_gvf.policy(obs=obsp, x=xp) # update weights for g in chain.from_iterable(gvfs): g.update(x, obs, action, action_prob, xp, obsp, actionp, action_probp) # send action gvf2main.put(actionp) # send data to plots gdata = [[g.data(x, obs, action, xp, obsp) for g in gs] for gs in gvfs] data = dict(ChainMap(*chain.from_iterable(gdata))) data['obs'] = obs gvf2plot.put(data) # go to next state obs = obsp x = xp action = actionp action_prob = action_probp print('Done learning!')
def learning_loop(exit_flag: mp.Value, gvfs: Sequence[Sequence[GTDLearner]], main2gvf: mp.SimpleQueue, gvf2plot: mp.SimpleQueue): action, action_prob, obs, x = None, None, None, None # get first state while exit_flag.value == 0 and obs is None: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value == 0: action, action_prob, obs, x = main2gvf.get() i = 1 # main loop while exit_flag.value == 0: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value: break i += 1 ude = False rupee = False if 5000 < i < 5100: ude = True if i == 7000: rupee = True # get data from servos actionp, action_probp, obsp, xp = main2gvf.get() # update weights for gs, xi, xpi in zip(gvfs, x, xp): for g in gs: g.update(action, action_prob, obs, obsp, xi, xpi, ude, rupee) # send data to plots gdata = [[g.data(xi, obs, action, xpi, obsp) for g in gs] for gs, xi, xpi in zip(gvfs, x, xp)] data = dict(ChainMap(*chain.from_iterable(gdata))) data['obs'] = obs gvf2plot.put(data) # go to next state obs = obsp x = xp action = actionp action_prob = action_probp print('Done learning!')
def start(self): t1 = time.time() downloader = Downloader( Config("hardworking_av_studio.ini", HARDWORKING_CONFIG_DICT, HARDWORKING_CONFIG_SCHEMA)) urls = ['https://www.dmmsee.zone/studio/0'] urls.extend([ 'https://www.dmmsee.zone/studio/{}{}'.format(i, word) for i in range(1, 40) for word in ' ' + string.ascii_lowercase ]) urls.extend([ 'https://www.dmmsee.zone/studio/{}'.format(i) for i in range(40, 400) ]) print(" config ".center(shutil.get_terminal_size().columns - 1, '*')) downloader.config.list_config() print(" download urls ".center(shutil.get_terminal_size().columns - 1, '*')) self.result = downloader.get_result(urls) self.result.show_time_cost() self.result.show_urls_status() print(" retry failed urls ".center( shutil.get_terminal_size().columns - 1, '*')) self.result.retry_failed_urls() self.result.show_urls_status() if os.path.exists("hardworking_av_studio.txt"): os.remove("hardworking_av_studio.txt") print(" analyzing result ".center( shutil.get_terminal_size().columns - 1, '*')) tmp_time = time.time() analyzing_result_process_number = downloader.config.get_config( 'multi', 'analyzing_result_process_number') queue = SimpleQueue() for i in range(analyzing_result_process_number): Process(target=self.screen_by_mini_date, args=(self.result.get_finished_urls() [i::analyzing_result_process_number], queue)).start() for i in tqdm(range(len(self.result.get_finished_urls())), total=len(self.result.get_finished_urls()), desc="analyzing result", unit="result", postfix={"process": analyzing_result_process_number}): queue.get() print("\nanalysis completed... time cost {:.2f}s".format(time.time() - tmp_time)) print(" result ".center(shutil.get_terminal_size().columns, '*')) print("The result has been written to the current folder:", os.path.join(os.getcwd(), "hardworking_av_studio.txt")) print("total time cost {:.2f}s".format(time.time() - t1)) return True
def _fit(self, X, y, blocks): """Fit base clustering estimators on X.""" self.blocks_ = blocks processes = [] # Here the blocks will be passed to subprocesses data_queue = SimpleQueue() # Here the results will be passed back result_queue = SimpleQueue() for x in range(self.n_jobs): processes.append( mp.Process(target=_parallel_fit, args=(self.fit_, self.partial_fit_, self.base_estimator, self.verbose, data_queue, result_queue))) processes[-1].start() # First n_jobs blocks are sent into the queue without waiting for the # results. This variable is a counter that takes care of this. presend = 0 blocks_computed = 0 blocks_all = len(np.unique(blocks)) for block in self._blocks(X, y, blocks): if presend >= self.n_jobs: b, clusterer = result_queue.get() blocks_computed += 1 if clusterer: self.clusterers_[b] = clusterer else: presend += 1 if self.partial_fit_: if block[0] in self.clusterers_: data_queue.put(('middle', block, self.clusterers_[b])) continue data_queue.put(('middle', block, None)) # Get the last results and tell the subprocesses to finish for x in range(self.n_jobs): if blocks_computed < blocks_all: print("%s blocks computed out of %s" % (blocks_computed, blocks_all)) b, clusterer = result_queue.get() blocks_computed += 1 if clusterer: self.clusterers_[b] = clusterer data_queue.put(('end', None, None)) time.sleep(1) return self
def _fit(self, X, y, blocks): """Fit base clustering estimators on X.""" self.blocks_ = blocks processes = [] # Here the blocks will be passed to subprocesses data_queue = SimpleQueue() # Here the results will be passed back result_queue = SimpleQueue() for x in range(self.n_jobs): processes.append(mp.Process(target=_parallel_fit, args=(self.fit_, self.partial_fit_, self.base_estimator, self.verbose, data_queue, result_queue))) processes[-1].start() # First n_jobs blocks are sent into the queue without waiting for the # results. This variable is a counter that takes care of this. presend = 0 blocks_computed = 0 blocks_all = len(np.unique(blocks)) for block in self._blocks(X, y, blocks): if presend >= self.n_jobs: b, clusterer = result_queue.get() blocks_computed += 1 if clusterer: self.clusterers_[b] = clusterer else: presend += 1 if self.partial_fit_: if block[0] in self.clusterers_: data_queue.put(('middle', block, self.clusterers_[b])) continue data_queue.put(('middle', block, None)) # Get the last results and tell the subprocesses to finish for x in range(self.n_jobs): if blocks_computed < blocks_all: print("%s blocks computed out of %s" % (blocks_computed, blocks_all)) b, clusterer = result_queue.get() blocks_computed += 1 if clusterer: self.clusterers_[b] = clusterer data_queue.put(('end', None, None)) time.sleep(1) return self
def modbus_start(self, ip, port): queue_receive = SimpleQueue() queue_send = SimpleQueue() Process(target=server.run_callback_server, args=(ip, port, queue_receive, queue_send)).start() while True: device, value = queue_receive.get() print("Registrador: %s -> Valor: %s" % (device, value)) print(device, value) regType = device[0:2] regNum = device[2] # Bool enviados para o cliente (python -> labview) if (regType == "di"): queue_send.put([1, 0, 0, 1, 1, 0, 0, 1]) # Bool recebidos do cliente (labview -> python) if (regType == "co"): if (value[0] == 1): self.measure.emit() # Int enviados para o cliente (python -> labview) if (regType == "ir"): self.inputReg.emit(regNum) # tempo para garantir que a variavel vai ser atualizada sleep(0.1) queue_send.put([self.inputRegData]) # Int recebido do cliente (labview -> python) if (regType == "hr"): self.holdReg.emit(regNum, value[0])
def processDatabaseUpdate( databaseFileName: str, databaseUpdateQueue: multiprocessing.SimpleQueue) -> None: # Open database database = Database(databaseFileName) # Process updates while not databaseUpdateQueue.empty(): # Get a task task = databaseUpdateQueue.get() taskName = task[0] # Update modification time if taskName == 'UpdateMtime': filePath = task[1] # Get file info fileInfo = database.getFile(filePath) # Update mtime fileInfo.stats['mtime'] = os.stat(filePath).st_mtime # Update database record database.setFile(fileInfo) # Remove a blob if taskName == 'RemoveBlob': blobId = task[1] # Remove blob database.removeBlob(blobId) # Close database database.commit() database.close()
def BlobRemoveProcess(blobRemoveQueue: multiprocessing.SimpleQueue, credentials: str, bucketName: str) -> None: # Get GCS bucket credentials = Credentials.from_service_account_info( json.loads(credentials)) client = storage.Client(project=credentials.project_id, credentials=credentials) bucket = client.get_bucket(bucketName) # Process tasks until received None while True: # Set process title setproctitle.setproctitle('BlobRemoveProcess') # Get task task = blobRemoveQueue.get() if task is None: break # Extract task name: str = task[0] # Update process title setproctitle.setproctitle('BlobRemoveProcess {}'.format(name)) # Remove blob try: bucket.delete_blob(name) except: print('Exception while deleting blob {}'.format(name))
class Network: def __init__(self): self.cmd_queue = SimpleQueue() self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) print(colored('[STCK]', 'grey'), colored('Network: Initing.', 'white')) def connect(self, server_ip, server_port=8200): self.sock.connect((server_ip, server_port)) print(colored('[STCK]', 'grey'), colored('Network: Connecting.', 'white')) def send(self, cmd): self.sock.send(('%s\n' % cmd).encode()) #print('Network: Send %s' % cmd) def receive(self): recvData = self.sock.recv(1024).decode("utf8").split('\n') for i in range(len(recvData)): self.cmd_queue.put(recvData[i]) def nextCmd(self): return self.cmd_queue.get() def hasCmd(self): return not self.cmd_queue.empty() def disconnect(self): print("Network: Closed") self.sock.close()
def worker(db, log_queue: multiprocessing.SimpleQueue, internal_log) -> None: db.connect() while True: item = log_queue.get() if item is None: break try: db.ensure_connected() db.insert_into('logs', item) except Exception as e: sleep_for = 10 with internal_log.error() as target: target.write('Could not save to database caused by: {0} {1}\n'.format(type(e), str(e))) if 'open' in dir(db.conn): target.write('Database handle state: {}\n'.format(db.conn.open)) target.write('Stack trace: ' + traceback.format_exc()) target.write('Current log: ') json.dump(item, target) target.write('\n\n') target.write('Retry in {} s\n'.format(sleep_for)) sleep(sleep_for) with internal_log.error() as target: target.write('Retrying now.\n'.format(sleep_for)) log_queue.put(item)
def call(*args, **kwargs): terminate = Event() queue = SimpleQueue() process = IProcess(target=Main, args=(terminate, queue, func, args, kwargs), daemon=daemon) process.start() #Get results/errors result, exception = queue.get() process.join(timeout=terminate) while process.is_alive(): process.terminate() #catch errors raised by the callback try: if callback is not None: callback(*cb_args, **cb_kwargs) except: if exception: result += (_format_tb(*sys.exc_info()),) else: result, exception = (_format_tb(*sys.exc_info()),), True if exception: _raise_tb_stack(result) else: return result
def compile_model(model_file, output_file=None): """ Compile network in separate thread To avoid initialising Theano in main thread, compilation must be done in a separate process. The reason for avoidance is that forking a process after CUDA context is initialised is not supported, so, if this is a multiprocess run, processes must be created before importing theano.sandbox.cuda. Where the network is already compiled, a temporary copy is created. :param model_file: File to read network from :param output_file: File to output to. If None, generate a filename :returns: A filename containing a compiled network. """ queue = SimpleQueue() p = Process(target=_compile_model, args=(queue, model_file, output_file)) p.start() p.join() if p.exitcode != 0: output_file = None raise ValueError("Model file {} was neither a network nor compiled network".format(model_file)) else: output_file = queue.get() return output_file
def init_worker(status_queue: multiprocessing.SimpleQueue, param_queue: multiprocessing.SimpleQueue, result_queue: multiprocessing.SimpleQueue) -> None: global result global coverage_run global py_hash_secret global py_random_seed # Make sure the generator is re-seeded, as we have inherited # the seed from the parent process. py_random_seed = random.SystemRandom().randbytes(8) random.seed(py_random_seed) result = ChannelingTestResult(result_queue) if not param_queue.empty(): server_addr, backend_dsn = param_queue.get() if server_addr is not None: os.environ['EDGEDB_TEST_CLUSTER_ADDR'] = json.dumps(server_addr) if backend_dsn: os.environ['EDGEDB_TEST_BACKEND_DSN'] = backend_dsn os.environ['EDGEDB_TEST_PARALLEL'] = '1' coverage_run = devmode.CoverageConfig.start_coverage_if_requested() py_hash_secret = cpython_state.get_py_hash_secret() status_queue.put(True)
def work(version_id, computer_id, index, size, busy_queue: SimpleQueue, result_queue: SimpleQueue, db: Database): time_limit = 100 while True: agents, waypoints = busy_queue.get() start_time = time.time() run_id, grid_data = db.get_grid(version_id, computer_id, index, time_limit, agents, waypoints, size, 20) overhead = time.time() - start_time res = None error = None try: res = func_timeout(time_limit, run_single_from_data, args=(grid_data, )) except FunctionTimedOut: pass except Exception as e: error = e result_queue.put((index, run_id, res, error, overhead)) busy_queue.put((agents, waypoints))
def run(self, tasks, render, update, render_args=(), render_kwargs={}, update_args=(), update_kwargs={}): # establish ipc queues using a manager process task_queue = SimpleQueue() result_queue = SimpleQueue() # start process to generate image samples producer = Process(target=self._producer, args=(tasks, task_queue)) producer.start() # start worker processes workers = [] for pid in range(self._processes): p = Process(target=self._worker, args=(render, render_args, render_kwargs, task_queue, result_queue)) p.start() workers.append(p) # consume results for _ in tasks: result = result_queue.get() update(result, *update_args, **update_kwargs) # shutdown workers for _ in workers: task_queue.put(None)
def test_fastq_removal(): inq = SimpleQueue() outq = SimpleQueue() with open(test_duplicates_path) as r: duplicates = ujson.load(r) duplicates_set = set(duplicates) rdrp = ReadDuplicateRemovalProcess(inq=inq, outq=outq, duplicated_ids=duplicates_set, ) rdrp.start() inq.put(test_data) inq.put('END') result = outq.get() rdrp.join() rdrp.terminate() # Correct number of duplicates removed assert len(result) == len(test_data_dedup) test_not_duplicated = [(r1.name, r2.name) for r1, r2 in result] expected_not_duplicated = [(r1.name, r2.name) for r1, r2 in test_data_dedup] # Correct duplicate names removed assert test_not_duplicated == expected_not_duplicated
def test_tracer_usage_multiprocess(): q = MPQueue() # Similar to test_multiprocess(), ensures that no collisions are # generated between parent and child processes while using # multiprocessing. # Note that we have to be wary of the size of the underlying # pipe in the queue: https://bugs.python.org/msg143081 def target(q): ids_list = list( chain.from_iterable((s.span_id, s.trace_id) for s in [tracer.start_span("s") for _ in range(10)]) ) q.put(ids_list) ps = [mp.Process(target=target, args=(q,)) for _ in range(30)] for p in ps: p.start() for p in ps: p.join() ids_list = list(chain.from_iterable((s.span_id, s.trace_id) for s in [tracer.start_span("s") for _ in range(100)])) ids = set(ids_list) assert len(ids) == len(ids_list), "Collisions found in ids" while not q.empty(): child_ids_list = q.get() child_ids = set(child_ids_list) assert len(child_ids) == len(child_ids_list), "Collisions found in subprocess ids" assert ids & child_ids == set() ids = ids | child_ids # accumulate the ids
def importData(simulator): testSim = simulator q = SimpleQueue() jobs = [] PERIOD_SIZE = 50 BATCH_SIZE = 100 BATCH_COUNT = 1 BATCH_OFFSET = 100 dates = testSim.getAllDates() index = list(range(BATCH_COUNT)) feed = [] threads = 16 running = False count = 0 while 1: if count < threads: for i in random.sample(index, threads-count if len(index) >= threads-count else len(index)): p = Process(target=testSim.processTimePeriod, args=(q, PERIOD_SIZE, dates, BATCH_SIZE * (i + BATCH_OFFSET) + PERIOD_SIZE, BATCH_SIZE)) jobs.append(p) p.start() index.remove(i) count = 0 for p in jobs: if not p.is_alive(): p.terminate() jobs.remove(p) else: count += 1 while not q.empty(): print('Getting') feed.append(q.get()) if count == 0 and len(index) == 0: break return feed
def _pin_memory_loop(in_queue: multiprocessing.SimpleQueue, out_queue: queue.Queue, done_event: threading.Event): while True: try: r = in_queue.get() except Exception: if done_event.is_set(): return raise if r is None: break if isinstance(r[1], torch_loader.ExceptionWrapper): out_queue.put(r) continue idx, batch_content = r batch_indices = batch_content[0] batch = batch_content[1:] try: batch = torch_loader.pin_memory_batch(batch) except Exception: out_queue.put((idx, torch_loader.ExceptionWrapper(sys.exc_info()))) else: out_queue.put((idx, [batch_indices] + batch))
def test_span_api_fork(): q = MPQueue() pid = os.fork() if pid > 0: # parent parent_ids_list = list( chain.from_iterable((s.span_id, s.trace_id) for s in [Span(None, None) for _ in range(100)]) ) parent_ids = set(parent_ids_list) assert len(parent_ids) == len(parent_ids_list), "Collisions found in parent process ids" child_ids_list = q.get() child_ids = set(child_ids_list) assert len(child_ids) == len(child_ids_list), "Collisions found in child process ids" assert parent_ids & child_ids == set() else: # child try: child_ids = list( chain.from_iterable((s.span_id, s.trace_id) for s in [Span(None, None) for _ in range(100)]) ) q.put(child_ids) finally: os._exit(0)
def test_fastq_parsing(): inq = SimpleQueue() outq = SimpleQueue() rdpp = ReadDeduplicationParserProcess(inq=inq, outq=outq, save_hashed_dict_path=test_json_path) rdpp.start() inq.put(test_data) inq.put('END') _ = outq.get() rdpp.join() rdpp.terminate() with open(test_json_path) as r: result = ujson.load(r) hash_function = functools.partial(xxhash.xxh64_intdigest, seed=42) result_expected = {str(hash_function(r1.name + r2.name)): hash_function(r1.sequence + r2.sequence) for r1, r2 in test_data} assert len(result) == len(test_data) assert result == result_expected assert len({x for x in result.values()}) == 2
def test_multiprocess(): q = MPQueue() def target(q): assert sum((_ is _rand.seed for _ in forksafe._registry)) == 1 q.put([_rand.rand64bits() for _ in range(100)]) ps = [mp.Process(target=target, args=(q,)) for _ in range(30)] for p in ps: p.start() for p in ps: p.join() assert p.exitcode == 0 ids_list = [_rand.rand64bits() for _ in range(1000)] ids = set(ids_list) assert len(ids_list) == len(ids), "Collisions found in ids" while not q.empty(): child_ids_list = q.get() child_ids = set(child_ids_list) assert len(child_ids_list) == len(child_ids), "Collisions found in subprocess ids" assert ids & child_ids == set() ids = ids | child_ids # accumulate the ids
def test_tracer_usage_fork(): q = MPQueue() pid = os.fork() # Similar test to test_fork() above except we use the tracer API. # In this case we expect to never have collisions. if pid > 0: # parent parent_ids_list = list( chain.from_iterable((s.span_id, s.trace_id) for s in [tracer.start_span("s") for _ in range(100)]) ) parent_ids = set(parent_ids_list) assert len(parent_ids) == len(parent_ids_list), "Collisions found in parent process ids" child_ids_list = q.get() child_ids = set(child_ids_list) assert len(child_ids) == len(child_ids_list), "Collisions found in child process ids" assert parent_ids & child_ids == set() else: # child try: child_ids = list( chain.from_iterable((s.span_id, s.trace_id) for s in [tracer.start_span("s") for _ in range(100)]) ) q.put(child_ids) finally: # Kill the process so it doesn't continue running the rest of the # test suite in a separate process. Note we can't use sys.exit() # as it raises an exception that pytest will detect as an error. os._exit(0)
def test_fork_pid_check(): q = MPQueue() pid = os.fork() # Generate random numbers in the parent and child processes after forking. # The child sends back their numbers to the parent where we check to see # if we get collisions or not. if pid > 0: # parent rns = {_rand.rand64bits(check_pid=True) for _ in range(100)} child_rns = q.get() if PYTHON_VERSION_INFO >= (3, 7): # Python 3.7+ have fork hooks which should be used # Hence we should not get any collisions assert rns & child_rns == set() else: # Python < 3.7 we have the pid check so there also # should not be any collisions. assert rns & child_rns == set() else: # child try: rngs = {_rand.rand64bits(check_pid=True) for _ in range(100)} q.put(rngs) finally: # Kill the process so it doesn't continue running the rest of the # test suite in a separate process. Note we can't use sys.exit() # as it raises an exception that pytest will detect as an error. os._exit(0)
def __init__(self, queue: SimpleQueue, address: str, port: int, entWin: Window): """Initialize a level generator conn: Used to communicate with the Chase address: Connection address to start a new mcpi connection port: Connection port to start a new mcpi connection entrance: The entrance window to begin level with """ self.queue = queue self.mc = mmc.Minecraft.create(address, port) self.entWin = entWin self._construct() self.players = [] while True: while not queue.empty(): rec: Tuple[Cmd, List] = queue.get() # New msg if rec[0] == Cmd.TERM: self._cleanup() return elif rec[0] == Cmd.ENT: self.players.extend(rec[1]) elif rec[0] == Cmd.EXI: for i in rec[1]: try: self.players.remove(i) except ValueError: sys.stderr.write( f"Player(id) {i} not found in {self}!") self._loop()
def run(self, runnable): # pylint: disable=W0201 self.runnable = runnable yield self.prepare_status("started") name = self.runnable.kwargs.get("name") # if name was passed correctly, run the Avocado Asset utility if name is not None: asset_hash = self.runnable.kwargs.get("asset_hash") algorithm = self.runnable.kwargs.get("algorithm") locations = self.runnable.kwargs.get("locations") expire = self.runnable.kwargs.get("expire") if expire is not None: expire = data_structures.time_to_seconds(str(expire)) cache_dirs = self.runnable.config.get("datadir.paths.cache_dirs") if cache_dirs is None: cache_dirs = settings.as_dict().get("datadir.paths.cache_dirs") # let's spawn it to another process to be able to update the # status messages and avoid the Asset to lock this process queue = SimpleQueue() process = Process( target=self._fetch_asset, args=( name, asset_hash, algorithm, locations, cache_dirs, expire, queue, ), ) process.start() while queue.empty(): time.sleep(RUNNER_RUN_STATUS_INTERVAL) yield self.prepare_status("running") output = queue.get() result = output["result"] stdout = output["stdout"] stderr = output["stderr"] else: # Otherwise, log the missing package name result = "error" stdout = "" stderr = 'At least name should be passed as kwargs using name="uri".' yield self.prepare_status("running", { "type": "stdout", "log": stdout.encode() }) yield self.prepare_status("running", { "type": "stderr", "log": stderr.encode() }) yield self.prepare_status("finished", {"result": result})
def _poll_config_changes(self, config_queue: SimpleQueue): """ Polls configuration changes from the dedicated queue and reloads filters when they happen :param SimpleQueue config_change_queue: queue for passing configuration changes """ while True: config_data = config_queue.get() self.build_probe_config(config_data, hotswap_only=True) self.reload_filters()
class AudioProcessingThread(threading.Thread): """ Chirp Connect audio processing thread """ DEBUG_AUDIO_FILENAME = 'chirp_audio.wav' def __init__(self, parent=None, *args, **kwargs): """ Initialise audio processing. In debug mode, the audio data is saved to file. """ self.sdk = parent.sdk self.sample_size = parent.sample_size self.block_size = parent.block_size self.sample_format = parent.sample_format self.process_input_fn = parent.process_input_fn self.sample_rate = float(parent.sdk.sample_rate) self.block_period = self.block_size / self.sample_rate or 0.1 self.wav_filename = parent.wav_filename or self.DEBUG_AUDIO_FILENAME self.input_queue = SimpleQueue() super(AudioProcessingThread, self).__init__(*args, **kwargs) if self.sdk.debug: import soundfile as sf self.wav_file = sf.SoundFile(self.wav_filename, mode='w', channels=1, samplerate=self.sdk.sample_rate) self.daemon = True self.start() def run(self): """ Continuously process any input data from circular buffer. Note: We need to sleep as much as possible in this thread to restrict CPU usage. """ while self.is_alive(): tstart = time.time() while not self.input_queue.empty(): data = self.input_queue.get() self.process_input_fn(data) if self.sdk.debug and not self.wav_file.closed: self.wav_file.buffer_write(data, dtype=self.sample_size) self.block_period = self.block_size / self.sample_rate tsleep = (self.block_period - ((time.time() - tstart))) if tsleep > 0: time.sleep(tsleep) def stop(self): """ In debug mode, close wav file """ if self.sdk.debug: self.wav_file.close()
def main(args: argparse.Namespace): global EXIT_CODE probe_workers = [] logging.basicConfig( stream=sys.stderr, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', ) curried_handler = partial(termination_handler, probe_workers) for s in HANDLED_SIGNALS: signal.signal(s, curried_handler) config_watcher = setup_config( args.config, watch_config=args.watch_config, min_watch_interval=args.health_check_period, ) out = smart_open(args.output_file, mode='w') output_queue = SimpleQueue() probes = load_probes( output_queue, args.extra_probe_path, args.extra_plugin_path, args.lost_event_telemetry, ) logging.info('Loaded probes: {}'.format(', '.join(probes))) if args.print_and_quit: for probe_name, probe in probes.items(): print('----- {} -----'.format(probe_name)) print(probe.expanded_bpf_text) print('\n') sys.exit(0) for probe in probes.values(): probe_workers.append( Process(target=deregister_signals(probe.start_polling))) probe_workers[-1].start() stop_wrapper = StopFlagWrapper() watchdog_thread = Thread( target=health_and_config_watchdog, args=(probe_workers, out, stop_wrapper, config_watcher, args.health_check_period), daemon=True, ) watchdog_thread.start() try: while True: print(output_queue.get(), file=out) out.flush() except RestartSignal: stop_wrapper.stop() raise except Exception as e: # Terminate everything if something goes wrong EXIT_CODE = 1 logging.error('Encountered unexpected error: {}'.format(e)) for worker in probe_workers: worker.terminate() sys.exit(EXIT_CODE)
def fork_process(logger, group=None, target=None, name=None, args=(), kwargs={}): """ Forks a child, making sure that all exceptions from the child are safely sent to the parent If a target raises an exception, the exception is re-raised in the parent process @return tuple consisting of process exit code and target's return value """ if is_windows(): logger.warn( "Not forking for %s due to Windows incompatibilities (see #184). " "Measurements (coverage, etc.) might be biased." % target ) return fake_windows_fork(group, target, name, args, kwargs) try: sys.modules["tblib.pickling_support"] except KeyError: import tblib.pickling_support tblib.pickling_support.install() q = SimpleQueue() def instrumented_target(*args, **kwargs): ex = tb = None try: send_value = (target(*args, **kwargs), None, None) except: _, ex, tb = sys.exc_info() send_value = (None, ex, tb) try: q.put(send_value) except: _, send_ex, send_tb = sys.exc_info() e_out = Exception(str(send_ex), send_tb, None if ex is None else str(ex), tb) q.put(e_out) p = Process(group=group, target=instrumented_target, name=name, args=args, kwargs=kwargs) p.start() result = q.get() p.join() if isinstance(result, tuple): if result[1]: raise_exception(result[1], result[2]) return p.exitcode, result[0] else: msg = "Fatal error occurred in the forked process %s: %s" % (p, result.args[0]) if result.args[2]: chained_message = "This error masked the send error '%s':\n%s" % ( result.args[2], "".join(traceback.format_tb(result.args[3])), ) msg += "\n" + chained_message ex = Exception(msg) raise_exception(ex, result.args[1])
def learning_loop(exit_flag: mp.Value, gvfs: Sequence[GTDLearner], main2gvf: mp.SimpleQueue, gvf2plot: mp.SimpleQueue): action, action_prob, obs, x = None, None, None, None # get first state while exit_flag.value == 0 and obs is None: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value == 0: action, action_prob, obs, x = main2gvf.get() # main loop while exit_flag.value == 0: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.001) if exit_flag.value: break # get data from servos actionp, action_probp, obsp, xp = main2gvf.get() # update weights for g in gvfs: g.update(action, action_prob, obs, obsp, x, xp) # send data to plots data = [[obs]] + [g.data(x, obs, action, xp, obsp) for g in gvfs] gvf2plot.put(data) # go to next state obs = obsp x = xp action = actionp action_prob = action_probp
def plotting_loop(exit_flag: mp.Value, gvf2plot: mp.SimpleQueue, plots: Sequence[Plot]): while exit_flag.value == 0: if locks: print('plot gp a 1 a') gplock.acquire() print('plot gp a 1 b') while exit_flag.value == 0 and gvf2plot.empty(): if locks: print('plot gp r 1 a') gplock.release() print('plot gp r 1 b') time.sleep(0.001) if locks: print('plot gp a 2 a') gplock.acquire() print('plot gp a 2 b') if locks: print('plot gp r 2 a') gplock.release() print('plot gp r 2 b') if exit_flag.value: break if locks: print('plot gp a 3 a') gplock.acquire() print('plot gp a 3 b') d = gvf2plot.get() if locks: print('plot gp r 3 a') gplock.release() print('plot gp r 3 b') for plot, data in zip(plots, d): plot.update(data) for plot in plots: try: index = np.arange(len(plot.y[0])) np.savetxt(f"{plot.title}.csv", np.column_stack(sum(((np.asarray(y),) for y in plot.y), (index,))), delimiter=',') except ValueError: continue
def _open_frontend(self): from multiprocessing import Process, SimpleQueue connection = SimpleQueue() frontend = Process( target=self._open_frontend_process, args=(connection, [k for k in sys.argv[1:] if k != "--frontend"])) frontend.start() cmdline = connection.get() frontend.join() if self.interactive: argv_backup = list(sys.argv) sys.argv[1:] = cmdline.split() Main.setup_argv(True, True) if self.interactive: sys.argv = argv_backup print("Running with the following command line: %s" % sys.argv)
def plotting_loop(exit_flag: mp.Value, gvf2plot: mp.SimpleQueue, plots: Sequence[Plot]): while exit_flag.value == 0: while exit_flag.value == 0 and gvf2plot.empty(): time.sleep(0.001) if exit_flag.value: break data = gvf2plot.get() for plot in plots: plot.update(data) for plot in plots: index = np.arange(len(plot.y[0])) np.savetxt(f"{plot.title}.csv", sum(((np.asarray(y),) for y in plot.y), (index,)), delimiter=',')
def run_clients(options, workingDir, db_table_set): # Spawn one client for each db.table, up to options.clients at a time exit_event = multiprocessing.Event() processes = [] error_queue = SimpleQueue() interrupt_event = multiprocessing.Event() sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) hook_counter = multiprocessing.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [] try: progress_info = [] arg_lists = [] for db, table in db_table_set: tableSize = int(options.retryQuery("count", query.db(db).table(table).info()['doc_count_estimates'].sum())) progress_info.append((multiprocessing.Value(ctypes.c_longlong, 0), multiprocessing.Value(ctypes.c_longlong, tableSize))) arg_lists.append((db, table, workingDir, options, error_queue, progress_info[-1], sindex_counter, hook_counter, exit_event, )) # Wait for all tables to finish while processes or arg_lists: time.sleep(0.1) while not error_queue.empty(): exit_event.set() # Stop immediately if an error occurs errors.append(error_queue.get()) processes = [process for process in processes if process.is_alive()] if len(processes) < options.clients and len(arg_lists) > 0: newProcess = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) newProcess.start() processes.append(newProcess) update_progress(progress_info, options) # If we were successful, make sure 100% progress is reported # (rows could have been deleted which would result in being done at less than 100%) if len(errors) == 0 and not interrupt_event.is_set() and not options.quiet: utils_common.print_progress(1.0, indent=4) # Continue past the progress output line and print total rows processed def plural(num, text, plural_text): return "%d %s" % (num, text if num == 1 else plural_text) if not options.quiet: print("\n %s exported from %s, with %s, and %s" % (plural(sum([max(0, info[0].value) for info in progress_info]), "row", "rows"), plural(len(db_table_set), "table", "tables"), plural(sindex_counter.value, "secondary index", "secondary indexes"), plural(hook_counter.value, "hook function", "hook functions") )) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if len(errors) != 0: # multiprocessing queues don't handle tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options.debug: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) raise RuntimeError("Errors occurred during export")
def measure_cmd_resources(self, cmd, name, legend, save_bs=False): """Measure system resource usage of a command""" def _worker(data_q, cmd, **kwargs): """Worker process for measuring resources""" try: start_time = datetime.now() ret = runCmd2(cmd, **kwargs) etime = datetime.now() - start_time rusage_struct = resource.getrusage(resource.RUSAGE_CHILDREN) iostat = OrderedDict() with open('/proc/{}/io'.format(os.getpid())) as fobj: for line in fobj.readlines(): key, val = line.split(':') iostat[key] = int(val) rusage = OrderedDict() # Skip unused fields, (i.e. 'ru_ixrss', 'ru_idrss', 'ru_isrss', # 'ru_nswap', 'ru_msgsnd', 'ru_msgrcv' and 'ru_nsignals') for key in ['ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw', 'ru_nivcsw']: rusage[key] = getattr(rusage_struct, key) data_q.put({'ret': ret, 'start_time': start_time, 'elapsed_time': etime, 'rusage': rusage, 'iostat': iostat}) except Exception as err: data_q.put(err) cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) log.info("Timing command: %s", cmd_str) data_q = SimpleQueue() try: proc = Process(target=_worker, args=(data_q, cmd,)) proc.start() data = data_q.get() proc.join() if isinstance(data, Exception): raise data except CommandError: log.error("Command '%s' failed", cmd_str) raise etime = data['elapsed_time'] measurement = OrderedDict([('type', self.SYSRES), ('name', name), ('legend', legend)]) measurement['values'] = OrderedDict([('start_time', data['start_time']), ('elapsed_time', etime), ('rusage', data['rusage']), ('iostat', data['iostat'])]) if save_bs: self.save_buildstats(name) self._append_measurement(measurement) # Append to 'times' array for globalres log e_sec = etime.total_seconds() self.times.append('{:d}:{:02d}:{:05.2f}'.format(int(e_sec / 3600), int((e_sec % 3600) / 60), e_sec % 60))
def spawn_import_clients(options, files_info): # Spawn one reader process for each db.table, as well as many client processes task_queue = SimpleQueue() error_queue = SimpleQueue() exit_event = multiprocessing.Event() interrupt_event = multiprocessing.Event() errors = [] reader_procs = [] client_procs = [] parent_pid = os.getpid() signal.signal(signal.SIGINT, lambda a, b: abort_import(a, b, parent_pid, exit_event, task_queue, client_procs, interrupt_event)) try: progress_info = [] rows_written = multiprocessing.Value(ctypes.c_longlong, 0) for i in xrange(options["clients"]): client_procs.append(multiprocessing.Process(target=client_process, args=(options["host"], options["port"], options["auth_key"], task_queue, error_queue, rows_written, options["force"], options["durability"]))) client_procs[-1].start() for file_info in files_info: progress_info.append((multiprocessing.Value(ctypes.c_longlong, -1), # Current lines/bytes processed multiprocessing.Value(ctypes.c_longlong, 0))) # Total lines/bytes to process reader_procs.append(multiprocessing.Process(target=table_reader, args=(options, file_info, task_queue, error_queue, progress_info[-1], exit_event))) reader_procs[-1].start() # Wait for all reader processes to finish - hooray, polling while len(reader_procs) > 0: time.sleep(0.1) # If an error has occurred, exit out early while not error_queue.empty(): exit_event.set() errors.append(error_queue.get()) reader_procs = [proc for proc in reader_procs if proc.is_alive()] update_progress(progress_info) # Wait for all clients to finish alive_clients = sum([client.is_alive() for client in client_procs]) for i in xrange(alive_clients): task_queue.put(StopIteration()) while len(client_procs) > 0: time.sleep(0.1) client_procs = [client for client in client_procs if client.is_alive()] # If we were successful, make sure 100% progress is reported if len(errors) == 0 and not interrupt_event.is_set(): print_progress(1.0) def plural(num, text): return "%d %s%s" % (num, text, "" if num == 1 else "s") # Continue past the progress output line print("") print("%s imported in %s" % (plural(rows_written.value, "row"), plural(len(files_info), "table"))) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if len(errors) != 0: # multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options["debug"]: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) if len(error) == 4: print("In file: %s" % error[3], file=sys.stderr) raise RuntimeError("Errors occurred during import")
def scan_regionset(regionset, options): """ This function scans all te region files in a regionset object and fills the ScannedRegionFile obj with the results """ total_regions = len(regionset.regions) total_chunks = 0 corrupted_total = 0 wrong_total = 0 entities_total = 0 too_small_total = 0 unreadable = 0 # init progress bar if not options.verbose: pbar = progressbar.ProgressBar( widgets=['Scanning: ', FractionWidget(), ' ', progressbar.Percentage(), ' ', progressbar.Bar(left='[',right=']'), ' ', progressbar.ETA()], maxval=total_regions) # queue used by processes to pass finished stuff q = SimpleQueue() pool = multiprocessing.Pool(processes=options.processes, initializer=_mp_pool_init,initargs=(regionset,options,q)) if not options.verbose: pbar.start() # start the pool # Note to self: every child process has his own memory space, # that means every obj recived by them will be a copy of the # main obj result = pool.map_async(multithread_scan_regionfile, regionset.list_regions(None), max(1,total_regions//options.processes)) # printing status region_counter = 0 while not result.ready() or not q.empty(): time.sleep(0.01) if not q.empty(): r = q.get() if r == None: # something went wrong scanning this region file # probably a bug... don't know if it's a good # idea to skip it continue if not isinstance(r,world.ScannedRegionFile): raise ChildProcessException(r) else: corrupted, wrong, entities_prob, shared_offset, num_chunks = r.get_counters() filename = r.filename # the obj returned is a copy, overwrite it in regionset regionset[r.get_coords()] = r corrupted_total += corrupted wrong_total += wrong total_chunks += num_chunks entities_total += entities_prob if r.status == world.REGION_TOO_SMALL: too_small_total += 1 elif r.status == world.REGION_UNREADABLE: unreadable += 1 region_counter += 1 if options.verbose: if r.status == world.REGION_OK: stats = "(c: {0}, w: {1}, tme: {2}, so: {3}, t: {4})".format( corrupted, wrong, entities_prob, shared_offset, num_chunks) elif r.status == world.REGION_TOO_SMALL: stats = "(Error: not a region file)" elif r.status == world.REGION_UNREADABLE: stats = "(Error: unreadable region file)" print("Scanned {0: <12} {1:.<43} {2}/{3}".format(filename, stats, region_counter, total_regions)) else: pbar.update(region_counter) if not options.verbose: pbar.finish() regionset.scanned = True
class InternalClient(Client): """This client is a fake client which is responsible for firing off all messages from the update notification side, and handling the routing of those messages to users watching. It does not have a socket, so it should not be included in the server's clients dictionary. """ def __init__(self, server, nickname, user, host='localhost'): self.server = server self.nickname = nickname self.realname = nickname self.user = user self.host = host self._readbuffer = "" self._writebuffer = "" self.request_queue = SimpleQueue() self.response_queue = SimpleQueue() # dict of board => list of users self.board_watchers = defaultdict(list) # dict of board, thread => list of users self.thread_watchers = defaultdict(lambda: defaultdict(list)) Process( target=Ami, name='immediate api worker', args=(self.request_queue, self.response_queue) ).start() def loop_hook(self): while not self.response_queue.empty(): result = self.response_queue.get() # Handle exceptions in-band from child workers here. if isinstance(result, StoredException): print(result.traceback) raise RuntimeError( "Exception caught from worker '{}', see above for exception details".format( result.process, )) logger.debug("read from response queue {}".format(result)) send_as = "/{}/{}".format(result.board, result.post_no) # Initial channel loads have identifiers, use them to find out # where to go if result.identifier: client, channel, target = result.identifier client = self.server.get_client(client) logger.debug("initial channel load, using identitifier info: sending to {} on {}".format(client, channel)) if isinstance(target, BoardTarget): self._send_message( client, channel, result.summary, sending_nick=send_as, ) continue elif isinstance(target, ThreadTarget): self._send_message( client, channel, result.comment, sending_nick=send_as, ) continue if result.is_reply: # Send to thread channel channel = "#/{}/{}".format(result.board, result.reply_to) logger.debug("sending reply to channel {}".format(channel)) # TODO: Remove users who have disconnected from the server here for client in self.thread_watchers[result.board][result.reply_to]: logger.debug("sending reply to {}".format(client)) self._send_message( client, channel, result.comment, sending_nick=send_as, ) else: channel = "#/{}/".format(result.board) logger.debug("sending thread update to channel {}".format(channel)) # TODO: Remove users who have disconnected from the server here for client in self.board_watchers[result.board]: self._send_message( client, channel, result.summary, sending_nick=send_as, ) def _parse_prefix(self, prefix): m = re.search( ":(?P<nickname>[^!]*)!(?P<username>[^@]*)@(?P<host>.*)", prefix ) return m.groupdict() @property def socket(self): raise AttributeError('InternalClients have no sockets') def message(self, message): pass # prefix, message = message.split(" ", 1) # prefix = self._parse_prefix(prefix) # self.sending_client = self.server.get_client(prefix['nickname']) # self._readbuffer = message + '\r\n' # self._parse_read_buffer() def client_joined(self, client, channel): logger.debug("InternalClient handling {} joined {}".format(client, channel)) channel_registration_map = { r'#/(.+)/$': self._client_register_board, r'#/(.+)/(\d+)$': self._client_register_thread, } matched_registration = False for regex, register_method in channel_registration_map.items(): m = re.match(regex, channel.name) if m: register_method(client, channel, *m.groups()) matched_registration = True break if not matched_registration: self._send_message( client, channel.name, "This channel ({}) doesn't look like a board. Nothing will happen in this channel.".format(channel.name) ) return def _handle_command(self, command, arguments): # sending_client = self.sending_client # self.sending_client = None # Add handling here for actual input from users other than joins pass def _client_register_board(self, client, channel, board): logger.debug("registering to board: {}, {}, {}".format(client, channel, board)) slash_board = '/{}/'.format(board) self._send_message( client, channel.name, "Welcome to {}, loading threads...".format(slash_board), sending_nick=slash_board, ) target = BoardTarget(board) self.request_queue.put( SubscriptionUpdate.make( action=Action.LoadAndFollow, target=target, payload=(client.nickname, channel.name, target), )) self.board_watchers[board].append(client) def _client_register_thread(self, client, channel, board, thread): logging.debug("registering to thread: {}, {}, {}, {}".format(client, channel, board, thread)) slash_board_thread = '/{}/{}'.format(board, thread) self._send_message( client, channel.name, "Welcome to >>>{}, loading posts...".format(slash_board_thread), sending_nick=slash_board_thread, ) target = ThreadTarget(board, thread) self.request_queue.put( SubscriptionUpdate.make( action=Action.LoadAndFollow, target=target, payload=(client.nickname, channel.name, target), )) # Thread reply_tos are ints when they come back from the API self.thread_watchers[board][int(thread)].append(client) def _send_message(self, client, channel, message, sending_nick=None): if sending_nick: real_nick = self.nickname self.nickname = sending_nick client.message( ":{} PRIVMSG {} :{}".format( self.prefix, channel, message, ) ) if sending_nick: self.nickname = real_nick
class QueuePool(object): Process = QueueProcess def __init__(self, callback, pool_size=1, check_intervall=2): self.task_queue = SimpleQueue() self.result_queue = SimpleQueue() self._callback = callback self._pool = {} # {process_name: process} self._tasks = {} # {task_id: process_name} for _ in range(pool_size): process = self.Process(self.task_queue, self.result_queue) self._pool[process.name] = process process.start() # Check for progress periodically TODO: stop timer when queue is empty! self.timer = QTimer() self.timer.timeout.connect(self._check_for_results) self.timer.start(check_intervall * 1000) def _check_for_results(self): while not self.result_queue.empty(): process_name, task_id, result_object, is_exception, is_ready = self.result_queue.get() if is_ready or is_exception: if task_id in self._tasks: del self._tasks[task_id] else: self._tasks[task_id] = process_name self._callback(task_id, result_object, is_exception, is_ready) def change_check_interval(self, new_interval_in_seconds): try: interval = float(new_interval_in_seconds) except ValueError: return self.timer.stop() self.timer.start(interval * 1000) def change_pool_size(self, new_pool_size): try: diff = int(new_pool_size) - len(self._pool) except ValueError: return if diff < 0: for _ in range(abs(diff)): process_name, process = self._pool.popitem() process.soft_interrupt.set() else: for _ in range(diff): process = QueueProcess(self.task_queue, self.result_queue, function=compute) self._pool[process.name] = process process.start() def add_task(self, task_id, *params): self.task_queue.put([task_id] + list(params)) def cancel_task(self, task_id): process_name = self._tasks.get(task_id) if process_name is None: # task is not active, but it might be part of task_queue where it shall be removed from task_objects = [] while not self.task_queue.empty(): task_objects.append(self.task_queue.get()) for obj in task_objects: if task_id != obj[0]: self.task_queue.put(obj) return process = self._pool.get(process_name) if process is None: # process might be already stopped -> ignore for now return process.hard_interrupt.set() def shutdown(self): for process in self._pool.values(): process.hard_interrupt.set() self.task_queue.put(None) # unblock queue def terminate(self): for process in self._pool.values(): if process.exitcode is None: process.terminate()
def learning_loop(exit_flag: mp.Value, gvfs: Sequence[Sequence[GTDLearner]], main2gvf: mp.SimpleQueue, gvf2plot: mp.SimpleQueue, parsrs: List[Callable]): action, action_prob, obs, x = None, None, None, None # get first state while exit_flag.value == 0 and obs is None: while exit_flag.value == 0 and main2gvf.empty(): time.sleep(0.01) if exit_flag.value == 0: if locks: print('gvf gm a 1 a') gmlock.acquire() print('gvf gm a 1 b') action, action_prob, obs, x = main2gvf.get() if locks: print('gvf gm r 1 a') gmlock.release() print('gvf gm r 1 b') # main loop # tt = 0 # ts = [] while exit_flag.value == 0: # ts.append(time.time() - tt) if tt > 0 else None # print(np.mean(ts)) # tt = time.time() if locks: print('gvf gm a 2 a') gmlock.acquire() print('gvf gm a 2 b') while exit_flag.value == 0 and main2gvf.empty(): if locks: print('gvf gm r 2 a') gmlock.release() print('gvf gm r 2 b') time.sleep(0.01) if locks: print('gvf gm a 3 a') gmlock.acquire() print('gvf gm a 3 b') if locks: print('gvf gm r 3 a') gmlock.release() print('gvf gm r 3 b') if exit_flag.value: break # get data from servos if locks: print('gvf gm a 4 a') gmlock.acquire() print('gvf gm a 4 b') actionp, action_probp, obsp, xp = main2gvf.get() if locks: print('gvf gm r 4 a') gmlock.release() print('gvf gm r 4 b') # update weights for gs, xi, xpi in zip(gvfs, x, xp): for g in gs: g.update(action, action_prob, obs, obsp, xi, xpi) # send data to plots gdata = [g.data(xi, obs, action, xpi, obsp) for gs, xi, xpi in zip(gvfs, x, xp) for g in gs] data = dict(ChainMap(*gdata)) data['obs'] = obs data['x'] = x data = [parse(data) for parse in parsrs] if locks: print('gvf gp a 1 a') gplock.acquire() print('gvf gp a 1 b') # data = np.copy(data) gvf2plot.put(data) if locks: print('gvf gp r 1 a') gplock.release() print('gvf gp r 1 b') # go to next state obs = obsp x = xp action = actionp action_prob = action_probp print('Done learning!')
while True: try: if not self.send_queue.empty(): # If their is something to send message = self.send_queue.get() message = str(len(message.encode())).zfill(4) + message # Prefix four digit byte amount connection.send(message.encode()) print("Sent message") size_prefix = connection.recv(4).decode() # Recieve four bytes if size_prefix == '': # If disconnected print("Connection finished") break if size_prefix: # If message was recieved self.recv_queue.put(connection.recv(int(size_prefix)).decode()) print("Recieved message") except Exception as error: if error.errno != EWOULDBLOCK: # Remove error messages due to non-blocking print(error) # Setup process stuff recv_queue = SimpleQueue() send_queue = SimpleQueue() server = Server(send_queue, recv_queue) server.start() while True: send_queue.put("mayonaise") print(recv_queue.get())
async def servo_loop(device: str, sids: Sequence[int], coder: KanervaCoder, main2gvf: mp.SimpleQueue, gvf2main: mp.SimpleQueue, **kwargs): # objects to read and write from servos sr, sw = await serial_asyncio.open_serial_connection(url=device, **kwargs) # set servo speeds to slowest possible for sid in sids: await send_msg(sr, sw, sid, [0x03, 0x20, 0x00, 0x01]) # set initial action action = initial_action # some constants read_data = [0x02, # read 0x24, # starting from 0x24 0x08] # a string of 8 bytes # read_all = [0x02, # read # 0x00, # starting from the beginning # 0x32] # all the bytes store_data = [] try: for _ in range(20000): # read data from servos byte_data = [await send_msg(sr, sw, sid, read_data) for sid in sids] # convert to human-readable data obs = sum([parse_data(bd) for bd in byte_data], []) + list(action) # get active tiles in kanerva coding active_pts = coder(obs) # send action and features to GVFs gvf_data = (obs, active_pts) main2gvf.put(gvf_data) # get action control GVFs action = gvf2main.get() # send action to servos instructions = [goal_instruction(a) for a in action if a is not None] for sid, instr in zip(sids, instructions): await send_msg(sr, sw, sid, instr) # record data for later store_data.append(gvf_data) np.save('offline_data.npy', store_data) except KeyboardInterrupt: pass finally: sr.read() await sw.drain() for sid in sids: write(sw, sid, [0x03, 0x18, 0x00]) # disable torque
def run_clients(options, db_table_set): # Spawn one client for each db.table exit_event = multiprocessing.Event() processes = [] error_queue = SimpleQueue() interrupt_event = multiprocessing.Event() sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [ ] try: sizes = get_all_table_sizes(options["host"], options["port"], options["auth_key"], db_table_set) progress_info = [] arg_lists = [] for db, table in db_table_set: progress_info.append((multiprocessing.Value(ctypes.c_longlong, 0), multiprocessing.Value(ctypes.c_longlong, sizes[(db, table)]))) arg_lists.append((options["host"], options["port"], options["auth_key"], db, table, options["directory_partial"], options["fields"], options["delimiter"], options["format"], error_queue, progress_info[-1], sindex_counter, exit_event)) # Wait for all tables to finish while len(processes) > 0 or len(arg_lists) > 0: time.sleep(0.1) while not error_queue.empty(): exit_event.set() # Stop rather immediately if an error occurs errors.append(error_queue.get()) processes = [process for process in processes if process.is_alive()] if len(processes) < options["clients"] and len(arg_lists) > 0: processes.append(multiprocessing.Process(target=export_table, args=arg_lists.pop(0))) processes[-1].start() update_progress(progress_info) # If we were successful, make sure 100% progress is reported # (rows could have been deleted which would result in being done at less than 100%) if len(errors) == 0 and not interrupt_event.is_set(): print_progress(1.0) # Continue past the progress output line and print total rows processed def plural(num, text, plural_text): return "%d %s" % (num, text if num == 1 else plural_text) print("") print("%s exported from %s, with %s" % (plural(sum([max(0, info[0].value) for info in progress_info]), "row", "rows"), plural(len(db_table_set), "table", "tables"), plural(sindex_counter.value, "secondary index", "secondary indexes"))) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if len(errors) != 0: # multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options["debug"]: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) raise RuntimeError("Errors occurred during export")