if pusher: self.pusher.start() self.puller = puller if puller: self.puller.start() if __name__ == "__main__": kill_switch = multiprocessing.Event() initializer(logging.INFO) queue_sub_to_push = multiprocessing.Queue() pusher = ZMQPusherQueue(queue_sub_to_push, kill_switch, host='127.0.0.1', port="5559") subscriber = ZMQSubscriberQueue(queue_sub_to_push, kill_switch, host='127.0.0.1', port="5558") puller = ZMQPuller(kill_switch, pull_host='127.0.0.1', pull_port="5559") service_manager = ServiceManager(subscriber, pusher, puller, kill_switch) try: s = service_manager.get_server()
L.append(tmp) predict = heapq.nlargest(pred_num, range(len(L)), L.__getitem__) res.append(predict) return res # Finetuning def pred_logi_classification(test_data, mu, params, pred_num=1): res = [] for data in test_data: value = np.dot(data, params[:, :-1].T) + params[:, -1] predict = heapq.nlargest(pred_num, range(len(value)), value.__getitem__) res.append(predict) return res # Testing on basic baselines (including cosine, euclid, finetuning) output = mp.Queue() def calc_accuracy(cls, data, oneshot_avg, logi_params, method): cnt = 0.0 result = [] if method == 'cosine': result = cosine_classification(data, oneshot_avg) elif method == 'eucilid': result = eucilid_classification(data, oneshot_avg) elif method == 'logi' or method == 'vager': result = pred_logi_classification(data, oneshot_avg, logi_params) for it in result: if cls in it: cnt += 1.0 output.put([cls, cnt / len(result)]) def testing(base_avg, oneshot_data, test_data, X, W, method, fusion, _logiparam=None):
def get_data(artist=None, album=None, title=None, otherinfo=None, \ request=(), timeout=None, filename=None, analyzer='first_match', plugin_filter=None): ''' Get data about a song :param otherinfo: Other metadata, not worthing a function parameter :type otherinfo: dict :param request: all needed metadata. If empty, all will be searched :type request: tuple :param timeout: timeout in seconds, None for no timeout :rtype: dict ''' # retrievers write in results; when every retriever has finished, waiter # write in results; analyzer read from results results = multiprocessing.Queue() # analyzer write in response, main process read (with optional timeout) response = multiprocessing.Queue() #this is a trick; finished will be filled with useless value, one per #process. When a process exits, it will pop. So, finished.join() is #equivalent to joining every process; the advantage is that it can be done #in a separate process finished = multiprocessing.JoinableQueue() # even worse trick: every "improvement" analyzer does is written here, # and it tries to be the only value, so that main thread can just get() best = multiprocessing.Queue() if analyzer is None: analyzer = 'first_match' analyzer = multiprocessing.Process(target=_get_analyzer(analyzer), args=(request, results, response, best)) analyzer.name = 'analyzer' analyzer.daemon = True analyzer.start() def retriever_wrapper(name, retriever, results, finished): '''Call a retriever, handle its results''' def wrapped(): '''Provide transparent concurrency for retrievers''' finished.get() try: res = retriever() except Exception as exc: results.put((name, 'error', exc)) else: results.put((name, 'ok', res)) finally: finished.task_done() return wrapped processes = [] for name, retriever in get_ready_retrievers(artist, album, title, \ otherinfo, request, timeout, filename, filter_=plugin_filter): finished.put(True) wrapped_retriever = retriever_wrapper(name, retriever, results, finished) p = multiprocessing.Process(target=wrapped_retriever) processes.append(p) p.daemon = True p.name = name p.start() def waiter(q, res): '''wait for every retriever to join, then unlock main flow''' q.join() # when every process has done res.put('finished') w = multiprocessing.Process(target=waiter, args=(finished, results)) w.daemon = True w.start() try: res = response.get(block=True, timeout=timeout) except Queue.Empty as exc: print('no response') try: best_res = best.get_nowait() except Queue.Empty: return None else: print('best I found:', best_res) return best_res else: return res
def __init__(self, *args, **kwargs): super(SimpleChromeArtifactsStage, self).__init__(*args, **kwargs) self._upload_queue = multiprocessing.Queue() self._pkg_dir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR, 'build', self._current_board, 'var', 'db', 'pkg')
def output_queue(self): """The output queue (a :class:`multiprocessing.Queue` object).""" return multiprocessing.Queue(self.concurrency)
for j in range(1, Xtrain.shape[0]): dis = np.linalg.norm(nowXtest - Xtrain[j, :]) if dis < dis_smallest: dis_smallest = dis idx = j ### Now idx is the index for the nearest neighbor ## check whether the predicted label matches the true label if ytest[i] == ytrain[idx]: correct += 1 acc = correct / float(Xtest.shape[0]) return acc start_time = time.time() sharedQueue = mp.Queue() coreCount = 4 delta = Xtest.shape[0] // 4 processes = list((mp.Process(target=subProcess, args=(Xtrain, ytrain, Xtest, ytest, sharedQueue, i * delta, (i + 1) * delta)) for i in range(coreCount - 1))) i = coreCount - 1 processes += [ mp.Process(target=subProcess, args=(Xtrain, ytrain, Xtest, ytest, sharedQueue, i * delta, Xtest.shape[0])) ] for p in processes: p.start()
def semisupervised_learning(g, truth_dict, kwargs_list, nprocs=1, size_ratio=0.1, use_bfs=False, flowmethod="mqi_weighted", use_spectral=True): l1reg_PR_all = [] l1reg_RC_all = [] l1reg_F1_all = [] flow_PR_all = [] flow_RC_all = [] flow_F1_all = [] l1reg_PR_curr = defaultdict(list) l1reg_RC_curr = defaultdict(list) l1reg_F1_curr = defaultdict(list) flow_PR_curr = defaultdict(list) flow_RC_curr = defaultdict(list) flow_F1_curr = defaultdict(list) def wrapper(pid, q_in, q_out): while True: kwargs, kwargs_id, trial_id, delta, ratio = q_in.get() if kwargs is None: break nlabels = len(list(truth_dict.keys())) l1reg_labels = np.zeros(g._num_vertices) - 1 true_labels = np.zeros(g._num_vertices) - 1 flow_labels = np.zeros(g._num_vertices) - 1 ranking = np.zeros(g._num_vertices) - 1 npositives = 0 for lid, label in enumerate(sorted(list(truth_dict.keys()))): truth = truth_dict[label] npositives += len(truth) true_labels[truth] = lid nseeds = int(ratio * len(truth)) np.random.seed(int(1000 * time.time()) % (2**32 - 1)) seeds = np.random.choice(truth, nseeds) if use_spectral: l1reg_ids, l1reg_vals = approximate_PageRank( g, seeds, **kwargs) sorted_indices = np.argsort(-1 * l1reg_vals) for i, idx in enumerate(sorted_indices): if ranking[l1reg_ids[idx]] == -1 or i < ranking[ l1reg_ids[idx]]: ranking[l1reg_ids[idx]] = i l1reg_labels[l1reg_ids[idx]] = lid if use_bfs: seeds = seed_grow_bfs(g, seeds, size_ratio) flow_output = flow_clustering(g, seeds, method=flowmethod, delta=delta)[0] for i, idx in enumerate(flow_output): if flow_labels[idx] == -1: flow_labels[idx] = lid else: flow_labels[idx] = nlabels + 1 if use_spectral: l1reg_PR = np.sum( (l1reg_labels == true_labels)) / (1.0 * np.sum(l1reg_labels != -1)) l1reg_RC = np.sum( (l1reg_labels == true_labels)) / (1.0 * npositives) l1reg_F1 = 2 * (l1reg_PR * l1reg_RC) / ( l1reg_PR + l1reg_RC) if (l1reg_PR + l1reg_RC) > 0 else 0 else: l1reg_PR, l1reg_RC, l1reg_F1 = 0, 0, 0 # l1reg_PR_curr.append(l1reg_PR) # l1reg_RC_curr.append(l1reg_RC) # l1reg_F1_curr.append() flow_PR = np.sum( (flow_labels == true_labels)) / (1.0 * np.sum(flow_labels != -1)) flow_RC = np.sum((flow_labels == true_labels)) / (1.0 * npositives) flow_F1 = 2 * (flow_PR * flow_RC) / (flow_PR + flow_RC) if ( flow_PR + flow_RC) > 0 else 0 # flow_PR_curr.append(flow_PR) # flow_RC_curr.append(flow_RC) # flow_F1_curr.append() q_out.put((kwargs_id, trial_id, l1reg_PR, l1reg_RC, l1reg_F1, flow_PR, flow_RC, flow_F1)) q_in, q_out = mp.Queue(), mp.Queue() ntrials = 20 for kwargs_id in range(len(kwargs_list)): kwargs = copy.deepcopy(kwargs_list[kwargs_id]) delta = kwargs["delta"] del kwargs["delta"] ratio = kwargs["ratio"] del kwargs["ratio"] for trial_id in range(ntrials): q_in.put((kwargs, kwargs_id, trial_id, delta, ratio)) for _ in range(nprocs): q_in.put((None, None, None, None, None)) procs = [ mp.Process(target=wrapper, args=(pid, q_in, q_out)) for pid in range(nprocs) ] for p in procs: p.start() ncounts = 0 while ncounts < len(kwargs_list) * ntrials: if ncounts % 10 == 0: print("Finished " + str(ncounts) + "/" + str(len(kwargs_list) * ntrials) + " experiments.") kwargs_id, trial_id, l1reg_PR, l1reg_RC, l1reg_F1, flow_PR, flow_RC, flow_F1 = q_out.get( ) l1reg_PR_curr[kwargs_id].append(l1reg_PR) l1reg_RC_curr[kwargs_id].append(l1reg_RC) l1reg_F1_curr[kwargs_id].append(l1reg_F1) flow_PR_curr[kwargs_id].append(flow_PR) flow_RC_curr[kwargs_id].append(flow_RC) flow_F1_curr[kwargs_id].append(flow_F1) if trial_id == ntrials - 1: l1reg_PR_all.append((np.mean(l1reg_PR_curr[kwargs_id]), np.std(l1reg_PR_curr[kwargs_id]))) l1reg_RC_all.append((np.mean(l1reg_RC_curr[kwargs_id]), np.std(l1reg_RC_curr[kwargs_id]))) l1reg_F1_all.append((np.mean(l1reg_F1_curr[kwargs_id]), np.std(l1reg_F1_curr[kwargs_id]))) flow_PR_all.append((np.mean(flow_PR_curr[kwargs_id]), np.std(flow_PR_curr[kwargs_id]))) flow_RC_all.append((np.mean(flow_RC_curr[kwargs_id]), np.std(flow_RC_curr[kwargs_id]))) flow_F1_all.append((np.mean(flow_F1_curr[kwargs_id]), np.std(flow_F1_curr[kwargs_id]))) ncounts += 1 for p in procs: p.join() del procs del p del q_in del q_out del wrapper return locals()
def optimize(self, num_vars, objective_function, gradient_function=None, variable_bounds=None, initial_point=None): num_procs = multiprocessing.cpu_count() - 1 num_procs = \ num_procs if self._max_processes is None else min(num_procs, self._max_processes) num_procs = num_procs if num_procs >= 0 else 0 if platform.system() == 'Darwin': # Changed in version 3.8: On macOS, the spawn start method is now the # default. The fork start method should be considered unsafe as it can # lead to crashes. # However P_BFGS doesn't support spawn, so we revert to single process. major, minor, _ = platform.python_version_tuple() if major > '3' or (major == '3' and minor >= '8'): num_procs = 0 logger.warning( "For MacOS, python >= 3.8, using only current process. " "Multiple core use not supported.") elif platform.system() == 'Windows': num_procs = 0 logger.warning("For Windows, using only current process. " "Multiple core use not supported.") queue = multiprocessing.Queue() # bounds for additional initial points in case bounds has any None values threshold = 2 * np.pi if variable_bounds is None: variable_bounds = [(-threshold, threshold)] * num_vars low = [(l if l is not None else -threshold) for (l, u) in variable_bounds] high = [(u if u is not None else threshold) for (l, u) in variable_bounds] def optimize_runner(_queue, _i_pt): # Multi-process sampling _sol, _opt, _nfev = self._optimize(num_vars, objective_function, gradient_function, variable_bounds, _i_pt) _queue.put((_sol, _opt, _nfev)) # Start off as many other processes running the optimize (can be 0) processes = [] for _ in range(num_procs): i_pt = aqua_globals.random.uniform( low, high) # Another random point in bounds proc = multiprocessing.Process(target=optimize_runner, args=(queue, i_pt)) processes.append(proc) proc.start() # While the one _optimize in this process below runs the other processes will # be running to. This one runs # with the supplied initial point. The process ones have their own random one sol, opt, nfev = self._optimize(num_vars, objective_function, gradient_function, variable_bounds, initial_point) for proc in processes: # For each other process we wait now for it to finish and see if it has # a better result than above proc.join() p_sol, p_opt, p_nfev = queue.get() if p_opt < opt: sol, opt = p_sol, p_opt nfev += p_nfev return sol, opt, nfev
def __init__(self): self.queue = multiprocessing.Queue()
'--numberofncs', dest='number_of_ncs', type=int, default=1, help='Number of NCS. (Default=1)') args = parser.parse_args() number_of_ncs = args.number_of_ncs camera_width = int(cv2.VideoCapture(0).get(cv2.CAP_PROP_FRAME_WIDTH)) camera_height = int(cv2.VideoCapture(0).get(cv2.CAP_PROP_FRAME_HEIGHT)) vidfps = 30 try: mp.set_start_method('forkserver') frameBuffer = mp.Queue(10) results = mp.Queue() # Start detection MultiStick # Activation of inferencer p = mp.Process(target=inferencer, args=(results, frameBuffer, number_of_ncs, camera_width, camera_height, vidfps), daemon=True) p.start() processes.append(p) sleep(number_of_ncs * 7) # Start streaming p = mp.Process(target=camThread,
DEVNULL = open(os.devnull, 'w') while True: ip = job_q.get() if ip is None: break try: subprocess.check_call(['ping', '-c1', ip], stdout=DEVNULL) results_q.put(ip) except: pass if __name__ == '__main__': pool_size = 255 jobs = multiprocessing.Queue() results = multiprocessing.Queue() pool = [ multiprocessing.Process(target=pinger, args=(jobs, results)) for i in range(pool_size) ] for p in pool: p.start() for i in range(150, 255): jobs.put('10.45.14.{0}'.format(i)) for p in pool: jobs.put(None)
Protocol: Camera process polls sonar, and takes picture. writes picture to directory. puts notification in queue This process checks queue, if set reads image and puts acknowledgement in queue once acknowledgement read, Camera process may take a new picture. """ pictureExists = False newPicture = False acceptNextImage = True objectImg = "/images/download.jpg" buffer = None # queues to pass messages between processes imageQueue = mp.Queue() ackQueue = mp.Queue() # class app is an instantiation of the touchscreen app. # It contains several pages including Login Page, Landing Page, Regular Items Page and Custom Items page class App(tk.Tk): def __init__(self, *args, **kwargs): tk.Tk.__init__(self, *args, **kwargs) self.attributes('-fullscreen', True) self.canvas = tk.Canvas(self, bg=backgroundColour) self.canvas.pack(fill=tk.BOTH, expand=True) # Set up Menu MainMenu(self)
log = logging.getLogger(f'send-{who}') log.addHandler(stdout_handler) log.setLevel(logging.INFO) return log log = get_logger('main') KAFKA_BOOTSTRAP_SERVERS = ['kafka:9092'] if 'KAFKA_BOOTSTRAP_SERVERS' in os.environ: KAFKA_BOOTSTRAP_SERVERS = os.environ['KAFKA_BOOTSTRAP_SERVERS'].split(',') NUVLA_ENDPOINT = (os.environ.get('NUVLA_ENDPOINT') or 'https://nuvla.io').rstrip('/') work_queue = multiprocessing.Queue() def kafka_consumer(topic, bootstrap_servers, group_id, auto_offset_reset='latest'): consumer = KafkaConsumer( topic, bootstrap_servers=bootstrap_servers, auto_offset_reset=auto_offset_reset, group_id=group_id, key_deserializer=lambda x: str(x.decode()), value_deserializer=lambda x: json.loads(x.decode())) log.info("Kafka consumer created.") return consumer
time.sleep(0.01) except Queue.Empty: pass except (IOError, EOFError, KeyboardInterrupt) as e: debug("got a breaking error: %s" % e) break except Exception as e: debug("EXCEPTION DURING WORKER PROCESSING: %s" % e) traceback.print_exc() break loader = DataLoader() workers = [] for i in range(NUM_WORKERS): main_q = multiprocessing.Queue() res_q = multiprocessing.Queue() worker_p = multiprocessing.Process(target=worker, args=(main_q, res_q, loader)) worker_p.start() workers.append((worker_p, main_q, res_q)) res_q = multiprocessing.Queue() res_p = multiprocessing.Process(target=results, args=(res_q, workers)) res_p.start() def send_data(obj): global cur_worker global workers global pending_results (w_proc, main_q, wrkr_q) = workers[cur_worker]
import multiprocessing as mp def job(q): res = 0 for i in range(1000): res += i + i**2 + i**3 q.put(res) if __name__=='__main__': q = mp.Queue() p1 = mp.Process(target=job,args=(q,)) p2 = mp.Process(target=job,args=(q,)) p1.start() p2.start() p1.join() p2.join() res1 = q.get() res2 = q.get() print(res1,res2)
queue.put(i) time.sleep(0.3) def read_data(queue): while True: # 判断队列是否为空 if queue.empty(): break value =queue.get() print(value) if __name__ == '__main__': #默认队列可以放入多个任意数据 queue = multiprocessing.Queue(3) add_proccess = multiprocessing.Process(target=add_data,args=(queue,)) read_proccess = multiprocessing.Process(target=read_data, args=(queue,)) #启进程执行任务 add_proccess.start() #进程等待一下,,加载完毕数据之后读取 read_proccess.start() #总结:多任务使用线程和进程 #从资源的角度来说,线程更加节省资源 #进程饿销毁的资源比较多 #从代码稳定性来说:多进程要比多线程稳定性要强 #因为一个进程挂掉,不会影响其他应用程序
def get_multiprocessing_logging_queue(): # pylint: disable=invalid-name ''' Mock ''' import multiprocessing return multiprocessing.Queue()
class MultiPlexIO: incoming_messages = multiprocessing.Queue() outgoing_messages = multiprocessing.Queue() exceptions = multiprocessing.Queue() flag = {"inc": True, "out": True, "exc": True} def __init__(self, **kwargs): self.host = kwargs.get("host") self.port = kwargs.get("port") self.qsize = kwargs.get("backlog") if self.qsize: self.role = "server" else: self.role = "client" self.proto = kwargs.get("proto") self.nbytes = kwargs.get("bytes") def serve_forever(self, consumer_target, *target_parameters): evproducer = multiprocessing.Process(target=self.evloop) evproducer.daemon = True consumer = multiprocessing.Process(target=consumer_target, args=(target_parameters)) consumer.daemon = True producer.start() consumer.start() producer.join(1) comsumer.join(1) def createsocket(self, port_offset): if self.proto == "tcp": sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) elif self.proto == "udp": sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) if self.role == "client": sock.connect((self.host, self.port)) elif self.role == "server": sock.setblocking(True) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((self.host, self.port + port_offset)) sock.listen(self.qsize) return sock def evloop(self): flag = True try: rlist = self.createsocket() wlist = self.createsocket() elist = self.createsocket() while flag: readable, writeable, exceptable = select.select([rlist], [wlist], [elist]) self.recvdata(readable) self.senddata(writeable) self.getexceptions(exceptable) except KeyboardInterrupt: flag = False def recvdata(self, readable): if self.role == "server": for each in readable: #data = each.accept()[0].recv(self.nbytes).decode("utf-8").split("\n") data = each.accept()[0].recv(self.nbytes).decode("utf-8") print(data) self.incoming_messages.put_nowait(data) if self.role == "client": for each in readable: data = each.recv(self.nbytes).decode("utf-8").split( "\n") # makes a list print(data) self.incoming_messages.put_nowait(data) def senddata(self, writeable): if self.role in ["client", "server"]: for each in writeable: if self.flag["out"] == False: data = self.outgoing_messages.get() print(data) each.send(bytes(data, "utf-8")) def getexceptions(self, exc): for each in exc: self.exceptions.put_nowait(each) def read_exceptions(self): while (self.exceptions.empty() != True): print(self.exceptions.get())
def setUp(self): self.prompt = "pipeline>" self.q_in = multiprocessing.Queue() self.q_out = multiprocessing.Queue()
def start_capture(self, cli=LinuxCLI(), interface='any', count=0, packet_type='', pcap_filter=None, max_size=0, timeout=None, callback=None, callback_args=None, blocking=False, save_dump_file=False, save_dump_filename=None): """ Capture <count> packets using tcpdump and add them to a Queue of PCAPPackets. Use wait_for_packets to retrieve the packets as a list (or wait for them to come in. The filter parameter should be a set of pcap_rules which can be combined to create practically any pcap_filter ruleset. A callback may also be provided which must be a callable which takes at least a single PCAPPacket as an argument. This callable will be called when each packet arrives along with any provided arguments (as a list). A timeout may be provided in case of a blocking call, which will limit the blocking call to timeout seconds. This time limit only applies to the execution of tcpdump if blocking is set to True. The optional save_dump_file parameter can be set to true to save the temporary packet capture file to the given save file name (use tcp.out.<timestamp> if name not provided) :type cli: LinuxCLI :type interface: str :type count: int :type packet_type: str :type pcap_filter: pcap_rule :type max_size: int :type timeout: int :type callback: callable :type callback_args: list[T] :type blocking: bool :type save_dump_file: bool :type save_dump_filename: str :return: """ # Don't run twice in a row if self.process is not None: raise exceptions.SubprocessFailedException( 'tcpdump process already started') # Set up synchronization queues and events self.data_queue = multiprocessing.Queue() self.subprocess_info_queue = multiprocessing.Queue() self.tcpdump_ready = multiprocessing.Event() self.tcpdump_error = multiprocessing.Event() self.tcpdump_stop = multiprocessing.Event() self.tcpdump_finished = multiprocessing.Event() self.tcpdump_ready.clear() self.tcpdump_error.clear() self.tcpdump_stop.clear() self.tcpdump_finished.clear() self.tcpdump_pid = None kwarg_map = {'cli': cli, 'interface': interface, 'count': count, 'packet_type': packet_type, 'pcap_filter': pcap_filter, 'max_size': max_size, 'flag_set': (self.tcpdump_ready, self.tcpdump_error, self.tcpdump_stop, self.tcpdump_finished), 'packet_queues': (self.data_queue, self.subprocess_info_queue), 'callback': callback, 'callback_args': callback_args, 'save_dump_file': save_dump_file, 'save_dump_filename': save_dump_filename } self.process = multiprocessing.Process(target=tcpdump_start, args=(kwarg_map,)) self.process.start() deadline_time = time.time() + TCPDUMP_LISTEN_START_TIMEOUT while not self.tcpdump_ready.is_set(): if time.time() > deadline_time: self.process.terminate() raise exceptions.SubprocessFailedException( "tcpdump failed to start listening within timeout") if self.tcpdump_error.is_set(): error_info = self.subprocess_info_queue.get(timeout=2) if 'error' in error_info: raise exceptions.SubprocessFailedException( 'tcpdump error { ' + 'retcode[' + str(error_info['returncode']) + '] ' + 'reason [' + error_info['error'] + '] ' + 'stdout [' + error_info['stdout'] + '] ' + 'stderr [' + error_info['stderr'] + '] }') raise exceptions.SubprocessFailedException( 'tcpdump error UNKNOWN') time.sleep(0) if blocking is True: self.process.join(timeout) if self.process.is_alive(): raise exceptions.SubprocessTimeoutException( 'tcpdump failed to receive packets within timeout')
def semisupervised_learning_with_improve(g, truth, kwargs_list, nprocs=1): input_size_all = [] l1reg_PR_all = [] l1reg_RC_all = [] l1reg_F1_all = [] mqi_PR_all = [] mqi_RC_all = [] mqi_F1_all = [] flow_PR_all = [] flow_RC_all = [] flow_F1_all = [] def wrapper(q_in, q_out): while True: kwargs = q_in.get() if kwargs is None: break delta = kwargs["delta"] del kwargs["delta"] ntrials = 0 input_size_curr = [] l1reg_PR_curr = [] l1reg_RC_curr = [] l1reg_F1_curr = [] mqi_PR_curr = [] mqi_RC_curr = [] mqi_F1_curr = [] flow_PR_curr = [] flow_RC_curr = [] flow_F1_curr = [] while ntrials < 20: seed_node = np.random.choice(truth) l1reg_output = spectral_clustering(g, [seed_node], **kwargs)[0] if len(l1reg_output) == 0: continue input_size_curr.append(len(l1reg_output)) if g._weighted: mqi_output = flow_clustering(g, l1reg_output, method="mqi_weighted")[0] flow_output = flow_clustering(g, l1reg_output, method="flow_weighted", delta=delta)[0] else: mqi_output = flow_clustering(g, l1reg_output, method="mqi")[0] flow_output = flow_clustering(g, l1reg_output, method="flow", delta=delta)[0] l1reg_PR = len(set(truth).intersection(l1reg_output)) / ( 1.0 * len(l1reg_output)) l1reg_RC = len( set(truth).intersection(l1reg_output)) / (1.0 * len(truth)) l1reg_PR_curr.append(l1reg_PR) l1reg_RC_curr.append(l1reg_RC) l1reg_F1_curr.append( 2 * (l1reg_PR * l1reg_RC) / (l1reg_PR + l1reg_RC)) if (l1reg_PR + l1reg_RC) > 0 else 0 mqi_PR = len(set(truth).intersection(mqi_output)) / ( 1.0 * len(mqi_output)) mqi_RC = len( set(truth).intersection(mqi_output)) / (1.0 * len(truth)) mqi_PR_curr.append(mqi_PR) mqi_RC_curr.append(mqi_RC) mqi_F1_curr.append( 2 * (mqi_PR * mqi_RC) / (mqi_PR + mqi_RC)) if (mqi_PR + mqi_RC) > 0 else 0 flow_PR = len(set(truth).intersection(flow_output)) / ( 1.0 * len(flow_output)) flow_RC = len( set(truth).intersection(flow_output)) / (1.0 * len(truth)) flow_PR_curr.append(flow_PR) flow_RC_curr.append(flow_RC) flow_F1_curr.append( 2 * (flow_PR * flow_RC) / (flow_PR + flow_RC)) if (flow_PR + flow_RC) > 0 else 0 ntrials += 1 q_out.put((np.mean(input_size_curr), np.std(input_size_curr), np.mean(l1reg_PR_curr), np.std(l1reg_PR_curr), np.mean(l1reg_RC_curr), np.std(l1reg_RC_curr), np.mean(l1reg_F1_curr), np.std(l1reg_F1_curr), np.mean(mqi_PR_curr), np.std(mqi_PR_curr), np.mean(mqi_RC_curr), np.std(mqi_RC_curr), np.mean(mqi_F1_curr), np.std(mqi_F1_curr), np.mean(flow_PR_curr), np.std(flow_PR_curr), np.mean(flow_RC_curr), np.std(flow_RC_curr), np.mean(flow_F1_curr), np.std(flow_F1_curr))) q_in, q_out = mp.Queue(), mp.Queue() for kwargs in kwargs_list: q_in.put(kwargs) for _ in range(nprocs): q_in.put(None) procs = [ mp.Process(target=wrapper, args=(q_in, q_out)) for _ in range(nprocs) ] for p in procs: p.start() ncounts = 0 while ncounts < len(kwargs_list): output = q_out.get() input_size_all.append((output[0], output[1])) l1reg_PR_all.append((output[2], output[3])) l1reg_RC_all.append((output[4], output[5])) l1reg_F1_all.append((output[6], output[7])) mqi_PR_all.append((output[8], output[9])) mqi_RC_all.append((output[10], output[11])) mqi_F1_all.append((output[12], output[13])) flow_PR_all.append((output[14], output[15])) flow_RC_all.append((output[16], output[17])) flow_F1_all.append((output[18], output[19])) ncounts += 1 for p in procs: p.join() return locals()
def run(self, genomic_files, tln_tables): """Run Prodigal across a set of genomes. Parameters ---------- genomic_files : dict Dictionary indicating the genomic and gene file for each genome. tln_tables : Dict[str, int] Mapping of genome id to user-specified translation table. """ # populate worker queue with data to process worker_queue = mp.Queue() writer_queue = mp.Queue() for genome_id, file_path in genomic_files.items(): worker_queue.put((genome_id, file_path, tln_tables.get(genome_id))) for _ in range(self.threads): worker_queue.put(None) try: manager = mp.Manager() out_dict = manager.dict() worker_proc = [ mp.Process(target=self._worker, args=(out_dict, worker_queue, writer_queue)) for _ in range(self.threads) ] writer_proc = mp.Process(target=self._writer, args=(len(genomic_files), writer_queue)) writer_proc.start() for p in worker_proc: p.start() for p in worker_proc: p.join() # Gracefully terminate the program. if p.exitcode != 0: raise ProdigalException( 'Prodigal returned a non-zero exit code.') writer_queue.put(None) writer_proc.join() except Exception: for p in worker_proc: p.terminate() writer_proc.terminate() raise ProdigalException( 'An exception was caught while running Prodigal.') # Report on any genomes which failed to have any genes called result_dict = dict() lq_gids = list() for gid, gid_dict in out_dict.items(): if os.path.getsize(gid_dict['aa_gene_path']) <= 1: lq_gids.append(gid) else: result_dict[gid] = gid_dict if len(lq_gids) > 0: self.logger.warning( f'Skipping {len(lq_gids)} of {len(genomic_files)} ' f'genomes as no genes were called by Prodigal. ' f'Check the genome quality (see gtdb.warnings.log).') self.warnings.warning( f'The following {len(lq_gids)} genomes have ' f'been excluded from analysis due to Prodigal ' f'failing to call any genes:') # If there are few low-quality genomes just output to console. if len(lq_gids) > 10: for lq_gid in lq_gids: self.warnings.info(lq_gid) else: for lq_gid in lq_gids: self.logger.warning(f'Skipping: {lq_gid}') self.warnings.info(lq_gid) return result_dict
def _run_scenario(self, cls, method_name, context, args): """Runs the specified benchmark scenario with given arguments. Every single benchmark scenario iteration is executed with specified frequency (runs per second) in a pool of processes. The scenario will be launched for a fixed number of times in total (specified in the config). :param cls: The Scenario class where the scenario is implemented :param method_name: Name of the method that implements the scenario :param context: Benchmark context that contains users, admin & other information, that was created before benchmark started. :param args: Arguments to call the scenario method with :returns: List of results fore each single scenario iteration, where each result is a dictionary """ times = self.config["times"] timeout = self.config.get("timeout", 0) # 0 means no timeout iteration_gen = utils.RAMInt() cpu_count = multiprocessing.cpu_count() processes_to_start = min(cpu_count, times, self.config.get("max_concurrency", times)) rps_per_worker = float(self.config["rps"]) / processes_to_start times_per_worker, times_overhead = divmod(times, processes_to_start) # Determine concurrency per worker concurrency_per_worker, concurrency_overhead = divmod( self.config.get("max_concurrency", times), processes_to_start) self._log_debug_info(times=times, timeout=timeout, cpu_count=cpu_count, processes_to_start=processes_to_start, rps_per_worker=rps_per_worker, times_per_worker=times_per_worker, times_overhead=times_overhead, concurrency_per_worker=concurrency_per_worker, concurrency_overhead=concurrency_overhead) result_queue = multiprocessing.Queue() def worker_args_gen(times_overhead, concurrency_overhead): """Generate arguments for process worker. Remainder of threads per process division is distributed to process workers equally - one thread per each process worker until the remainder equals zero. The same logic is applied to concurrency overhead. :param times_overhead: remaining number of threads to be distributed to workers :param concurrency_overhead: remaining number of maximum concurrent threads to be distributed to workers """ while True: yield (result_queue, iteration_gen, timeout, rps_per_worker, times_per_worker + (times_overhead and 1), concurrency_per_worker + (concurrency_overhead and 1), context, cls, method_name, args, self.aborted) if times_overhead: times_overhead -= 1 if concurrency_overhead: concurrency_overhead -= 1 process_pool = self._create_process_pool( processes_to_start, _worker_process, worker_args_gen(times_overhead, concurrency_overhead)) self._join_processes(process_pool, result_queue)
def test_generate_VOEvent(self): """ Test that the VOEvent generator converts a trigger into a VOEvent """ # init VOEvent Generator control_queue = mp.Queue() generator = VOEventGenerator(control_queue) # overwrite server location generator.server_host = 'localhost' # events are only generated if send = True generator.send_events = True # start the generator generator.start() sleep(1) # create two triggers, generator should pick highest S/N # define utc of trigger with highest S/N, which is also in the VOEvent filename trigger_utc = '2019-01-02T18:00:00.0' triggers = [{ 'dm': 56.791, 'dm_err': .2, 'width': 2.5, 'snr': 10, 'flux': 0.5, 'ra': 83.63322083333333, 'dec': 22.01446111111111, 'ymw16': 0, 'semiMaj': 15., 'semiMin': 15., 'name': 'B0531+21', 'cb': 0, 'importance': 0.1, 'utc': '2019-01-01T18:00:00.0', 'test': True }, { 'dm': 56.791, 'dm_err': .2, 'width': 2.5, 'snr': 50, 'flux': 0.5, 'ra': 83.63322083333333, 'dec': 22.01446111111111, 'ymw16': 0, 'semiMaj': 15., 'semiMin': 15., 'name': 'B0531+21', 'cb': 17, 'importance': 0.1, 'utc': trigger_utc, 'test': True }] # get the queue VOEventQueueServer.register('get_queue') queue_server = VOEventQueueServer( address=(generator.server_host, generator.server_port), authkey=generator.server_auth.encode()) queue_server.connect() queue = queue_server.get_queue() # send the triggers for trigger in triggers: queue.put(trigger) # wait and stop sleep(5) control_queue.put('stop') generator.join() # check the output file filename = os.path.join(generator.voevent_dir, "{}.xml".format(trigger_utc)) self.assertTrue(os.path.isfile(filename)) # remove output file os.remove(filename)
import multiprocessing import time def count(q): q.put("from client") print(q.get()) c = 80000000 if __name__ == '__main__': q = multiprocessing.Queue() t1 = multiprocessing.Process(target=count, args=(q,)) t1.start() q.put("from server: hello") print(q.get()) t1.join()
if args.cfg_file is not None: cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) cfg.GPU_ID = args.gpu_id # -------------------------------------------------------------------------- # Pycaffe doesn't reliably free GPU memory when instantiated nets are # discarded (e.g. "del net" in Python code). To work around this issue, each # training stage is executed in a separate process using # multiprocessing.Process. # -------------------------------------------------------------------------- # queue for communicated results between processes mp_queue = mp.Queue() # solves, iters, etc. for each training stage solvers, max_iters, rpn_test_prototxt = get_solvers(args.net_name) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print 'Stage 1 RPN, init from ImageNet model' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' cfg.TRAIN.SNAPSHOT_INFIX = 'stage1' mp_kwargs = dict(queue=mp_queue, imdb_name=args.imdb_name, init_model=args.pretrained_model, solver=solvers[0], max_iters=max_iters[0], cfg=cfg) p = mp.Process(target=train_rpn, kwargs=mp_kwargs)
# initialize webcam video object cap = cv2.VideoCapture(0) # width & height of webcam video in pixels -> adjust to your size # adjust values if you see black bars on the sides of capture window frameWidth = 1280 frameHeight = 720 # set width and height in pixels cap.set(cv2.CAP_PROP_FRAME_WIDTH, frameWidth) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, frameHeight) # enable auto gain cap.set(cv2.CAP_PROP_GAIN, 0) # creating a queue to share data to speech process speakQ = multiprocessing.Queue() # creating speech process to not hang processor p1 = multiprocessing.Process(target=speak, args=(speakQ, )) # starting process 1 - speech p1.start() # keeps program running forever until ctrl+c or window is closed while True: # disable scientific notation for clarity np.set_printoptions(suppress=True) # Create the array of the right shape to feed into the keras model. # We are inputting 1x 224x224 pixel RGB image.
my_queue.put(f"{i}: from {mp.current_process().name}", timeout=0.1) # [block[, timeout]] except queue.Full: print('Queue is full') # if i <= 4 and not my_queue.full(): # my_queue.put(f"{i}: from {mp.current_process().name}", timeout=0.1) # [block[, timeout]] except ValueError: print('Queue is closed') break if i == 4: my_queue.put("CLOSED!") my_queue.close() if __name__ == '__main__': my_queue = mp.Queue(maxsize=32) prod_1 = mp.Process(target=proc_func, name="prod 1", args=(my_queue, )) prod_2 = mp.Process(target=proc_func, name="prod 2", args=(my_queue, )) prod_1.start() prod_2.start() closed_count = 0 while my_queue: try: text = my_queue.get(block=False, timeout=0.1) if type(text) is str and len(text) > 0: print(text) # [block[, timeout]] if text == "CLOSED!": closed_count += 1 if closed_count == 2: print('END') break
async def test_grpc_data_runtime_graceful_shutdown(close_method): args = set_pea_parser().parse_args([]) cancel_event = multiprocessing.Event() handler_closed_event = multiprocessing.Event() slow_executor_block_time = 1.0 pending_requests = 5 sent_queue = multiprocessing.Queue() def start_runtime(args, cancel_event, sent_queue, handler_closed_event): with GRPCDataRuntime(args, cancel_event) as runtime: runtime._data_request_handler.handle = lambda *args, **kwargs: time.sleep( slow_executor_block_time ) runtime._data_request_handler.close = ( lambda *args, **kwargs: handler_closed_event.set() ) async def mock(msg): sent_queue.put('') runtime._grpclet.send_message = mock runtime.run_forever() runtime_thread = Process( target=start_runtime, args=(args, cancel_event, sent_queue, handler_closed_event), daemon=True, ) runtime_thread.start() assert GRPCDataRuntime.wait_for_ready_or_shutdown( timeout=5.0, ctrl_address=f'{args.host}:{args.port_in}', shutdown_event=Event() ) request_start_time = time.time() async def task_wrapper(adress, messages_received): msg = _create_test_data_message(len(messages_received)) await Grpclet._create_grpc_stub(adress).Call(msg) messages_received.append(msg) sent_requests = 0 messages_received = [] tasks = [] for i in range(pending_requests): tasks.append( asyncio.create_task( task_wrapper(f'{args.host}:{args.port_in}', messages_received) ) ) sent_requests += 1 await asyncio.sleep(1.0) if close_method == 'TERMINATE': runtime_thread.terminate() else: GRPCDataRuntime.cancel(cancel_event) assert not handler_closed_event.is_set() runtime_thread.join() assert pending_requests == sent_requests assert sent_requests == len(messages_received) assert sent_queue.qsize() == pending_requests assert ( time.time() - request_start_time >= slow_executor_block_time * pending_requests ) assert handler_closed_event.is_set() assert not GRPCDataRuntime.is_ready(f'{args.host}:{args.port_in}')
task_idx, np.mean(total_rewards[-21:-1]), np.min(total_rewards[-21:-1]), np.max(total_rewards[-21:-1]), np.mean(total_steps[-21:-1]), len(total_rewards), ) train_data_queue.put( ([rb['s'] for rb in replay_buffer], returns, advs, [rb['a'] for rb in replay_buffer], summary)) ################################## coordinator_queue = mp.Queue() train_data_queue = mp.Queue() workers = [] for i in range(num_workers): workers.append( mp.Process(target=worker, args=(i, coordinator_queue, train_data_queue), daemon=True)) workers[-1].start() server = tf.train.Server(cluster, job_name='learner', task_index=0) sess = tf.Session(server.target) ppo = PPO(sess) sess.run(tf.global_variables_initializer())