def spin_crawl_threads(state, classifiers, MAX_BIT_SIZE, MAX_DL_THREADS, image_path): print("Running threads...") manager = Manager() location_q = manager.Queue(maxsize=16) image_q = manager.Queue(maxsize=64) state_lock = manager.Lock() generate_location = Process(target=generate_location_thread, args=(location_q, MAX_BIT_SIZE), name="generate_location") classification = Process(target=classification_thread, args=(image_q, classifiers, image_path, state, state_lock), name="classification") download_image_t = Process(target=download_image_thread, args=(location_q, image_q, MAX_DL_THREADS), name="download_image") download_image_t.start() classification.start() generate_location.start() def kill_threads(): for thread in active_children(): thread.terminate() atexit.register(kill_threads) download_image_t.join() classification.join() generate_location.join()
def _save_data(self, path, cb, concat): if not os.path.exists(path): os.makedirs(path) target_filename = os.path.join(path, "{0}.jb".format(self.name)) if self._saving_cache: while True in [p.is_alive() for p in self._saving_ps]: time.sleep(1e-3) p = Process( target=_caches_to_file, args=( self._save_path, self._start_data_segment, self._cur_data_segment, target_filename, cb, concat, ), ) p.start() self._start_data_segment = self._cur_data_segment else: data = self._extract_q(0) p = Process(target=_dump, args=(data, target_filename, cb)) p.start()
def _launch_aggregators(self): """ Launches the various data aggregators, which serialize data from all processes. * DataAggregator - sqlite database for crawl data * LevelDBAggregator - leveldb database for javascript files """ # DataAggregator self.aggregator_status_queue = Queue() self.data_aggregator = Process(target=DataAggregator.DataAggregator, args=(self.manager_params, self.aggregator_status_queue)) self.data_aggregator.daemon = True self.data_aggregator.start() self.manager_params[ 'aggregator_address'] = self.aggregator_status_queue.get( ) # socket location: (address, port) # LevelDB Aggregator if self.ldb_enabled: self.ldb_status_queue = Queue() self.ldb_aggregator = Process( target=LevelDBAggregator.LevelDBAggregator, args=(self.manager_params, self.ldb_status_queue)) self.ldb_aggregator.daemon = True self.ldb_aggregator.start() self.manager_params['ldb_address'] = self.ldb_status_queue.get( ) # socket location: (address, port)
def test2(): """ Launch planetary_imager and process its data """ import os Process(target=worker_process, args=()).start() Process(target=sink_process, args=()).start() os.system('planetary_imager &')
def test_worker_sink(): """ Launch planetary_imager and process its data """ from multiprocess import Process import os Process(target=worker_process, args=()).start() Process(target=sink_process, args=()).start()
def test1(): """ Run the ventilator defined in this function. """ Process(target=start_ventilator, args=()).start() #time.sleep(0.1) Process(target=worker_process, args=()).start() Process(target=sink_process, args=()).start()
def test_client(): """ Launch planetary_imager and process its data """ Process(target=start_ventilator, args=()).start() Process(target=worker_process, args=()).start() Process(target=sink_process, args=()).start() Process(target=client_process, args=()).start()
def mp_process(self,nprocs,func,*args): images=args[0] # def worker(imgs,i,chunksize, out_q,func,*args): # """ The worker function, invoked in a process. 'images' is a # list of images to span the process upon. The results are placed in # a dictionary that's pushed to a queue. # """ # outdict = {} # for imn in range(len(imgs)-1): # print(i*chunksize+imn) # outdict[i*chunksize+imn] = func(imgs[imn],imgs[imn+1],*args[1:],i*chunksize+imn) # out_q.put(outdict) # Each process will get 'chunksize' nums and a queue to put his out # dict into out_q = Queue() chunksize = int(math.ceil((len(images)-1) / float(nprocs))) procs = [] print("Chunks of size:",chunksize) for i in range(nprocs): if i == nprocs-1: p = Process( target=worker, args=(images[chunksize * i:len(images)-1],i,chunksize,out_q,func,*args)) procs.append(p) p.start() self.loading.progress2['value']+=chunksize self.update() else: p = Process( target=worker, args=(images[chunksize * i:chunksize * (i + 1)+1],i,chunksize,out_q,func,*args)) procs.append(p) p.start() self.loading.progress2['value']+=chunksize self.update() # Collect all results into a single result dict. We know how many dicts # with results to expect. resultdict = {} for i in range(nprocs): resultdict.update(out_q.get()) # Wait for all worker processes to finish for p in procs: p.join() results=[] for j in range(len(resultdict)): results.append(resultdict[j]) return results
def __run_processes(target, inqueue, outqueue=None): threads = max(1, (NUM_THREADS - 1)) for n in range(threads): if outqueue: p = Process(target=target, args=(n, inqueue, outqueue)) else: p = Process(target=target, args=(n, inqueue)) p.daemon = True p.start() pp = Process(target=__progress, args=(-1, inqueue)) pp.daemon = True pp.start() inqueue.close() inqueue.join()
def main(args): lines = [] word2int = {} int2word = {} count = 0 line_count = 0 pid = 0 readFile = open(args.FILE_PATH, 'r') writeFile = open(args.OUTPUT_FILE_PATH, 'w') start = time() cpu_count = mp.cpu_count() pool = Pool(cpu_count - 1) processes = [] print('Starting everything...') lock = mp.Lock() for line in readFile: print('line count: {}'.format(line_count)) word1 = line.split('\n')[0].split('\t')[0].split('/')[0] word2 = line.split('\n')[0].split('\t')[1].split('/')[1] weight = line.split('\n')[0].split('\t')[-1] lines.append([word1, word2, weight]) for word in [word1, word2]: if not word in word2int: word2int[word] = count int2word[count] = word count += 1 line_count += 1 if line_count % 8000000 == 0: # senf the lines to be written in new file p = Process(target=assignNumbers, args=(pid, args, word2int, writeFile, lock, lines, len(lines))) processes.append(p) p.start() pid += 1 lines = [] for process in processes: process.join() end = time() print('Total time for the whole process : {} seconds'.format(end - start)) print('proceddings with writign mappings') # pool.map(writeFileModule, [(word2int, 'word2int.eng'), (int2word, 'int2word.eng')]) P = Process(target=writeFileModule, args=(word2int, 'word2int.eng')) Q = Process(target=writeFileModule, args=(int2word, 'int2word.eng')) P.start() Q.start() P.join() Q.join() readFile.close() writeFile.close()
def mp_processTR(self, nprocs, func, *args): images = args[0] TrMat = args[1] TrMatNull = np.array([[[1., 0., 0.], [0., 1., 0.]]]) TrM = np.vstack((TrMatNull, TrMat)) fnames = args[2] print(len(images), len(TrM), len(fnames)) out_q = Queue() chunksize = int(math.ceil((len(images) - 1) / float(nprocs))) procs = [] print("Chunks of size:", chunksize) for i in range(nprocs): if i == nprocs - 1: p = Process(target=workerTR, args=(images[chunksize * i:len(images)], TrM[chunksize * i:len(images)], fnames[chunksize * i:len(images)], i, chunksize, out_q, func, *args[3:])) procs.append(p) p.start() self.loading.progress2['value'] += chunksize self.update() else: p = Process(target=workerTR, args=(images[chunksize * i:chunksize * (i + 1)], TrM[chunksize * i:chunksize * (i + 1)], fnames[chunksize * i:chunksize * (i + 1)], i, chunksize, out_q, func, *args[3:])) procs.append(p) p.start() self.loading.progress2['value'] += chunksize self.update() # Collect all results into a single result dict. We know how many dicts # with results to expect. resultdict = {} for i in range(nprocs): resultdict.update(out_q.get()) # Wait for all worker processes to finish for p in procs: p.join() results = [] for j in range(len(resultdict)): results.append(resultdict[j]) return results
def api_curtains_control(status): if status in STATES: curtain_status_all = get_curtain_status_all() if status == "open" and (curtain_status_all == "closed" or curtain_status_all == "partlyopen"): curtain_status_right = get_curtain_status("right") p_right = None if curtain_status_right == "closed": p_right = Process(target=open_curtain, args=('right', )) p_right.start() curtain_status_left = get_curtain_status("left") p_left = None if curtain_status_left == "closed": p_left = Process(target=open_curtain, args=('left', )) p_left.start() if p_right is not None: p_right.join() if p_left is not None: p_left.join() return get_curtain_status_all() elif status == "closed" and (curtain_status_all == "open" or curtain_status_all == "partlyopen"): curtain_status_right = get_curtain_status("right") p_right = None if curtain_status_right == "open": p_right = Process(target=close_curtain, args=('right', )) p_right.start() curtain_status_left = get_curtain_status("left") p_left = None if curtain_status_left == "open": p_left = Process(target=close_curtain, args=('left', )) p_left.start() if p_right is not None: p_right.join() if p_left is not None: p_left.join() return get_curtain_status_all() return { "error": 400, "curtain_status_all": curtain_status_all, "status": status }, 400
def _launch_aggregators(self): """Launch the necessary data aggregators""" if self.manager_params["output_format"] == "local": self.data_aggregator = SqliteAggregator.SqliteAggregator( self.manager_params, self.browser_params) elif self.manager_params["output_format"] == "s3": self.data_aggregator = S3Aggregator.S3Aggregator( self.manager_params, self.browser_params) else: raise Exception("Unrecognized output format: %s" % self.manager_params["output_format"]) self.data_aggregator.launch() self.manager_params[ 'aggregator_address'] = self.data_aggregator.listener_address # open connection to aggregator for saving crawl details self.sock = clientsocket(serialization='dill') self.sock.connect(*self.manager_params['aggregator_address']) # TODO refactor ldb aggregator to use new base classes if self.ldb_enabled: self.ldb_status_queue = Queue() self.ldb_aggregator = Process( target=LevelDBAggregator.LevelDBAggregator, args=(self.manager_params, self.ldb_status_queue)) self.ldb_aggregator.daemon = True self.ldb_aggregator.start() # socket location: (address, port) self.manager_params['ldb_address'] = self.ldb_status_queue.get()
def start(self, initializer=None, initargs=()): ''' Spawn a server process for this manager object ''' assert self._state.value == State.INITIAL if initializer is not None and not hasattr(initializer, '__call__'): raise TypeError('initializer must be a callable') # pipe over which we will retrieve address of server reader, writer = connection.Pipe(duplex=False) # spawn process which runs a server self._process = Process( target=type(self)._run_server, args=(self._registry, self._address, self._authkey, self._serializer, writer, initializer, initargs), ) ident = ':'.join(str(i) for i in self._process._identity) self._process.name = type(self).__name__ + '-' + ident self._process.start() # get address of server writer.close() self._address = reader.recv() reader.close() # register a finalizer self._state.value = State.STARTED self.shutdown = util.Finalize( self, type(self)._finalize_manager, args=(self._process, self._address, self._authkey, self._state, self._Client), exitpriority=0 )
def test(): NUMBER_OF_PROCESSES = 4 TASKS1 = [(mul, (i, 7)) for i in range(20)] TASKS2 = [(plus, (i, 8)) for i in range(10)] # Create queues task_queue = Queue() done_queue = Queue() # Submit tasks list(map(task_queue.put, TASKS1)) # Start worker processes for i in range(NUMBER_OF_PROCESSES): Process(target=worker, args=(task_queue, done_queue)).start() # Get and print results print('Unordered results:') for i in range(len(TASKS1)): print('\t', done_queue.get()) # Add more tasks using `put()` instead of `putMany()` for task in TASKS2: task_queue.put(task) # Get and print some more results for i in range(len(TASKS2)): print('\t', done_queue.get()) # Tell child processes to stop for i in range(NUMBER_OF_PROCESSES): task_queue.put('STOP')
def _solve(self, X, Y, batch_size): ''' Solve the multi-objective problem by multiple scalarized single-objective solvers. ''' # generate scalarization weights weights = np.random.random((batch_size, self.problem.n_obj)) weights /= np.expand_dims(np.sum(weights, axis=1), 1) # initial solutions X = np.vstack([X, lhs(X.shape[1], batch_size)]) F = self.problem.evaluate(X, return_values_of=['F']) # optimization xs, ys = [], [] queue = Queue() n_active_process = 0 for i in range(batch_size): x0 = X[np.argmin(augmented_tchebicheff(F, weights[i]))] Process(target=optimization, args=(self.problem, x0, weights[i], queue)).start() n_active_process += 1 if n_active_process >= self.n_process: x, y = queue.get() xs.append(x) ys.append(y) n_active_process -= 1 # gather result for _ in range(n_active_process): x, y = queue.get() xs.append(x) ys.append(y) return np.array(xs), np.array(ys)
def fit(self, X, Y): assert not self.fit_done assert len(X) == len(Y) possible_labels = list(set(y_val for y in Y for y_val in y)) job_labels = np.array_split(possible_labels, self.n_jobs) with Manager() as manager: X_proxy = manager.list(X) Y_proxy = manager.list(Y) output_queue = Queue() processes = [ Process(target=sequential_execute, args=(output_queue, get_binary_clf_from_multilabel, [{ 'X': X_proxy, 'Y': Y_proxy, 'label': lbl, 'return_label': True } for lbl in job])) for job in job_labels ] [p.start() for p in processes] results = [output_queue.get() for lbl in possible_labels] # needs to be flattened [p.join() for p in processes] self.classifiers = dict(results) self.fit_done = True
def make_csv(self, lock): file1 = open(self.out_csv1, "w") file1.write("id" + ',' + "level" + '\n') file2 = open(self.out_csv2, "w") file2.write("id" + ',' + "object_name" + '\n') file1.close() file2.close() if __name__ == '__main__': list_of_process = [] list_of_queue1 = [] list_of_queue2 = [] for i in range(len(self.list_of_zips)): list_of_queue1.append(Queue()) list_of_queue2.append(Queue()) list_of_process.append( Process(target=self.parse_Zip, args=(i, lock, list_of_queue1[i], list_of_queue2[i]))) for i in range(len(self.list_of_zips)): list_of_process[i].start() file1 = open(self.out_csv1, "a") for i in range(len(self.list_of_zips)): while list_of_queue1[i].empty() is False: file1.write(list_of_queue1[i].get()[0] + ',' + list_of_queue1[i].get()[1] + '\n') file1.close() '''
def multiprocessor(inpipe, outpipe, controlpipe): def returner_process(inp, outp, task): args, kwargs = inp.get() outpipe.put(task(*args, **kwargs)) return True jobs = [] while True: done = [x for x in jobs if x.ready()] if done: jobs = [x for x in jobs if x not in done] # Avoids race condition! else: sleep(0.1) for thing in done: thing.successful() assert thing.get() while len(jobs) < process_count: cmd = controlpipe.get() if cmd == stop_signal: break elif cmd == True: newjob = Process(target=returner_process, args=(inpipe, outpipe)) newjob.start() jobs.append(newjob) # I *think* the pipes have to be passed explicitly, # but I haven't checked. else: raise Exception outpipe.put(stop_signal)
def test_replace(delay=0.): zwo.init_workers() zwo.init_sink() Process(target=start_ventilator, args=(delay, )).start() while True: time.sleep(10) zwo.replace_workers(zwo.dummy_worker)
def run_parkinglot_expt(net, n): "Run experiment" seconds = args.time # Start the bandwidth and cwnd monitors in the background monitor = Process(target=monitor_devs_ng, args=('%s/bwm.txt' % args.dir, 1.0)) monitor.start() start_tcpprobe() # Get receiver and clients recvr = net.getNodeByName('receiver') #sender1 = net.getNodeByName('h1') # Start the receiver port = 5001 recvr.cmd('iperf -s -p', port, '> %s/iperf_server.txt' % args.dir, '&') #waitListening(sender1, recvr, port) # TODO: start the sender iperf processes and wait for the flows to finish # Hint: Use getNodeByName() to get a handle on each sender. # Hint: Use sendCmd() and waitOutput() to start iperf and wait for them to finish # Hint: waitOutput waits for the command to finish allowing you to wait on a particular process on the host # iperf command to start flow: 'iperf -c %s -p %s -t %d -i 1 -yc > %s/iperf_%s.txt' % (recvr.IP(), 5001, seconds, args.dir, node_name) # Hint (not important): You may use progress(t) to track your experiment progress ### begin my code # get list of all hosts h = [] # Python list of hosts for i in range(n): #print "DEBUG", 'h%s' % (i+1) h.append(net.getNodeByName('h%s' % (i + 1))) # wait for ports on all iperf clients for i in range(n): waitListening(h[i], recvr, port) # send iperf cmd to all hosts for i in range(n): node_name = 'h%s' % (i + 1) h[i].sendCmd('iperf -c %s -p %s -t %d -i 1 -yc > %s/iperf_%s.txt' % (recvr.IP(), port, seconds, args.dir, node_name)) # wait for commands to finish iperf_results = {} progress(seconds) # show progress while waiting for i in range(n): iperf_results[h[i].name] = h[i].waitOutput() ### end my code recvr.cmd('kill %iperf') # Shut down monitors monitor.terminate() stop_tcpprobe()
def start_httpd(handler_class=SimpleHTTPRequestHandler): clear_httpd_messages() httpd_process = Process(target=run_httpd_forever, args=(handler_class,)) httpd_process.start() httpd_url = HTTPD_MESSAGE_QUEUE.get() return httpd_process, httpd_url
def _launch_loggingserver(self): """ sets up logging server """ self.logging_status_queue = Queue() loggingserver = Process(target=MPLogger.loggingserver, args=(self.manager_params['log_file'], self.logging_status_queue, )) loggingserver.daemon = True loggingserver.start() return loggingserver
def call_job_fn(self, key, job_fn, args): # pylint: disable-next=import-outside-toplevel,no-name-in-module,import-error from multiprocess import Process # pylint: disable-next=not-callable proc = Process(target=job_fn, args=(key, self._make_progress_key(key), args)) proc.start() return proc.pid
def launch(self, listener_process_runner, *args): """Launch the aggregator listener process""" args = (self.manager_params, self.status_queue, self.shutdown_queue) + args self.listener_process = Process(target=listener_process_runner, args=args) self.listener_process.daemon = True self.listener_process.start() self.listener_address = self.status_queue.get()
def test_replace_decluster(delay=0.): worker = zwo.make_worker_function(1, decluster=True) #Process(target = worker).start() zwo.init_workers(worker) zwo.init_sink() Process(target=start_ventilator_bigarr, args=(delay, )).start() while True: time.sleep(10) zwo.replace_workers(worker)
def launch(self, listener_process_runner): """Launch the aggregator listener process""" self.status_queue = Queue() self.listener_process = Process(target=listener_process_runner, args=(self.manager_params, self.status_queue)) self.listener_process.daemon = True self.listener_process.start() self.listener_address = self.status_queue.get()
def submit(self, func, *args, **kwargs): # noinspection PyUnresolvedReferences from multiprocess import Process, Pipe from concurrent.futures import Future fut, (c0, c1) = Future(), Pipe(False) task = Process(target=self._target, args=(c1.send, func, args, kwargs)) self.tasks[fut] = task task.start() return self._set_future(fut, c0.recv())
def test_kill(): zwo.init_workers() zwo.init_sink() Process(target=start_ventilator).start() time.sleep(1) while True: zwo.kill_workers() time.sleep(0.5) zwo.init_workers() time.sleep(0.5)
def runpool(address, number_of_processes): # create a single server object -- children will each inherit a copy server = HTTPServer(address, RequestHandler) # create child processes to act as workers for i in range(number_of_processes-1): Process(target=serve_forever, args=(server,)).start() # main process also acts as a worker serve_forever(server)