def compute_feature_extraction(extractor, data): manager = mp.Manager() q = manager.Queue(maxsize=8) process_list = [] pre_compute = mp.Process(target=pre_compute_sillhouttes, args=(data, q, extractor)) pre_compute.start() num_processes = 4 for i in range(num_processes): time.sleep(random.random() * 0.1) p = mp.Process( target=listener, args=(extractor, q, i), ) process_list.append(p) p.start() pre_compute.join() for i in range(num_processes + 5): q.put((None, None)) for p in process_list: p.join() return True
def __init__(self, num_of_process: int, mapper: Callable, reducer: Callable, mapper_queue_size: int = 0, reducer_queue_size: int = 0): self._mapper_queue = mp.Queue(maxsize=mapper_queue_size) self._reducer_queue = ChunkedQueue(maxsize=reducer_queue_size) self._result_queue = ChunkedQueue() self._mapper_cmd_queue = [mp.Queue() for _ in range(num_of_process)] self._reducer_cmd_queue = [mp.Queue() for _ in range(num_of_process)] self._manager_cmd_queue = mp.Queue() self._manager_process = mp.Process(target=self._run_manager) self._mapper_process = [ mp.Process(target=self._run_mapper, args=(i, )) for i in range(num_of_process) ] self._reducer_process = [ mp.Process(target=self._run_reducer, args=(i, )) for i in range(num_of_process) ] self._mapper = mapper self._reducer = reducer self._num_of_process = num_of_process
def setup(self): """ Set up filesystem in user space for http and https so that we can retrieve tiles from remote sources. Parameters ---------- tmp_dir: string The temporary directory where to create the http and https directories """ from simple_httpfs import HttpFs if not op.exists(self.http_directory): os.makedirs(self.http_directory) if not op.exists(self.https_directory): os.makedirs(self.https_directory) if not op.exists(self.diskcache_directory): os.makedirs(self.diskcache_directory) self.teardown() disk_cache_size = 2**25 lru_capacity = 400 def start_fuse(directory, protocol): try: # this import can cause problems on systems that don't have libfuse # installed so let's only try it if absolutely necessary from fuse import FUSE # This is a bit confusing. I think `fuse` (lowercase) is used # above in get_filepath() line 50 and 52. If that's not the # case than this assignment is useless and get_filepath() is # broken fuse = FUSE( HttpFs( protocol, disk_cache_size=disk_cache_size, disk_cache_dir=self.diskcache_directory, lru_capacity=lru_capacity, ), directory, foreground=False, # allow_other=True ) except RuntimeError as e: if str(e) != "1": raise e proc1 = mp.Process(target=start_fuse, args=[self.http_directory, "http"]) proc1.start() proc1.join() proc2 = mp.Process(target=start_fuse, args=[self.https_directory, "https"]) proc2.start() proc2.join()
def multicore(): q = mp.Queue() p1 = mp.Process(target=job, args=(q, )) p2 = mp.Process(target=job, args=(q, )) p1.start() p2.start() p1.join() p2.join() res1 = q.get() res2 = q.get() print('multicore:', res1 + res2)
def main(): enable = mp.Value('i', 0) while True: #判断是否开始 sign = input("请输入控制命令:") print('sign = {}'.format(sign)) if sign == '1': #bytes([0x01]) print('connect') enable.value = 1 mp.Process(target=motionControl, args=(enable, )).start() mp.Process(target=displayImage, args=(enable, )).start() elif sign == '2': print('disconnect') enable.value = 0
def test_qoi_run_server(tmpdir, proxy, RE, hw): def delayed_sigint(delay): # pragma: no cover time.sleep(delay) print("killing") os.kill(os.getpid(), signal.SIGINT) def run_exp(delay): # pragma: no cover time.sleep(delay) print("running exp") p = Publisher(proxy[0], prefix=b"raw") RE.subscribe(p) det = SynSignal(func=lambda: np.ones(10), name="gr") RE(bp.count([det], md=dict(analysis_stage="raw"))) RE(bp.count([det], md=dict(analysis_stage="pdf"))) # Run experiment in another process (after delay) exp_proc = multiprocess.Process(target=run_exp, args=(2, ), daemon=True) exp_proc.start() # send the message that will eventually kick us out of the server loop threading.Thread(target=delayed_sigint, args=(10, )).start() L = [] try: print("running server") qoi_run_server(_publisher=lambda *x: L.append(x)) except KeyboardInterrupt: print("finished server") exp_proc.terminate() exp_proc.join() assert L
def start_record_server(): tcpserver = RecordServer() server_proc = mp.Process( target=tcpserver.serve_until_stopped) server_proc.start() return server_proc
def show(self): # credit for this code goes to the higlass-python team: https://github.com/higlass/higlass-python for p in list(self.processes.keys()): if self.port == p: print("delete ", self.processes[p]) self.processes[p].kill() del self.processes[p] time.sleep(0.5) #print(self.processes) uuid = slugid.nice() target = partial(eventlet.wsgi.server, sock=eventlet.listen(('localhost', self.port)), site=fApp) self.processes[self.port] = mp.Process( target=target) #self.startServer, args=(q,)) self.processes[self.port].start() self.connected = False while not self.connected: try: url = "http://{}:{}/".format('localhost', self.port) r = requests.head(url) if r.ok: self.connected = True except requests.ConnectionError: time.sleep(0.2)
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False): if nprocs == 1: return run_async(graph, sleep=sleep, raise_errors=raise_errors) nprocs = nprocs or mp.cpu_count() // 2 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) ioq = mp.Queue(len(graph.funcs.keys())) cpuq = mp.Queue(len(graph.funcs.keys())) procs = [mp.Process(target=run_scheduler, args=(graph, sleep, ioq, cpuq, raise_errors)) for _ in range(nprocs)] for proc in procs: proc.start() while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info( 'pid {}: queueing task {}'.format(os.getpid(), task)) if task in graph.io_bound: ioq.put(task) else: cpuq.put(task) time.sleep(sleep) if raise_errors and sum(not p.exitcode for p in procs): raise RuntimeError('An async task has failed. Please check your logs') return tgraph.recover_values_from_manager(graph)
def run_parallel_async(graph, nprocs=None, sleep=0.2): if nprocs == 1: return run_async(graph) nprocs = nprocs or mp.cpu_count() // 2 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) ioq = mp.Queue(len(graph.funcs.keys())) cpuq = mp.Queue(len(graph.funcs.keys())) for _ in range(nprocs): proc = mp.Process(target=run_scheduler, args=(graph, sleep, ioq, cpuq)) proc.start() while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info('pid {}: queueing task {}'.format( os.getpid(), task)) if task in graph.io_bound: ioq.put(task) else: cpuq.put(task) time.sleep(sleep) return tgraph.recover_values_from_manager(graph)
def shodan(self, scope_file, output): """ The Shodan module:\n Look-up information on the target IP address(es) using Shodan's API.\n A Shodan API key is required. """ asciis.print_art() print( green( "[+] Shodan Module Selected: O.D.I.N. will check Shodan for the provided domains \ and IPs.")) if __name__ == "__main__": report = reporter.Reporter(output) scope, ip_list, domains_list = report.prepare_scope(scope_file) # Create empty job queue jobs = [] shodan_report = multiprocess.Process(name="Shodan Report", target=report.create_shodan_table, args=(ip_list, domains_list)) jobs.append(shodan_report) for job in jobs: print(green("[+] Starting new process: {}".format(job.name))) job.start() for job in jobs: job.join() report.close_out_reporting() print(green("[+] Job's done! Your results are in {}.".format(output)))
def start_train(resume): urllib.request.urlretrieve(const.url + '/network_file', 'deliverables/network.py') urllib.request.urlretrieve(const.url + '/config_file', 'deliverables/input_params.py') urllib.request.urlretrieve(const.url + '/observation_file', 'deliverables/observation.py') urllib.request.urlretrieve(const.url + '/curriculum_file', 'deliverables/curriculum.py') num_workers = mp.cpu_count() - 1 should_stop = mp.Value(c_bool, False) while True: worker_processes = [] # create_worker(0, should_stop) # Start process 1 - n, running in other processes for w_num in range(0, num_workers): process = mp.Process(target=create_worker, args=(w_num, should_stop)) process.start() sleep(0.5) worker_processes.append(process) try: for p in worker_processes: p.join() except KeyboardInterrupt: should_stop.value = True print("Looks like we're done")
def main(): initialize() game = setup_game() with mp.Manager() as manager: exc = manager.Queue() arg_list = [] for i in range(0, 3): arg_list.append(((i, i + 3), game, False, exc)) arg_list.append(((5, 5), game, True, exc)) proc_list = [] for arg in arg_list: proc_list.append(mp.Process(target=function, args=arg)) proc_list[-1].start() print("Number of active children post start: %d" % len(mp.active_children())) for p in proc_list: p.join() if (not exc.empty()): e = exc.get() print(e.message) print("Number active children post join: %d " % len(mp.active_children())) print(mp.active_children()) print(mp.current_process())
def test_pipespeed(): c, d = processing.Pipe() cond = processing.Condition() elapsed = 0 iterations = 1 while elapsed < delta: iterations *= 2 p = processing.Process(target=pipe_func, args=(d, cond, iterations)) cond.acquire() p.start() cond.wait() cond.release() result = None t = _timer() while result != 'STOP': result = c.recv() elapsed = _timer() - t p.join() print(iterations, 'objects passed through connection in',elapsed,'seconds') print('average number/sec:', iterations/elapsed)
def start(self, log_file='/tmp/hgserver.log', log_level=logging.INFO): for puid in list(self.processes.keys()): print("terminating:", puid) self.processes[puid].terminate() del self.processes[puid] self.app = create_app( self.tilesets, __name__, log_file=log_file, log_level=log_level) # we're going to assign a uuid to each server process so that if anything # goes wrong, the variable referencing the process doesn't get lost uuid = slugid.nice().decode('utf8') if self.port is None: self.port = get_open_port() target = partial(self.app.run, threaded=True, debug=True, host='0.0.0.0', port=self.port, use_reloader=False) self.processes[uuid] = mp.Process(target=target) self.processes[uuid].start() self.connected = False while not self.connected: try: url = 'http://{}:{}/api/v1'.format(self.host, self.port) r = requests.head(url) if r.ok: self.connected = True except requests.ConnectionError as err: time.sleep(.2)
def run(params:List[Parameter],bt_log:bool, pe:bool) -> pandas.DataFrame: njobs = len(params) results = [] jobs = [] receivers = [] for i in range(njobs) : initializer = Initializer(params[i].population_size,params[i].landscape) init_pop = initializer.init_with_const(params[i].nucleotides, params[i].base_paires) receiver, sender = mp.Pipe(False) p = mp.Process(target=ea_without_crossover, args=(init_pop, params[i], bt_log, pe,sender,)) jobs.append(p) receivers.append(receiver) for i in range(njobs) : jobs[i].start() results.append(receivers[i].recv()) jobs[i].join() best_solutions = [] for rst in results : for ind in rst[1]: if ind.fitness == 1.0 : best_solutions.append([ind.rna_sequence, ind.fitness, ind.mfe,1/landscape.ens_defect(params.landscape.target_structure,ind.rna_sequence)]) df = pandas.DataFrame(best_solutions, columns=["Sequence", "Fitness","MFE","ED"]) return df
def test_tomo_run_server_3d_pencil(tmpdir, proxy, RE, hw): def delayed_sigint(delay): # pragma: no cover time.sleep(delay) print("killing") os.kill(os.getpid(), signal.SIGINT) def run_exp(delay): # pragma: no cover time.sleep(delay) print("running exp") p = Publisher(proxy[0], prefix=b"an") RE.subscribe(p) RE( bp.grid_scan( [hw.noisy_det], hw.motor3, 0, 2, 2, hw.motor1, 0, 2, 2, True, hw.motor2, 0, 2, 2, True, md={ "tomo": { "type": "pencil", "rotation": "motor1", "translation": "motor2", "stack": "motor3", "center": 1, } }, )) # Run experiment in another process (after delay) exp_proc = multiprocess.Process(target=run_exp, args=(2, ), daemon=True) exp_proc.start() # send the message that will eventually kick us out of the server loop threading.Thread(target=delayed_sigint, args=(10, )).start() L = [] try: print("running server") tomo_run_server(_publisher=lambda *x: L.append(x), algorithm="fbp") except KeyboardInterrupt: print("finished server") exp_proc.terminate() exp_proc.join() assert L
def main(): enable = mp.Value('i', 0) imgQueue = mp.Queue(0) imgQueueBin = mp.Queue(0) while True: sign = receive() if sign == 1: print('connect') enable.value = 1 mp.Process(target=motionControl, args=(enable, imgQueue, imgQueueBin)).start() mp.Process(target=displayImage, args=(enable, imgQueue, imgQueueBin)).start() elif sign == 2: print('disconnect') enable.value = 0
def people(self, client, domain): """ Uses TheHarvester and EmailHunter to locate email addresses and social media profiles. Profiles are cross-referenced with HaveIBeenPwned, Twitter's API, and search engines to try to find security breaches, pastes, and social media accounts.\n Several API keys are required for all of the look-ups: EmailHunter and Twitter. """ asciis.print_art() print( green( "[+] People Module Selected: O.D.I.N. will run only modules for email addresses \ and social media.")) # Perform prep work for reporting setup_reports(client) output_report = "reports/{}/OSINT_DB.db".format(client) if __name__ == "__main__": report = reporter.Reporter(output_report) # Create empty job queue jobs = [] company_info = multiprocess.Process( name="Company Info Report", target=report.create_company_info_table, args=(domain, )) jobs.append(company_info) employee_report = multiprocess.Process( name="Employee Report", target=report.create_people_table, args=(domain, client)) jobs.append(employee_report) for job in jobs: print(green("[+] Starting new process: {}".format(job.name))) job.start() for job in jobs: job.join() report.close_out_reporting() print( green("[+] Job's done! Your results are in {}.".format( output_report)))
def run(self): with mb.Manager() as m: results = m.dict() p = mb.Process(target=QLWinSingleTest._run_test, args=(self, results)) p.start() p.join() if "exception" not in results: return results['result'] else: raise RuntimeError(f"\n\nGot an exception during subprocess:\n\n{results['exception']}")
def start_servers(target1: Callable, target2: Callable, target3: Callable, metrics_target: Callable) -> None: """ Start servers Parameters ---------- target1 Main flask process target2 Auxiliary flask process """ if USE_MULTIPROCESS: logger.info("Using alternative multiprocessing library") else: logger.info("Using standard multiprocessing library") p2 = None if target2: p2 = mp.Process(target=target2, daemon=False) p2.start() p3 = None if target3: p3 = mp.Process(target=target3, daemon=True) p3.start() p4 = None if metrics_target: p4 = mp.Process(target=metrics_target, daemon=True) p4.start() target1() if p2: p2.join() if p3: p3.join() if p4: p4.join()
def setup(self): """ Set up filesystem in user space for http and https so that we can retrieve tiles from remote sources. Parameters ---------- tmp_dir: string The temporary directory where to create the http and https directories """ from simple_httpfs import HttpFs if not op.exists(self.http_directory): os.makedirs(self.http_directory) if not op.exists(self.https_directory): os.makedirs(self.https_directory) if not op.exists(self.diskcache_directory): os.makedirs(self.diskcache_directory) try: sh.umount(self.http_directory) except Exception as ex: pass try: sh.umount(self.https_directory) except Exception as ex: pass disk_cache_size = 2**25 disk_cache_dir = self.diskcache_directory lru_capacity = 400 print( "self.diskcache_directory", self.diskcache_directory, op.exists(self.diskcache_directory), ) def start_fuse(directory): print("starting fuse") fuse = FUSE( HttpFs( "http", disk_cache_size=disk_cache_size, disk_cache_dir=self.diskcache_directory, lru_capacity=lru_capacity, ), directory, foreground=False, ) proc = mp.Process(target=start_fuse, args=[self.http_directory]) proc.start() proc.join()
def run(self): with mb.Manager() as m: results = m.dict() p = mb.Process(target=QLWinSingleTest._run_test, args=(self, results)) p.start() p.join() if "exception" not in results: return results['result'] else: raise results['exception']
def start_break_timer(self, length): # Basically the same implementation as the timer for the pomodoro, but this one cannot be paused print('\nBreak started\n') current_length = multiprocess.Queue() break_process = multiprocess.Process(target=countdown, args=(length, current_length, self.sound_file)) break_process.start() break_process.join() break_process.terminate() input('Press ENTER to start another pomodoro\r')
def generate(self: object) -> None: print("Beginning data generation...") start_seconds = time.time() queue = multiprocess.Queue() w = multiprocess.Process(target=self.write, args=(queue, "STOP")) jobs = [] for i in range(0, 5): p = multiprocess.Process(target=self.gen, args=(queue, )) jobs.append(p) p.start() w.start() for i, item in enumerate(jobs): item.join() queue.put("STOP") w.join() elapsed_time = (time.time() - start_seconds) / 60 print("Generation completed. Elapsed time: ", "{0:.2f}".format(elapsed_time), " minutes")
def startProcess(source, threads, midFolder): #PARAMETERS vilDirec = returnDirectoryCSV(source) totalLines = len(vilDirec) numLines = int(totalLines / threads) + 1 processes = [] arguments = [] finished = [] allDone = False count = 0 for s in range(0, totalLines, numLines): if (s + numLines > totalLines): end = totalLines else: end = s + numLines arg = [midFolder, vilDirec[s:end], count] process = mp.Process(target=scrape, args=arg) processes.append(process) arguments.append(arg) finished.append(False) count += 1 for p in processes: p.start() while not allDone: for i in range(len(processes)): curProcess = processes[i] if not curProcess.is_alive() and finished[i] == False: processes[i] = mp.Process(target=scrape, args=arguments[i]) processes[i].start() sleep(15) if not processes[i].is_alive(): finished[i] = True allDone = True for b in finished: if not b: allDone = False break
def test_join_timeout(): p = processing.Process(target=join_timeout_func) p.start() print('waiting for process to finish') while 1: p.join(timeout=1) if not p.is_alive(): break print('.', end=' ') sys.stdout.flush()
def server(): started_event = multiprocessing.Event() exit_event = multiprocessing.Event() process = multiprocessing.Process( target=lambda: MPServer(started_event, exit_event).run(rate=1000)) process.start() started_event.wait() # Starting processes is really slow on Windows time.sleep(0.1) # Give the server a head start at stdout yield exit_event.set() process.join(timeout=1) process.terminate()
def server_with_on_sub_handler(sub_mgr, schema, on_sub_handler): app = create_app(sub_mgr, schema, on_sub_handler) process = multiprocess.Process(target=app_worker, kwargs={ 'app': app, 'port': TEST_PORT }) process.start() yield process.terminate()
def server_with_keep_alive(sub_mgr, schema): app = create_app(sub_mgr, schema, {'keep_alive': .250}) process = multiprocess.Process(target=app_worker, kwargs={ 'app': app, 'port': TEST_PORT }) process.start() yield process.terminate()