def fork(fn, *args, daemon=False): ''' Creates a child process that runs the given function. @sig fork :: (* -> _) -> * -> Bool -> _ ''' with gipc.pipe(): gipc.start_process(target=fn, args=args, daemon=daemon)
def test_watch(self): """ INTEGRATION: Receive a watch event from other process """ self.client.set('/test-key', 'test-value') queue = multiprocessing.Queue() def change_value(key, newValue): c = etcd_gevent.Client(port=6001) c.set(key, newValue) def watch_value(key, queue): c = etcd_gevent.Client(port=6001) queue.put(c.watch(key).value) watcher = gipc.start_process( target=watch_value, args=('/test-key', queue)) watcher.start() time.sleep(1) changer = gipc.start_process( target=change_value, args=('/test-key', 'new-test-value',)) changer.start() try: value = queue.get(timeout=2) finally: watcher.join(timeout=5) changer.join(timeout=5) assert value == 'new-test-value'
def test_exitcode_previous_to_join(self): p = None if multiprocessing.get_start_method() == 'spawn': # Spawn cannot pass lambdas to subprocesses # as the process space is not copied p = start_process(gevent.sleep, (SHORTTIME, )) else: p = start_process(lambda: gevent.sleep(SHORTTIME)) # Assume that the child process is still alive when the next # line is executed by the interpreter (there is no guarantee # for that, but it's rather likely). assert p.exitcode is None # Expect the child watcher mechanism to pick up # and process the child process termination event # (within at most two seconds). The `gevent.sleep()` # invocations allow for libev event loop iterations, # two of which are required after the OS delivers the # SIGCHLD signal to the parent process: one iteration # invokes the child reap loop, and the next invokes # the libev callback associated with the termination # event. deadline = time.time() + 2 while time.time() < deadline: if p.exitcode is not None: assert p.exitcode == 0 p.join() return gevent.sleep(ALMOSTZERO) raise Exception('Child termination not detected')
def test_childchildcomm(self): m = {("KLADUSCH", ): "foo"} pr = start_process(ipc_readchild, args=(self.rh, m)) pw = start_process(ipc_writechild, args=(self.wh, m)) pr.join() pw.join() assert pr.exitcode == 0 assert pw.exitcode == 0 self.rh2.close() self.wh2.close()
def main(): with gipc.pipe() as (r, w): p = gipc.start_process(target=child_process, args=(r, )) p = gipc.start_process(target=random_process) wg = gevent.spawn(writegreenlet, w) try: p.join() except KeyboardInterrupt: wg.kill(block=True) p.terminate() p.join()
def initializeManyRouters(self, config, number, background): '''Initialize many routers and background if required Args: config (Wishbone.config.configfile:ConfigFile): The router configration number (int): The number of instances to intialize background (bool): Whether to background the routers or not ''' if background: pid_file = PIDFile(self.pid) with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, detach_process=True): if self.instances == 1: sys.stdout.write("\nWishbone instance started with pid %s\n" % (os.getpid())) sys.stdout.flush() pid_file.create([os.getpid()]) self.initializeOneRouter(config) else: for instance in range(self.instances): self.routers.append( gipc.start_process( self.initializeOneRouter, args=(config, ), daemon=True ) ) pids = [str(p.pid) for p in self.routers] print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids)))) pid_file.create(pids) self.bootstrapBlock() else: if self.instances == 1: sys.stdout.write("\nInstance started in foreground with pid %s\n" % (os.getpid())) self.initializeOneRouter(config) else: for instance in range(self.instances): self.routers.append( gipc.start_process( self.initializeOneRouter, args=(config, ), daemon=True ) ) pids = [str(p.pid) for p in self.routers] print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids)))) self.bootstrapBlock()
def open_session(session_id): client_id = str(uuid.uuid1()) if not session_id in INTERPRETER: cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe() output_queue_from_interpreter, output_queue = gipc.pipe() RESULT[session_id] = dict() config = static_config.get_config() session = config.get(session_id) session_cfg = config.get_config(session.name) synoptic = session_cfg.get('synoptic') if synoptic: SYNOPTIC[session_id] = synoptic INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter, args=(session_id, cmds_queue, output_queue), kwargs={"beacon_host": os.environ.get("BEACON_HOST"), "beacon_port": os.environ.get("BEACON_PORT") }) EXECUTION_QUEUE[session_id].put((None, "syn", (None,))) output_queue_from_interpreter.get() #ack OUTPUT_QUEUE[session_id] = dict() gevent.spawn(handle_output, session_id, output_queue_from_interpreter) RESULT[session_id][client_id] = gevent.event.AsyncResult() OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue() root_path = os.path.dirname(os.path.abspath(__file__)) contents = file(os.path.join(root_path, "shell.html"), "r") template = Template(contents.read()) return template.render(client_uuid=repr(client_id))
def processLauncher(self): with gipc.pipe() as (r, w): self.comms.append((r, w)) p = gipc.start_process(target=self.sessionLauncher, args=((r, self.greenthreads))) p.join()
def start(self, auth_msg=''): """ Begins processing commands from the server. """ self.pipe, child_end = gipc.pipe() self._socket_connect() self.reader = gipc.start_process( target=ServerReader, args=(child_end, os.getpid(), self.socket.fileno(), self.tracer.pause_signal), ) with Timeout(5, QdbFailedToConnect(self.tracer.address, self.tracer.retry_attepts), green=self.green): # Receive a message to know that the reader is ready to begin. self.pipe.get() self.send( fmt_msg( 'start', { 'uuid': self.tracer.uuid, 'auth': auth_msg, 'local': (0, 0), }, serial=pickle.dumps, ) ) atexit.register(self.stop)
def __init__(self, worker_func, task_count, worker_count=-1): # initialize completion task workers # if number of workers is not specified, set it to the number of CPUs if worker_count == -1: worker_count = 8 self.workers = CallbackTaskWorkers(worker_func, worker_count) # start result manager if PROC_TYPE == ProcType.gipc: self.result_manager = gipc.start_process(target=result_manager, args=(task_count,)) elif PROC_TYPE == ProcType.multiprocessing: self.result_manager = Process(target=result_manager, args=(task_count,)) self.result_manager.start() elif PROC_TYPE == ProcType.greenlet: self.result_manager = gevent.spawn(result_manager, task_count) # initialize completion task runner self.runner = CallbackTaskRunner() # wait for all workers to subscribe to control channel # MUST be placed above all other commands in __init__ to work log.debug('Waiting for %d workers to initialize...', worker_count) context = zmq.Context() syncservice = context.socket(zmq.REP) syncservice.bind(ZMQ_WORKER_SYNC_CH) subscribers = 0 while subscribers < worker_count: msg = syncservice.recv() syncservice.send(msg) # send synchronization reply subscribers = subscribers + 1 log.debug('%d workers initialized', worker_count)
def test_exitcode_sigkill(self): p = start_process(p_child_b) p.join() if not WINDOWS: assert p.exitcode == -signal.SIGKILL else: assert p.exitcode == 1
def _process(init, initargs, maxtasksperchild, func, itemq, resultq): if maxtasksperchild is None: task_limit = iter(int, 1) # no limit else: task_limit = range(maxtasksperchild) proc = None with gipc.pipe() as (i_send, items), gipc.pipe() as (results, r_send): try: with gipc_process_error_handler(): args = (init, initargs, func, i_send, r_send) proc = gipc.start_process(target=_worker, args=args) log.debug("start worker: %s", proc.pid) for x in task_limit: items.put(itemq.get()) result = results.get() resultq.put(result) if result is _Stop: log.debug("worker stopped: %s", proc.pid) return _Stop items.put(_Stop) except ProcessError: log.error("process error %s", proc.pid) finally: if proc is not None: proc.join() if hasattr(proc, "close"): # added in Python 3.7 # HACK gipc bug https://github.com/jgehrcke/gipc/issues/90 proc._popen.poll = lambda: proc.exitcode proc.close()
def test_wsgi_scenario(self): from gevent.wsgi import WSGIServer def serve(http_server): http_server.serve_forever() def hello_world(environ, start_response): # Generate response in child process. with pipe() as (reader, writer): start_response('200 OK', [('Content-Type', 'text/html')]) rg = start_process( target=complchild_test_wsgi_scenario_respgen, args=(writer, )) response = reader.get() rg.join() assert rg.exitcode == 0 return [response] http_server = WSGIServer(('localhost', 0), hello_world) servelet = gevent.spawn(serve, http_server) # Wait for server being bound to socket. while True: if http_server.address[1] != 0: break gevent.sleep(0.05) client = start_process(target=complchild_test_wsgi_scenario_client, args=(http_server.address, )) client.join() assert client.exitcode == 0 servelet.kill() servelet.get() # get() is join and re-raises Exception.
def handle_http_request(_, start_response): """I am executed in a greenlet whenever an HTTP request came in.""" # Start constructing an HTTP response. Build the header first. start_response('200 OK', [('Content-Type', 'text/html')]) # What would we like to respond to the client? Let's ask a worker # process to generate the HTTP response body for us. (When would we do # this in the real world? For example when response generation requires # significant CPU-bound work as in image processing or when compiling a # complex PDF document, ...). In cases like this the CPU-bound work can # severely impact the responsiveness of the HTTP server process.) with gipc.pipe() as (r, w): # Start the child process (each client connection makes the server # process spawn a child process!). Pass the write end of the pipe to # the child process. p = gipc.start_process(target=child_msg_generator, args=(w, )) # Read the message from the child process. body = r.get() # Reap child (call wait(), remove it from process table). p.join() assert p.exitcode == 0 # Write HTTP response body. return [body]
def test_join_timeout(self): p = start_process(gevent.sleep, args=(0.1, )) p.join(ALMOSTZERO) assert p.is_alive() p.join() assert p.exitcode == 0 _call_close_method_if_exists(p)
def spawn_processes(confs): """Spawn subprocesses with a config similar to spawn_greenlets.""" processes = [] for conf in confs: p = gipc.start_process(target=conf[0], args=tuple(conf[1:])) processes.append(p) return processes
def open_session(session_id): client_id = str(uuid.uuid1()) if not session_id in INTERPRETER: cmds_queue, EXECUTION_QUEUE[session_id] = gipc.pipe() output_queue_from_interpreter, output_queue = gipc.pipe() RESULT[session_id] = dict() config = static_config.get_config() session = config.get(session_id) session_cfg = config.get_config(session.name) synoptic = session_cfg.get('synoptic') if synoptic: SYNOPTIC[session_id] = synoptic INTERPRETER[session_id] = gipc.start_process( interpreter.start_interpreter, args=(session_id, cmds_queue, output_queue), kwargs={ "beacon_host": os.environ.get("BEACON_HOST"), "beacon_port": os.environ.get("BEACON_PORT") }) EXECUTION_QUEUE[session_id].put((None, "syn", (None, ))) output_queue_from_interpreter.get() #ack OUTPUT_QUEUE[session_id] = dict() gevent.spawn(handle_output, session_id, output_queue_from_interpreter) RESULT[session_id][client_id] = gevent.event.AsyncResult() OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue() root_path = os.path.dirname(os.path.abspath(__file__)) contents = file(os.path.join(root_path, "shell.html"), "r") template = Template(contents.read()) return template.render(client_uuid=repr(client_id))
def test_orphaned_signal_watcher(self): # Install libev-based signal watcher. s = gevent.signal(signal.SIGTERM, signals_test_sigterm_handler) # Normal behavior: signal handlers become inherited by children. # Bogus behavior of libev-based signal watchers in child process: # They should not be active anymore when 'orphaned' (when their # corresponding event loop has been destroyed). What happens, however: # The old handler stays active and registering a new handler does not # 'overwrite' the old one -- both are active. # Since this test is about testing the behavior of 'orphaned' libev # signal watchers, the signal must be transmitted *after* event loop # recreation, so wait here for the child process to go through # the hub & event loop destruction (and recreation) process before # sending the signal. Waiting is realized with sync through pipe. # Without cleanup code in gipc, the inherited but orphaned libev signal # watcher would be active in the fresh event loop and trigger the # handler. This is a problem. With cleanup code, this handler must # never be called. Child exitcode 20 means that the inherited handler # has been called, -15 (-signal.SIGTERM) means that the child was # actually killed by SIGTERM within a certain short time interval. # Returncode 0 would mean that the child finished normally after that # short time interval. with pipe() as (r, w): p = start_process(signals_test_child_a, (w, )) assert r.get() == p.pid os.kill(p.pid, signal.SIGTERM) p.join() if not WINDOWS: assert p.exitcode == -signal.SIGTERM else: assert p.exitcode == signal.SIGTERM s.cancel()
def ipc_child_f(w, m): assert len(get_all_handles()) == 1 i = start_process(ipc_child_f2, args=(w, m)) i.join() assert i.exitcode == 0 with raises(GIPCClosed): w.close()
def test_threadpool_resolver_mp(self): h = gevent.get_hub() t = h.threadpool r = h.resolver p = start_process(target=complchild_test_threadpool_resolver_mp) p.join(timeout=1) assert p.exitcode == 0
def start(self): '''Maps to the CLI command and starts one or more Wishbone processes in background. ''' router_config = ConfigFile(self.config, 'SYSLOG').dump() pid_file = PIDFile(self.pid) with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, detach_process=True): if self.instances == 1: sys.stdout.write("\nWishbone instance started with pid %s\n" % (os.getpid())) sys.stdout.flush() pid_file.create([os.getpid()]) self.initializeRouter(router_config) else: for instance in range(self.instances): self.routers.append( gipc.start_process(self.initializeRouter, args=(router_config, ), daemon=True)) pids = [str(p.pid) for p in self.routers] print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids)))) pid_file.create(pids) self.bootstrapBlock()
def real_start(self): pr, pw = gipc.pipe() self.puppet_process = gipc.start_process(target=SockServer(Puppet).run, kwargs={"pipe": pw}) port = pr.get() self.puppet = SockClient(("localhost", port), keep_alive=False) self.puppet.hire_worker()
def open_session(session_id): client_id = str(uuid.uuid1()) if not session_id in INTERPRETER: read_config(SHELL_CONFIG_FILE) cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe() output_queue_from_interpreter, output_queue = gipc.pipe() RESULT[session_id] = dict() setup_file = SETUP.get(session_id, {}).get("file") config_objects_names = SETUP.get(session_id, {}).get("config_objects") INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter, args=(setup_file, config_objects_names, cmds_queue, output_queue)) EXECUTION_QUEUE[session_id].put((None, "syn", (None,))) output_queue_from_interpreter.get() #ack OUTPUT_QUEUE[session_id] = dict() gevent.spawn(handle_output, session_id, output_queue_from_interpreter) RESULT[session_id][client_id] = gevent.event.AsyncResult() OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue() root_path = os.path.dirname(os.path.abspath(__file__)) contents = file(os.path.join(root_path, "shell.html"), "r") template = Template(contents.read()) return template.render(client_uuid=repr(client_id))
def start(self): '''Maps to the CLI command and starts one or more Wishbone processes in background. ''' router_config = ConfigFile(self.config, 'SYSLOG').dump() pid_file = PIDFile(self.pid) with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, detach_process=True): if self.instances == 1: sys.stdout.write("\nWishbone instance started with pid %s\n" % (os.getpid())) sys.stdout.flush() pid_file.create([os.getpid()]) self.initializeRouter(router_config) else: for instance in range(self.instances): self.routers.append( gipc.start_process( self.initializeRouter, args=(router_config, ), daemon=True ) ) pids = [str(p.pid) for p in self.routers] print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids)))) pid_file.create(pids) self.bootstrapBlock()
def __init__(self, app): super(PoWService, self).__init__(app) cpu_pct = self.app.config['pow']['cpu_pct'] self.cpipe, self.ppipe = gipc.pipe(duplex=True) self.worker_process = gipc.start_process( target=powworker_process, args=(self.cpipe, cpu_pct)) self.app.services.chain.on_new_head_candidate_cbs.append(self.on_new_head_candidate) self.hashrate = 0
def duplex(): with pipe() as (r, w): with pipe() as (r2, w2): p = start_process(complchild_test_multi_duplex, (r, w2)) w.put("msg") assert r2.get() == "msg" p.join() assert p.exitcode == 0
def reloader(i): global running, workers while running: print 'Starting worker {}.'.format(i) workers[i] = gipc.start_process(server, (i,), daemon=True, name='worker{}'.format(i)) workers[i].join() print 'Worker {} has just quit!'.format(i) print 'Done with worker {}'.format(i)
def test_threadpool_resolver_mp(self): h = gevent.get_hub() t = h.threadpool r = h.resolver p = start_process(target=complchild_test_threadpool_resolver_mp) # Note(JP): seen this fail once on Windows CI with a timeout of 1 s. p.join(timeout=2) assert p.exitcode == 0
def test_terminate(self): p = start_process(gevent.sleep, args=(1, )) # Test __repr__ and __str__ p.__repr__() p.terminate() p.join() p.__repr__() assert p.exitcode == -signal.SIGTERM
def test_handlecount(self): p = start_process(ipc_handlecounter1, args=(self.rh, self.rh2)) p.join() assert p.exitcode == 0 # After passing two handles to the child, only 2 must be left open. assert len(get_all_handles()) == 2 self.wh.close() self.wh2.close()
def setup_children_process(): global gConfig if 'children_wsgi' in gConfig['wsgi']: if len(list(gConfig['wsgi']['children_wsgi'].keys())): for k in gConfig['wsgi']['children_wsgi'].keys(): if 'config' in gConfig['wsgi']['children_wsgi'][k]: config = gConfig['wsgi']['children_wsgi'][k]['config'] p = gipc.start_process(target=run_children_process, args=(config, ), daemon=True)
def test_whatever_2(self): """ Time-synchronize two child processes via two unidirectional channels. Uses timeout control in children. """ # First pipe for sync. with pipe() as (syncreader, syncwriter): # Second pipe for communication. with pipe() as (r, w): # Send messages pw = start_process(usecase_child_b, args=(w, syncreader)) # Receive messages pr = start_process(usecase_child_c, args=(r, syncwriter)) pw.join() pr.join() assert pw.exitcode == 0 assert pr.exitcode == 5
def test_close_raises_if_called_prematurely(self): p = start_process(p_child_a) assert p.is_alive() with raises(ValueError, match="while it is still running"): p.close() p.terminate() p.join() p.close()
def initialize(PK, size=1): global _procs, myPK myPK = PK _procs = [] for s in range(size): (r,w) = gipc.pipe(duplex=True) p = gipc.start_process(_worker, args=(PK, r,)) _procs.append((p,w))
def test_exitcode_sigkill(self): p = start_process(p_child_b) p.join() if not WINDOWS: assert p.exitcode == -signal.SIGKILL else: assert p.exitcode == 1 _call_close_method_if_exists(p)
def bulk_operation(self, serializer, index, client, **options): procs = options.get('multi') if not procs: procs = os.cpu_count() procs = procs if procs > 1 else 2 data_size = serializer.fetch_data_length() chunk_size = int(data_size / (procs * 4)) def parallel_params(): for chunk_num in range(procs * 4 + 1): _options = options.copy() _options['parallel_chunk_num'] = chunk_num _options['parallel_chunk_size'] = chunk_size yield hash(serializer), index, _options if not self.pipe_fd_mapping: # reset multiprocessing.Process-sensitive elements just before forking self.parallel_prep() for n in range(procs): h1, h2 = gipc.pipe(duplex=True) self.pipe_fd_mapping[h1._reader._fd] = ( h1, gipc.start_process(process_func, args=(h2, ), name="esdocs-proc-{}".format(n))) # wait until sub-process workers check in ready = 0 while ready != procs: readable, _, _ = select(self.pipe_fd_mapping.keys(), [], []) for fd in readable: h1, proc = self.pipe_fd_mapping[fd] msg, data = h1.get() if msg == 'GOOD-MORNING-DAVE': ready += 1 parallel_chunk_params = parallel_params() # populate all the processes with an even number of chunks to process chunks_remaining = True chunks_to_process = 0 while chunks_remaining: for h1, proc in self.pipe_fd_mapping.values(): try: h1.put(next(parallel_chunk_params)) chunks_to_process += 1 except StopIteration: chunks_remaining = False break while chunks_to_process > 0: readable, _, _ = select(self.pipe_fd_mapping.keys(), [], []) for fd in readable: h1, proc = self.pipe_fd_mapping[fd] msg, data = h1.get() if msg == 'DONE-CHUNK': chunks_to_process -= 1
def _run_job_python(job_signaler, job, results): res = _Result(job, None) p = gipc.start_process(target=job.cmd) job_signaler.add(res, p.terminate) p.join() job_signaler.rm(res) if p.exitcode != 0 and res.e is None: res.e = ExecError(p.exitcode) results.send(res)
def main(): with gipc.pipe() as (readend, writeend): # Start "writer" Greenlet. Provide it with the pipe with write end g = gevent.spawn(writelet, writeend) # Start "reader" child process. Provide it with the pipe read end p = gipc.start_process(target=readchild, args=(readend,)) g.join() p.join()
def _create_model_from_factory(manager, email, path, create_model, params, start_stop_rows, table): schema = PARAM_SCHEMAS_SERVE[path] model_params = _parse_params(params, schema) inputs = {x: _get_input(params, x) for x in schema['input_types']} with gipc.pipe() as (reader, writer): p = gipc.start_process(target=create_model, args=(writer, model_params, inputs, schema, start_stop_rows, table, email)) row = reader.get() p.join() return {'row': base64.b64encode(row)}
def test_early_readchild_exit_write_from_child(self): pr = start_process(ipc_readonce_then_exit, (self.rh, )) pw = start_process(ipc_endless_write_for_early_reader_exit, (self.wh, )) # This test is to make sure equivalent behavior as in test # `test_early_readchild_exit` when the writing process is a # child process itself (above, the write process in the initial # process). Since gipc's child process creation # routine messes around with signal handlers, this test makes # sure that SIGPIPE is ignored in the child and that a # failing write attempt (after early read child exit) results # in an exception raised in the writing process. pr.join() pw.join() assert pr.exitcode == 0 assert pw.exitcode == 0 self.rh2.close() self.wh2.close()
def init(self): for _ in xrange(self.process_size): child_pipe_end, parent_pipe_end = gipc.pipe(duplex=True) self.child_pipe_ends.append(child_pipe_end) self.parent_pipe_ends.append(parent_pipe_end) p = gipc.start_process(target=self.process_target, args=(child_pipe_end, ), daemon=True) self.processes.append(p) #p.join() self.loop_get_result()
def __init__(self, worker_func, worker_count): self.workers = [None]*worker_count for wrk_num in range(worker_count): if PROC_TYPE == ProcType.gipc: self.workers[wrk_num] = gipc.start_process(target=self._callback_worker, args=(wrk_num, worker_func)) elif PROC_TYPE == ProcType.multiprocessing: self.workers[wrk_num] = Process(target=self._callback_worker, args=(wrk_num, worker_func)) self.workers[wrk_num].start() elif PROC_TYPE == ProcType.greenlet: self.workers[wrk_num] = gevent.spawn(self._callback_worker, wrk_num, worker_func)
def __init__(self, num_procs=multiprocessing.cpu_count(), num_greenlets=50): super(GIPCExecutor, self).__init__() self.procs = [] self.pipes = [] self.results = {} for _ in xrange(num_procs): cend, pend = gipc.pipe(duplex=True) proc = gipc.start_process(self._proc_loop, args=(cend, num_greenlets), daemon=True) self.procs.append(proc) self.pipes.append(pend) self.pipes = itertools.cycle(self.pipes) self.result_pool = self._result_pool()
def start(self): """ Starts the task's process """ self.pipe, pipe_child = gipc.pipe(duplex=True) self.running = True self.process = gipc.start_process( target=self._worker, kwargs={ 'pipe': pipe_child, } ) self.reader = gevent.spawn(self._reader)
def fork_service(klass, concurrency=10, queue_size=None): """simply create service process, returns tuple like (service_process, client) you can communicate through client's call/call_async/call_callback methods. when you use fork_service(), please make sure to call client.close() and service_process.join()""" global _internal_pid _internal_pid += 1 child_ch, parent_ch = gipc.pipe(duplex=True) service = klass() args = (child_ch, _internal_pid, concurrency, queue_size) service_process = gipc.start_process(target=service, args=args) client = IPCRPCClient(parent_ch) return (service_process, client)
def start(self): pipe_parent, pipe_child = gipc.pipe(duplex=True) self.stream = GateStreamServerEndpoint(pipe_parent) stream_child = GateStreamWorkerEndpoint(pipe_child) self.process = gipc.start_process( target=self._target, kwargs={ 'stream': stream_child, '_pipe': pipe_child, } ) logging.debug('Started child process %s', self.process.pid) self.stream_reader = gevent.spawn(self._stream_reader)
def main(): global running, channel_mngr, workers # Make green psycopg: extensions.set_wait_callback(gevent_wait_callback) # Decide how many processes to use try: proc_count = int(sys.argv[1]) except: proc_count = multiprocessing.cpu_count() workers = [None] * proc_count channel.init(proc_count) gevent.signal(signal.SIGINT, sig_quit) gevent.signal(signal.SIGTERM, sig_quit) # Remove the sockets directory shutil.rmtree(sock_dir, True) os.mkdir(sock_dir) # Spawn the reloaders for the workers reloaders = [gevent.spawn(reloader, i) for i in xrange(proc_count)] setproctitle.setproctitle('diggems master') # Channel manager process reloader: while running: print 'Starting channel manager process.' proc = gipc.start_process(channel.rpc_dispatcher, daemon=True, name='channel_mngr') proc.join() print 'Channel manager process has quit.' print 'Done with channel manager' gevent.joinall(reloaders) print 'All done, quiting'
def debug(self): '''Maps to the CLI command and starts Wishbone in foreground. ''' router_config = ConfigFile(self.config, 'STDOUT').dump() if self.instances == 1: sys.stdout.write("\nInstance started in foreground with pid %s\n" % (os.getpid())) self.initializeRouter(router_config) else: for instance in range(self.instances): self.routers.append( gipc.start_process( self.initializeRouter, args=(router_config, ), daemon=True ) ) pids = [str(p.pid) for p in self.routers] print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids)))) self.bootstrapBlock()
def start(self): start_process(self.run)
f.write(f.readline()) f.flush() f.close() def server(): StreamServer(("localhost", PORT), serve).serve_forever() def clientprocess(): t1 = time.time() clients = [gevent.spawn(client) for _ in xrange(N_CLIENTS)] gevent.joinall(clients) duration = time.time() - t1 print "{} clients served within {:.2f} s".format(N_CLIENTS, duration) def client(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(("localhost", PORT)) f = sock.makefile() f.write(MSG) f.flush() assert f.readline() == MSG f.close() if __name__ == "__main__": s = gevent.spawn(server) c = gipc.start_process(clientprocess) c.join() s.kill() s.join()
def run(self): e = gipc.start_process(target=Executor, args=("Executor",)) # c = gipc.start_process(target=Collector, args=('Collector',)) e.join()
def hire_worker(self): tq_worker, self.task_queue = gipc.pipe(duplex=True) self.worker = gipc.start_process(target=process_worker, args=(tq_worker,)) self.operator = gevent.spawn(self.receive_info)
def run(self): e = gipc.start_process(target = Executor,args=('Executor',)) e.join()