def fork(fn, *args, daemon=False):
'''
Creates a child process that runs the given function.
@sig fork :: (* -> _) -> * -> Bool -> _
'''
with gipc.pipe():
gipc.start_process(target=fn, args=args, daemon=daemon)
def test_watch(self):
""" INTEGRATION: Receive a watch event from other process """
self.client.set('/test-key', 'test-value')
queue = multiprocessing.Queue()
def change_value(key, newValue):
c = etcd_gevent.Client(port=6001)
c.set(key, newValue)
def watch_value(key, queue):
c = etcd_gevent.Client(port=6001)
queue.put(c.watch(key).value)
watcher = gipc.start_process(
target=watch_value, args=('/test-key', queue))
watcher.start()
time.sleep(1)
changer = gipc.start_process(
target=change_value, args=('/test-key', 'new-test-value',))
changer.start()
try:
value = queue.get(timeout=2)
finally:
watcher.join(timeout=5)
changer.join(timeout=5)
assert value == 'new-test-value'
def test_exitcode_previous_to_join(self):
p = None
if multiprocessing.get_start_method() == 'spawn':
# Spawn cannot pass lambdas to subprocesses
# as the process space is not copied
p = start_process(gevent.sleep, (SHORTTIME, ))
else:
p = start_process(lambda: gevent.sleep(SHORTTIME))
# Assume that the child process is still alive when the next
# line is executed by the interpreter (there is no guarantee
# for that, but it's rather likely).
assert p.exitcode is None
# Expect the child watcher mechanism to pick up
# and process the child process termination event
# (within at most two seconds). The `gevent.sleep()`
# invocations allow for libev event loop iterations,
# two of which are required after the OS delivers the
# SIGCHLD signal to the parent process: one iteration
# invokes the child reap loop, and the next invokes
# the libev callback associated with the termination
# event.
deadline = time.time() + 2
while time.time() < deadline:
if p.exitcode is not None:
assert p.exitcode == 0
p.join()
return
gevent.sleep(ALMOSTZERO)
raise Exception('Child termination not detected')
def test_childchildcomm(self):
m = {("KLADUSCH", ): "foo"}
pr = start_process(ipc_readchild, args=(self.rh, m))
pw = start_process(ipc_writechild, args=(self.wh, m))
pr.join()
pw.join()
assert pr.exitcode == 0
assert pw.exitcode == 0
self.rh2.close()
self.wh2.close()
def main():
with gipc.pipe() as (r, w):
p = gipc.start_process(target=child_process, args=(r, ))
p = gipc.start_process(target=random_process)
wg = gevent.spawn(writegreenlet, w)
try:
p.join()
except KeyboardInterrupt:
wg.kill(block=True)
p.terminate()
p.join()
def main():
with gipc.pipe() as (r, w):
p = gipc.start_process(target=child_process, args=(r, ))
p = gipc.start_process(target=random_process)
wg = gevent.spawn(writegreenlet, w)
try:
p.join()
except KeyboardInterrupt:
wg.kill(block=True)
p.terminate()
p.join()
def initializeManyRouters(self, config, number, background):
'''Initialize many routers and background if required
Args:
config (Wishbone.config.configfile:ConfigFile): The router configration
number (int): The number of instances to intialize
background (bool): Whether to background the routers or not
'''
if background:
pid_file = PIDFile(self.pid)
with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, detach_process=True):
if self.instances == 1:
sys.stdout.write("\nWishbone instance started with pid %s\n" % (os.getpid()))
sys.stdout.flush()
pid_file.create([os.getpid()])
self.initializeOneRouter(config)
else:
for instance in range(self.instances):
self.routers.append(
gipc.start_process(
self.initializeOneRouter,
args=(config, ),
daemon=True
)
)
pids = [str(p.pid) for p in self.routers]
print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids))))
pid_file.create(pids)
self.bootstrapBlock()
else:
if self.instances == 1:
sys.stdout.write("\nInstance started in foreground with pid %s\n" % (os.getpid()))
self.initializeOneRouter(config)
else:
for instance in range(self.instances):
self.routers.append(
gipc.start_process(
self.initializeOneRouter,
args=(config, ),
daemon=True
)
)
pids = [str(p.pid) for p in self.routers]
print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids))))
self.bootstrapBlock()
def open_session(session_id):
client_id = str(uuid.uuid1())
if not session_id in INTERPRETER:
cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe()
output_queue_from_interpreter, output_queue = gipc.pipe()
RESULT[session_id] = dict()
config = static_config.get_config()
session = config.get(session_id)
session_cfg = config.get_config(session.name)
synoptic = session_cfg.get('synoptic')
if synoptic:
SYNOPTIC[session_id] = synoptic
INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter,
args=(session_id, cmds_queue, output_queue),
kwargs={"beacon_host": os.environ.get("BEACON_HOST"), "beacon_port": os.environ.get("BEACON_PORT") })
EXECUTION_QUEUE[session_id].put((None, "syn", (None,)))
output_queue_from_interpreter.get() #ack
OUTPUT_QUEUE[session_id] = dict()
gevent.spawn(handle_output, session_id, output_queue_from_interpreter)
RESULT[session_id][client_id] = gevent.event.AsyncResult()
OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue()
root_path = os.path.dirname(os.path.abspath(__file__))
contents = file(os.path.join(root_path, "shell.html"), "r")
template = Template(contents.read())
return template.render(client_uuid=repr(client_id))
def processLauncher(self):
with gipc.pipe() as (r, w):
self.comms.append((r, w))
p = gipc.start_process(target=self.sessionLauncher,
args=((r, self.greenthreads)))
p.join()
def start(self, auth_msg=''):
"""
Begins processing commands from the server.
"""
self.pipe, child_end = gipc.pipe()
self._socket_connect()
self.reader = gipc.start_process(
target=ServerReader,
args=(child_end, os.getpid(),
self.socket.fileno(),
self.tracer.pause_signal),
)
with Timeout(5, QdbFailedToConnect(self.tracer.address,
self.tracer.retry_attepts),
green=self.green):
# Receive a message to know that the reader is ready to begin.
self.pipe.get()
self.send(
fmt_msg(
'start', {
'uuid': self.tracer.uuid,
'auth': auth_msg,
'local': (0, 0),
},
serial=pickle.dumps,
)
)
atexit.register(self.stop)
def __init__(self, worker_func, task_count, worker_count=-1):
# initialize completion task workers
# if number of workers is not specified, set it to the number of CPUs
if worker_count == -1:
worker_count = 8
self.workers = CallbackTaskWorkers(worker_func, worker_count)
# start result manager
if PROC_TYPE == ProcType.gipc:
self.result_manager = gipc.start_process(target=result_manager,
args=(task_count,))
elif PROC_TYPE == ProcType.multiprocessing:
self.result_manager = Process(target=result_manager,
args=(task_count,))
self.result_manager.start()
elif PROC_TYPE == ProcType.greenlet:
self.result_manager = gevent.spawn(result_manager, task_count)
# initialize completion task runner
self.runner = CallbackTaskRunner()
# wait for all workers to subscribe to control channel
# MUST be placed above all other commands in __init__ to work
log.debug('Waiting for %d workers to initialize...', worker_count)
context = zmq.Context()
syncservice = context.socket(zmq.REP)
syncservice.bind(ZMQ_WORKER_SYNC_CH)
subscribers = 0
while subscribers < worker_count:
msg = syncservice.recv()
syncservice.send(msg) # send synchronization reply
subscribers = subscribers + 1
log.debug('%d workers initialized', worker_count)
def test_exitcode_sigkill(self):
p = start_process(p_child_b)
p.join()
if not WINDOWS:
assert p.exitcode == -signal.SIGKILL
else:
assert p.exitcode == 1
def _process(init, initargs, maxtasksperchild, func, itemq, resultq):
if maxtasksperchild is None:
task_limit = iter(int, 1) # no limit
else:
task_limit = range(maxtasksperchild)
proc = None
with gipc.pipe() as (i_send, items), gipc.pipe() as (results, r_send):
try:
with gipc_process_error_handler():
args = (init, initargs, func, i_send, r_send)
proc = gipc.start_process(target=_worker, args=args)
log.debug("start worker: %s", proc.pid)
for x in task_limit:
items.put(itemq.get())
result = results.get()
resultq.put(result)
if result is _Stop:
log.debug("worker stopped: %s", proc.pid)
return _Stop
items.put(_Stop)
except ProcessError:
log.error("process error %s", proc.pid)
finally:
if proc is not None:
proc.join()
if hasattr(proc, "close"): # added in Python 3.7
# HACK gipc bug https://github.com/jgehrcke/gipc/issues/90
proc._popen.poll = lambda: proc.exitcode
proc.close()
def test_wsgi_scenario(self):
from gevent.wsgi import WSGIServer
def serve(http_server):
http_server.serve_forever()
def hello_world(environ, start_response):
# Generate response in child process.
with pipe() as (reader, writer):
start_response('200 OK', [('Content-Type', 'text/html')])
rg = start_process(
target=complchild_test_wsgi_scenario_respgen,
args=(writer, ))
response = reader.get()
rg.join()
assert rg.exitcode == 0
return [response]
http_server = WSGIServer(('localhost', 0), hello_world)
servelet = gevent.spawn(serve, http_server)
# Wait for server being bound to socket.
while True:
if http_server.address[1] != 0:
break
gevent.sleep(0.05)
client = start_process(target=complchild_test_wsgi_scenario_client,
args=(http_server.address, ))
client.join()
assert client.exitcode == 0
servelet.kill()
servelet.get() # get() is join and re-raises Exception.
def handle_http_request(_, start_response):
"""I am executed in a greenlet whenever an HTTP request came in."""
# Start constructing an HTTP response. Build the header first.
start_response('200 OK', [('Content-Type', 'text/html')])
# What would we like to respond to the client? Let's ask a worker
# process to generate the HTTP response body for us. (When would we do
# this in the real world? For example when response generation requires
# significant CPU-bound work as in image processing or when compiling a
# complex PDF document, ...). In cases like this the CPU-bound work can
# severely impact the responsiveness of the HTTP server process.)
with gipc.pipe() as (r, w):
# Start the child process (each client connection makes the server
# process spawn a child process!). Pass the write end of the pipe to
# the child process.
p = gipc.start_process(target=child_msg_generator, args=(w, ))
# Read the message from the child process.
body = r.get()
# Reap child (call wait(), remove it from process table).
p.join()
assert p.exitcode == 0
# Write HTTP response body.
return [body]
def test_join_timeout(self):
p = start_process(gevent.sleep, args=(0.1, ))
p.join(ALMOSTZERO)
assert p.is_alive()
p.join()
assert p.exitcode == 0
_call_close_method_if_exists(p)
def spawn_processes(confs):
"""Spawn subprocesses with a config similar to spawn_greenlets."""
processes = []
for conf in confs:
p = gipc.start_process(target=conf[0], args=tuple(conf[1:]))
processes.append(p)
return processes
def open_session(session_id):
client_id = str(uuid.uuid1())
if not session_id in INTERPRETER:
cmds_queue, EXECUTION_QUEUE[session_id] = gipc.pipe()
output_queue_from_interpreter, output_queue = gipc.pipe()
RESULT[session_id] = dict()
config = static_config.get_config()
session = config.get(session_id)
session_cfg = config.get_config(session.name)
synoptic = session_cfg.get('synoptic')
if synoptic:
SYNOPTIC[session_id] = synoptic
INTERPRETER[session_id] = gipc.start_process(
interpreter.start_interpreter,
args=(session_id, cmds_queue, output_queue),
kwargs={
"beacon_host": os.environ.get("BEACON_HOST"),
"beacon_port": os.environ.get("BEACON_PORT")
})
EXECUTION_QUEUE[session_id].put((None, "syn", (None, )))
output_queue_from_interpreter.get() #ack
OUTPUT_QUEUE[session_id] = dict()
gevent.spawn(handle_output, session_id, output_queue_from_interpreter)
RESULT[session_id][client_id] = gevent.event.AsyncResult()
OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue()
root_path = os.path.dirname(os.path.abspath(__file__))
contents = file(os.path.join(root_path, "shell.html"), "r")
template = Template(contents.read())
return template.render(client_uuid=repr(client_id))
def test_orphaned_signal_watcher(self):
# Install libev-based signal watcher.
s = gevent.signal(signal.SIGTERM, signals_test_sigterm_handler)
# Normal behavior: signal handlers become inherited by children.
# Bogus behavior of libev-based signal watchers in child process:
# They should not be active anymore when 'orphaned' (when their
# corresponding event loop has been destroyed). What happens, however:
# The old handler stays active and registering a new handler does not
# 'overwrite' the old one -- both are active.
# Since this test is about testing the behavior of 'orphaned' libev
# signal watchers, the signal must be transmitted *after* event loop
# recreation, so wait here for the child process to go through
# the hub & event loop destruction (and recreation) process before
# sending the signal. Waiting is realized with sync through pipe.
# Without cleanup code in gipc, the inherited but orphaned libev signal
# watcher would be active in the fresh event loop and trigger the
# handler. This is a problem. With cleanup code, this handler must
# never be called. Child exitcode 20 means that the inherited handler
# has been called, -15 (-signal.SIGTERM) means that the child was
# actually killed by SIGTERM within a certain short time interval.
# Returncode 0 would mean that the child finished normally after that
# short time interval.
with pipe() as (r, w):
p = start_process(signals_test_child_a, (w, ))
assert r.get() == p.pid
os.kill(p.pid, signal.SIGTERM)
p.join()
if not WINDOWS:
assert p.exitcode == -signal.SIGTERM
else:
assert p.exitcode == signal.SIGTERM
s.cancel()
def ipc_child_f(w, m):
assert len(get_all_handles()) == 1
i = start_process(ipc_child_f2, args=(w, m))
i.join()
assert i.exitcode == 0
with raises(GIPCClosed):
w.close()
def test_threadpool_resolver_mp(self):
h = gevent.get_hub()
t = h.threadpool
r = h.resolver
p = start_process(target=complchild_test_threadpool_resolver_mp)
p.join(timeout=1)
assert p.exitcode == 0
def start(self):
'''Maps to the CLI command and starts one or more Wishbone processes in background.
'''
router_config = ConfigFile(self.config, 'SYSLOG').dump()
pid_file = PIDFile(self.pid)
with DaemonContext(stdout=sys.stdout,
stderr=sys.stderr,
detach_process=True):
if self.instances == 1:
sys.stdout.write("\nWishbone instance started with pid %s\n" %
(os.getpid()))
sys.stdout.flush()
pid_file.create([os.getpid()])
self.initializeRouter(router_config)
else:
for instance in range(self.instances):
self.routers.append(
gipc.start_process(self.initializeRouter,
args=(router_config, ),
daemon=True))
pids = [str(p.pid) for p in self.routers]
print(("\nInstances started in foreground with pid %s\n" %
(", ".join(pids))))
pid_file.create(pids)
self.bootstrapBlock()
def real_start(self):
pr, pw = gipc.pipe()
self.puppet_process = gipc.start_process(target=SockServer(Puppet).run,
kwargs={"pipe": pw})
port = pr.get()
self.puppet = SockClient(("localhost", port), keep_alive=False)
self.puppet.hire_worker()
def open_session(session_id):
client_id = str(uuid.uuid1())
if not session_id in INTERPRETER:
read_config(SHELL_CONFIG_FILE)
cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe()
output_queue_from_interpreter, output_queue = gipc.pipe()
RESULT[session_id] = dict()
setup_file = SETUP.get(session_id, {}).get("file")
config_objects_names = SETUP.get(session_id, {}).get("config_objects")
INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter,
args=(setup_file, config_objects_names, cmds_queue, output_queue))
EXECUTION_QUEUE[session_id].put((None, "syn", (None,)))
output_queue_from_interpreter.get() #ack
OUTPUT_QUEUE[session_id] = dict()
gevent.spawn(handle_output, session_id, output_queue_from_interpreter)
RESULT[session_id][client_id] = gevent.event.AsyncResult()
OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue()
root_path = os.path.dirname(os.path.abspath(__file__))
contents = file(os.path.join(root_path, "shell.html"), "r")
template = Template(contents.read())
return template.render(client_uuid=repr(client_id))
def start(self, auth_msg=''):
"""
Begins processing commands from the server.
"""
self.pipe, child_end = gipc.pipe()
self._socket_connect()
self.reader = gipc.start_process(
target=ServerReader,
args=(child_end, os.getpid(),
self.socket.fileno(),
self.tracer.pause_signal),
)
with Timeout(5, QdbFailedToConnect(self.tracer.address,
self.tracer.retry_attepts),
green=self.green):
# Receive a message to know that the reader is ready to begin.
self.pipe.get()
self.send(
fmt_msg(
'start', {
'uuid': self.tracer.uuid,
'auth': auth_msg,
'local': (0, 0),
},
serial=pickle.dumps,
)
)
atexit.register(self.stop)
def start(self):
'''Maps to the CLI command and starts one or more Wishbone processes in background.
'''
router_config = ConfigFile(self.config, 'SYSLOG').dump()
pid_file = PIDFile(self.pid)
with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, detach_process=True):
if self.instances == 1:
sys.stdout.write("\nWishbone instance started with pid %s\n" % (os.getpid()))
sys.stdout.flush()
pid_file.create([os.getpid()])
self.initializeRouter(router_config)
else:
for instance in range(self.instances):
self.routers.append(
gipc.start_process(
self.initializeRouter,
args=(router_config, ),
daemon=True
)
)
pids = [str(p.pid) for p in self.routers]
print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids))))
pid_file.create(pids)
self.bootstrapBlock()
def __init__(self, app):
super(PoWService, self).__init__(app)
cpu_pct = self.app.config['pow']['cpu_pct']
self.cpipe, self.ppipe = gipc.pipe(duplex=True)
self.worker_process = gipc.start_process(
target=powworker_process, args=(self.cpipe, cpu_pct))
self.app.services.chain.on_new_head_candidate_cbs.append(self.on_new_head_candidate)
self.hashrate = 0
def duplex():
with pipe() as (r, w):
with pipe() as (r2, w2):
p = start_process(complchild_test_multi_duplex, (r, w2))
w.put("msg")
assert r2.get() == "msg"
p.join()
assert p.exitcode == 0
def reloader(i):
global running, workers
while running:
print 'Starting worker {}.'.format(i)
workers[i] = gipc.start_process(server, (i,), daemon=True, name='worker{}'.format(i))
workers[i].join()
print 'Worker {} has just quit!'.format(i)
print 'Done with worker {}'.format(i)
def test_threadpool_resolver_mp(self):
h = gevent.get_hub()
t = h.threadpool
r = h.resolver
p = start_process(target=complchild_test_threadpool_resolver_mp)
# Note(JP): seen this fail once on Windows CI with a timeout of 1 s.
p.join(timeout=2)
assert p.exitcode == 0
def test_terminate(self):
p = start_process(gevent.sleep, args=(1, ))
# Test __repr__ and __str__
p.__repr__()
p.terminate()
p.join()
p.__repr__()
assert p.exitcode == -signal.SIGTERM
def test_handlecount(self):
p = start_process(ipc_handlecounter1, args=(self.rh, self.rh2))
p.join()
assert p.exitcode == 0
# After passing two handles to the child, only 2 must be left open.
assert len(get_all_handles()) == 2
self.wh.close()
self.wh2.close()
def setup_children_process():
global gConfig
if 'children_wsgi' in gConfig['wsgi']:
if len(list(gConfig['wsgi']['children_wsgi'].keys())):
for k in gConfig['wsgi']['children_wsgi'].keys():
if 'config' in gConfig['wsgi']['children_wsgi'][k]:
config = gConfig['wsgi']['children_wsgi'][k]['config']
p = gipc.start_process(target=run_children_process, args=(config, ), daemon=True)
def test_whatever_2(self):
"""
Time-synchronize two child processes via two unidirectional channels.
Uses timeout control in children.
"""
# First pipe for sync.
with pipe() as (syncreader, syncwriter):
# Second pipe for communication.
with pipe() as (r, w):
# Send messages
pw = start_process(usecase_child_b, args=(w, syncreader))
# Receive messages
pr = start_process(usecase_child_c, args=(r, syncwriter))
pw.join()
pr.join()
assert pw.exitcode == 0
assert pr.exitcode == 5
def test_close_raises_if_called_prematurely(self):
p = start_process(p_child_a)
assert p.is_alive()
with raises(ValueError, match="while it is still running"):
p.close()
p.terminate()
p.join()
p.close()
def initialize(PK, size=1):
global _procs, myPK
myPK = PK
_procs = []
for s in range(size):
(r,w) = gipc.pipe(duplex=True)
p = gipc.start_process(_worker, args=(PK, r,))
_procs.append((p,w))
def test_exitcode_sigkill(self):
p = start_process(p_child_b)
p.join()
if not WINDOWS:
assert p.exitcode == -signal.SIGKILL
else:
assert p.exitcode == 1
_call_close_method_if_exists(p)
def bulk_operation(self, serializer, index, client, **options):
procs = options.get('multi')
if not procs:
procs = os.cpu_count()
procs = procs if procs > 1 else 2
data_size = serializer.fetch_data_length()
chunk_size = int(data_size / (procs * 4))
def parallel_params():
for chunk_num in range(procs * 4 + 1):
_options = options.copy()
_options['parallel_chunk_num'] = chunk_num
_options['parallel_chunk_size'] = chunk_size
yield hash(serializer), index, _options
if not self.pipe_fd_mapping:
# reset multiprocessing.Process-sensitive elements just before forking
self.parallel_prep()
for n in range(procs):
h1, h2 = gipc.pipe(duplex=True)
self.pipe_fd_mapping[h1._reader._fd] = (
h1,
gipc.start_process(process_func,
args=(h2, ),
name="esdocs-proc-{}".format(n)))
# wait until sub-process workers check in
ready = 0
while ready != procs:
readable, _, _ = select(self.pipe_fd_mapping.keys(), [], [])
for fd in readable:
h1, proc = self.pipe_fd_mapping[fd]
msg, data = h1.get()
if msg == 'GOOD-MORNING-DAVE':
ready += 1
parallel_chunk_params = parallel_params()
# populate all the processes with an even number of chunks to process
chunks_remaining = True
chunks_to_process = 0
while chunks_remaining:
for h1, proc in self.pipe_fd_mapping.values():
try:
h1.put(next(parallel_chunk_params))
chunks_to_process += 1
except StopIteration:
chunks_remaining = False
break
while chunks_to_process > 0:
readable, _, _ = select(self.pipe_fd_mapping.keys(), [], [])
for fd in readable:
h1, proc = self.pipe_fd_mapping[fd]
msg, data = h1.get()
if msg == 'DONE-CHUNK':
chunks_to_process -= 1
def _run_job_python(job_signaler, job, results): res = _Result(job, None) p = gipc.start_process(target=job.cmd) job_signaler.add(res, p.terminate) p.join() job_signaler.rm(res) if p.exitcode != 0 and res.e is None: res.e = ExecError(p.exitcode) results.send(res)
def main():
with gipc.pipe() as (readend, writeend):
# Start "writer" Greenlet. Provide it with the pipe with write end
g = gevent.spawn(writelet, writeend)
# Start "reader" child process. Provide it with the pipe read end
p = gipc.start_process(target=readchild, args=(readend,))
g.join()
p.join()
def _create_model_from_factory(manager, email, path, create_model, params, start_stop_rows, table):
schema = PARAM_SCHEMAS_SERVE[path]
model_params = _parse_params(params, schema)
inputs = {x: _get_input(params, x) for x in schema['input_types']}
with gipc.pipe() as (reader, writer):
p = gipc.start_process(target=create_model, args=(writer, model_params, inputs, schema, start_stop_rows, table, email))
row = reader.get()
p.join()
return {'row': base64.b64encode(row)}
def test_early_readchild_exit_write_from_child(self):
pr = start_process(ipc_readonce_then_exit, (self.rh, ))
pw = start_process(ipc_endless_write_for_early_reader_exit,
(self.wh, ))
# This test is to make sure equivalent behavior as in test
# `test_early_readchild_exit` when the writing process is a
# child process itself (above, the write process in the initial
# process). Since gipc's child process creation
# routine messes around with signal handlers, this test makes
# sure that SIGPIPE is ignored in the child and that a
# failing write attempt (after early read child exit) results
# in an exception raised in the writing process.
pr.join()
pw.join()
assert pr.exitcode == 0
assert pw.exitcode == 0
self.rh2.close()
self.wh2.close()
def init(self):
for _ in xrange(self.process_size):
child_pipe_end, parent_pipe_end = gipc.pipe(duplex=True)
self.child_pipe_ends.append(child_pipe_end)
self.parent_pipe_ends.append(parent_pipe_end)
p = gipc.start_process(target=self.process_target, args=(child_pipe_end, ), daemon=True)
self.processes.append(p)
#p.join()
self.loop_get_result()
def __init__(self, worker_func, worker_count):
self.workers = [None]*worker_count
for wrk_num in range(worker_count):
if PROC_TYPE == ProcType.gipc:
self.workers[wrk_num] = gipc.start_process(target=self._callback_worker,
args=(wrk_num, worker_func))
elif PROC_TYPE == ProcType.multiprocessing:
self.workers[wrk_num] = Process(target=self._callback_worker,
args=(wrk_num, worker_func))
self.workers[wrk_num].start()
elif PROC_TYPE == ProcType.greenlet:
self.workers[wrk_num] = gevent.spawn(self._callback_worker, wrk_num, worker_func)
def __init__(self, num_procs=multiprocessing.cpu_count(), num_greenlets=50):
super(GIPCExecutor, self).__init__()
self.procs = []
self.pipes = []
self.results = {}
for _ in xrange(num_procs):
cend, pend = gipc.pipe(duplex=True)
proc = gipc.start_process(self._proc_loop, args=(cend, num_greenlets),
daemon=True)
self.procs.append(proc)
self.pipes.append(pend)
self.pipes = itertools.cycle(self.pipes)
self.result_pool = self._result_pool()
def start(self):
"""
Starts the task's process
"""
self.pipe, pipe_child = gipc.pipe(duplex=True)
self.running = True
self.process = gipc.start_process(
target=self._worker,
kwargs={
'pipe': pipe_child,
}
)
self.reader = gevent.spawn(self._reader)
def fork_service(klass, concurrency=10, queue_size=None):
"""simply create service process, returns tuple like (service_process, client)
you can communicate through client's call/call_async/call_callback methods.
when you use fork_service(), please make sure to call
client.close() and service_process.join()"""
global _internal_pid
_internal_pid += 1
child_ch, parent_ch = gipc.pipe(duplex=True)
service = klass()
args = (child_ch, _internal_pid, concurrency, queue_size)
service_process = gipc.start_process(target=service, args=args)
client = IPCRPCClient(parent_ch)
return (service_process, client)
def start(self):
pipe_parent, pipe_child = gipc.pipe(duplex=True)
self.stream = GateStreamServerEndpoint(pipe_parent)
stream_child = GateStreamWorkerEndpoint(pipe_child)
self.process = gipc.start_process(
target=self._target,
kwargs={
'stream': stream_child,
'_pipe': pipe_child,
}
)
logging.debug('Started child process %s', self.process.pid)
self.stream_reader = gevent.spawn(self._stream_reader)
def main():
global running, channel_mngr, workers
# Make green psycopg:
extensions.set_wait_callback(gevent_wait_callback)
# Decide how many processes to use
try:
proc_count = int(sys.argv[1])
except:
proc_count = multiprocessing.cpu_count()
workers = [None] * proc_count
channel.init(proc_count)
gevent.signal(signal.SIGINT, sig_quit)
gevent.signal(signal.SIGTERM, sig_quit)
# Remove the sockets directory
shutil.rmtree(sock_dir, True)
os.mkdir(sock_dir)
# Spawn the reloaders for the workers
reloaders = [gevent.spawn(reloader, i) for i in xrange(proc_count)]
setproctitle.setproctitle('diggems master')
# Channel manager process reloader:
while running:
print 'Starting channel manager process.'
proc = gipc.start_process(channel.rpc_dispatcher, daemon=True, name='channel_mngr')
proc.join()
print 'Channel manager process has quit.'
print 'Done with channel manager'
gevent.joinall(reloaders)
print 'All done, quiting'
def debug(self):
'''Maps to the CLI command and starts Wishbone in foreground.
'''
router_config = ConfigFile(self.config, 'STDOUT').dump()
if self.instances == 1:
sys.stdout.write("\nInstance started in foreground with pid %s\n" % (os.getpid()))
self.initializeRouter(router_config)
else:
for instance in range(self.instances):
self.routers.append(
gipc.start_process(
self.initializeRouter,
args=(router_config, ),
daemon=True
)
)
pids = [str(p.pid) for p in self.routers]
print(("\nInstances started in foreground with pid %s\n" % (", ".join(pids))))
self.bootstrapBlock()
def start(self):
start_process(self.run)
f.write(f.readline())
f.flush()
f.close()
def server():
StreamServer(("localhost", PORT), serve).serve_forever()
def clientprocess():
t1 = time.time()
clients = [gevent.spawn(client) for _ in xrange(N_CLIENTS)]
gevent.joinall(clients)
duration = time.time() - t1
print "{} clients served within {:.2f} s".format(N_CLIENTS, duration)
def client():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("localhost", PORT))
f = sock.makefile()
f.write(MSG)
f.flush()
assert f.readline() == MSG
f.close()
if __name__ == "__main__":
s = gevent.spawn(server)
c = gipc.start_process(clientprocess)
c.join()
s.kill()
s.join()
def real_start(self):
pr, pw = gipc.pipe()
self.puppet_process = gipc.start_process(target=SockServer(Puppet).run, kwargs={"pipe": pw})
port = pr.get()
self.puppet = SockClient(("localhost", port), keep_alive=False)
self.puppet.hire_worker()
def run(self):
e = gipc.start_process(target=Executor, args=("Executor",))
# c = gipc.start_process(target=Collector, args=('Collector',))
e.join()
def hire_worker(self):
tq_worker, self.task_queue = gipc.pipe(duplex=True)
self.worker = gipc.start_process(target=process_worker, args=(tq_worker,))
self.operator = gevent.spawn(self.receive_info)
def run(self):
e = gipc.start_process(target = Executor,args=('Executor',))
e.join()
