def open_session(session_id):
client_id = str(uuid.uuid1())
if not session_id in INTERPRETER:
cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe()
output_queue_from_interpreter, output_queue = gipc.pipe()
RESULT[session_id] = dict()
config = static_config.get_config()
session = config.get(session_id)
session_cfg = config.get_config(session.name)
synoptic = session_cfg.get('synoptic')
if synoptic:
SYNOPTIC[session_id] = synoptic
INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter,
args=(session_id, cmds_queue, output_queue),
kwargs={"beacon_host": os.environ.get("BEACON_HOST"), "beacon_port": os.environ.get("BEACON_PORT") })
EXECUTION_QUEUE[session_id].put((None, "syn", (None,)))
output_queue_from_interpreter.get() #ack
OUTPUT_QUEUE[session_id] = dict()
gevent.spawn(handle_output, session_id, output_queue_from_interpreter)
RESULT[session_id][client_id] = gevent.event.AsyncResult()
OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue()
root_path = os.path.dirname(os.path.abspath(__file__))
contents = file(os.path.join(root_path, "shell.html"), "r")
template = Template(contents.read())
return template.render(client_uuid=repr(client_id))
def open_session(session_id):
client_id = str(uuid.uuid1())
if not session_id in INTERPRETER:
read_config(SHELL_CONFIG_FILE)
cmds_queue,EXECUTION_QUEUE[session_id] = gipc.pipe()
output_queue_from_interpreter, output_queue = gipc.pipe()
RESULT[session_id] = dict()
setup_file = SETUP.get(session_id, {}).get("file")
config_objects_names = SETUP.get(session_id, {}).get("config_objects")
INTERPRETER[session_id] = gipc.start_process(interpreter.start_interpreter,
args=(setup_file, config_objects_names, cmds_queue, output_queue))
EXECUTION_QUEUE[session_id].put((None, "syn", (None,)))
output_queue_from_interpreter.get() #ack
OUTPUT_QUEUE[session_id] = dict()
gevent.spawn(handle_output, session_id, output_queue_from_interpreter)
RESULT[session_id][client_id] = gevent.event.AsyncResult()
OUTPUT_QUEUE[session_id][client_id] = gevent.queue.Queue()
root_path = os.path.dirname(os.path.abspath(__file__))
contents = file(os.path.join(root_path, "shell.html"), "r")
template = Template(contents.read())
return template.render(client_uuid=repr(client_id))
def main():
filename_reader, filename_writer = gipc.pipe()
url_reader, url_writer = gipc.pipe()
filter_phrases = ['lol', 'haha', 'wow', 'incredible']
#filter_phrases = []
filter_phrases += ['jpg', 'jpeg']
processes = spawn_processes([
(twitter_process, url_writer, filter_phrases),
(image_fetch_process, url_reader, filename_writer),
(detector_process, filename_reader),
])
while True:
gevent.sleep(1)
def fork(fn, *args, daemon=False):
'''
Creates a child process that runs the given function.
@sig fork :: (* -> _) -> * -> Bool -> _
'''
with gipc.pipe():
gipc.start_process(target=fn, args=args, daemon=daemon)
def start(self, auth_msg=''):
"""
Begins processing commands from the server.
"""
self.pipe, child_end = gipc.pipe()
self._socket_connect()
self.reader = gipc.start_process(
target=ServerReader,
args=(child_end, os.getpid(),
self.socket.fileno(),
self.tracer.pause_signal),
)
with Timeout(5, QdbFailedToConnect(self.tracer.address,
self.tracer.retry_attepts),
green=self.green):
# Receive a message to know that the reader is ready to begin.
self.pipe.get()
self.send(
fmt_msg(
'start', {
'uuid': self.tracer.uuid,
'auth': auth_msg,
'local': (0, 0),
},
serial=pickle.dumps,
)
)
atexit.register(self.stop)
def initialize(PK, size=1):
global _procs, myPK
myPK = PK
_procs = []
for s in range(size):
(r,w) = gipc.pipe(duplex=True)
p = gipc.start_process(_worker, args=(PK, r,))
_procs.append((p,w))
def __init__(self, app):
super(PoWService, self).__init__(app)
cpu_pct = self.app.config['pow']['cpu_pct']
self.cpipe, self.ppipe = gipc.pipe(duplex=True)
self.worker_process = gipc.start_process(
target=powworker_process, args=(self.cpipe, cpu_pct))
self.app.services.chain.on_new_head_candidate_cbs.append(self.on_new_head_candidate)
self.hashrate = 0
def main():
buffer_reader, buffer_writer = gipc.pipe()
processes = spawn_processes([
(sampler_process, buffer_writer, sample_counter()),
(renderer_process, buffer_reader,),
])
while True:
gevent.sleep(1)
def _create_model_from_factory(manager, email, path, create_model, params, start_stop_rows, table):
schema = PARAM_SCHEMAS_SERVE[path]
model_params = _parse_params(params, schema)
inputs = {x: _get_input(params, x) for x in schema['input_types']}
with gipc.pipe() as (reader, writer):
p = gipc.start_process(target=create_model, args=(writer, model_params, inputs, schema, start_stop_rows, table, email))
row = reader.get()
p.join()
return {'row': base64.b64encode(row)}
def main():
with gipc.pipe() as (readend, writeend):
# Start "writer" Greenlet. Provide it with the pipe with write end
g = gevent.spawn(writelet, writeend)
# Start "reader" child process. Provide it with the pipe read end
p = gipc.start_process(target=readchild, args=(readend,))
g.join()
p.join()
def test_json(self):
import json
data = {"a": 100}
enc = lambda o: json.dumps(o).encode("ascii")
dec = lambda b: json.loads(b.decode("ascii"))
with pipe(encoder=enc, decoder=dec) as (r, w):
gw = gevent.spawn(self.writelet, w, data)
gr = gevent.spawn(self.readlet, r)
assert data == gr.get()
gw.join()
def test_lock_out_of_context_pair_2(self):
with raises(GIPCLocked):
with pipe() as (r, w):
gr = gevent.spawn(lambda r: r.get(), r)
gevent.sleep(SHORTTIME)
# Context manager tries to close writer first, succeeds,
# and fails during closing reader.
gr.kill(block=False)
gevent.sleep(-1)
r.close()
def init(self):
for _ in xrange(self.process_size):
child_pipe_end, parent_pipe_end = gipc.pipe(duplex=True)
self.child_pipe_ends.append(child_pipe_end)
self.parent_pipe_ends.append(parent_pipe_end)
p = gipc.start_process(target=self.process_target, args=(child_pipe_end, ), daemon=True)
self.processes.append(p)
#p.join()
self.loop_get_result()
def main():
with gipc.pipe() as (r, w):
p = gipc.start_process(target=child_process, args=(r, ))
wg = gevent.spawn(writegreenlet, w)
try:
p.join()
except KeyboardInterrupt:
wg.kill(block=True)
p.terminate()
p.join()
def test_single_writer(self):
r, w = pipe()
with r as foo:
fd = foo._fd
with raises(OSError):
os.close(fd)
assert len(get_all_handles()) == 1
with raises(GIPCClosed):
r.close()
w.close()
def start(self):
"""
Starts the task's process
"""
self.pipe, pipe_child = gipc.pipe(duplex=True)
self.running = True
self.process = gipc.start_process(target=self._worker,
kwargs={
'pipe': pipe_child,
})
self.reader = gevent.spawn(self._reader)
def main():
with gipc.pipe() as (r, w):
p = gipc.start_process(target=child_process, args=(r, ))
p = gipc.start_process(target=random_process)
wg = gevent.spawn(writegreenlet, w)
try:
p.join()
except KeyboardInterrupt:
wg.kill(block=True)
p.terminate()
p.join()
def benchmark(N, msg):
result = None
with gipc.pipe() as (syncr, syncw):
with gipc.pipe() as (reader, writer):
p = gipc.start_process(writer_process,
kwargs={
'writer': writer,
'syncr': syncr,
'N': N,
'msg': msg
})
# Synchronize with child process
syncw.put("SYN")
assert reader.get() == "ACK"
t = timer()
while result != 'stop':
result = reader.get()
elapsed = timer() - t
p.join()
return N, elapsed
def __enter__(self):
log.debug("starting case diff process")
self.calls_pipe = gipc.pipe()
self.stats_pipe = gipc.pipe()
calls, self.calls = self.calls_pipe.__enter__()
self.stats, stats = self.stats_pipe.__enter__()
debug = log.isEnabledFor(logging.DEBUG)
is_rebuild = self.statedb.is_rebuild
args = (
self.queue_class,
calls,
stats,
self.statedb.domain,
self.state_path,
is_rebuild,
debug,
)
self.process = gipc.start_process(target=run_case_diff_queue, args=args)
self.status_logger = gevent.spawn(self.run_status_logger)
return self
def hello_world(environ, start_response):
# Generate response in child process.
with pipe() as (reader, writer):
start_response('200 OK', [('Content-Type', 'text/html')])
rg = start_process(
target=complchild_test_wsgi_scenario_respgen,
args=(writer, ))
response = reader.get()
rg.join()
assert rg.exitcode == 0
return [response]
def initialize(PK, size=1):
global _procs, myPK
myPK = PK
_procs = []
for s in range(size):
(r, w) = gipc.pipe(duplex=True)
p = gipc.start_process(_worker, args=(
PK,
r,
))
_procs.append((p, w))
def test_circular_forward(self):
with pipe(True) as (p11, p12):
with pipe(True) as (p21, p22):
with pipe(True) as (p31, p32):
# Spawn two forwarders.
forwarder1 = start_process(duplchild_circular_forward,
(p12, p21))
forwarder2 = start_process(duplchild_circular_forward,
(p22, p31))
# Make sure that only 2 of 6 handles are usable.
for h in (p12, p21, p22, p31):
with raises(GIPCClosed):
h.put(0)
# Send messages on their journey through children.
for _ in range(100):
p11.put("BABUUUZ")
assert p32.get() == "BABUUUZ"
p11.put("stop")
forwarder1.join()
forwarder2.join()
def test_context_close(self):
with pipe(duplex=True) as (h1, h2):
fd1 = h1._reader._fd
fd2 = h1._writer._fd
fd3 = h2._reader._fd
fd4 = h2._writer._fd
# Make sure the C file descriptors are closed.
for f in (fd1, fd2, fd3, fd4):
with raises(OSError):
os.close(f)
def main():
try:
buffer_reader, buffer_writer = gipc.pipe()
processes = spawn_processes([
(sampler_process, buffer_writer, sample_counter()),
(renderer_process, buffer_reader,),
])
while True:
gevent.sleep(1)
except KeyboardInterrupt:
pass
def main():
with gipc.pipe(duplex=True) as (cend, pend):
# `cend` is the channel end for the child, `pend` for the parent.
p = gipc.start_process(writer_process, args=(cend, ))
# Synchronize with child process.
pend.put("SYN")
assert pend.get() == "ACK"
# Now in sync with child.
ptime = timer()
ctime = pend.get()
p.join()
print("Time delta: %.8f s." % abs(ptime - ctime))
def test_time_sync(self):
with pipe(duplex=True) as (cend, pend):
p = start_process(duplchild_time_sync, args=(cend, ))
pend.put("SYN")
assert pend.get() == "ACK"
ptime = time.time()
ctime = pend.get()
# Require small time delta. Note: on Windows on a machine with
# diverse load I have seen this time difference to be 0.02 seconds.
# See https://github.com/jgehrcke/gipc/issues/70.
assert abs(ptime - ctime) < 0.03
p.join()
def __init__(self, count, **args):
self.pids = []
self._p = [] # processes
self._r = gevent.queue.Queue() # ready
self.init(None, args)
for num in range(count):
one, two = gipc.pipe(True)
proc = gipc.start_process(target=self.remote,
args=(one, num, args))
self.pids.append(proc.pid)
self._r.put(two)
self._p.append((proc, two))
def __init__(self, num_procs=multiprocessing.cpu_count(), num_greenlets=50):
super(GIPCExecutor, self).__init__()
self.procs = []
self.pipes = []
self.results = {}
for _ in xrange(num_procs):
cend, pend = gipc.pipe(duplex=True)
proc = gipc.start_process(self._proc_loop, args=(cend, num_greenlets),
daemon=True)
self.procs.append(proc)
self.pipes.append(pend)
self.pipes = itertools.cycle(self.pipes)
self.result_pool = self._result_pool()
def test_context_single_close(self):
h1, h2 = pipe(duplex=True)
with h1 as side1:
fd1 = side1._reader._fd
fd2 = side1._writer._fd
# Make sure the C file descriptors are closed.
with raises(OSError):
os.close(fd1)
with raises(OSError):
os.close(fd2)
with raises(GIPCClosed):
h1.close()
h2.close()
def start(self):
"""
Starts the task's process
"""
self.pipe, pipe_child = gipc.pipe(duplex=True)
self.running = True
self.process = gipc.start_process(
target=self._worker,
kwargs={
'pipe': pipe_child,
}
)
self.reader = gevent.spawn(self._reader)
def start(self):
pipe_parent, pipe_child = gipc.pipe(duplex=True)
self.stream = GateStreamServerEndpoint(pipe_parent)
stream_child = GateStreamWorkerEndpoint(pipe_child)
self.process = gipc.start_process(target=self._target,
kwargs={
'stream': stream_child,
'_pipe': pipe_child,
})
logging.debug('Started child process %s', self.process.pid)
self.stream_reader = gevent.spawn(self._stream_reader)
def test_lock_out_of_context_pair(self):
with raises(GIPCLocked):
with pipe() as (r, w):
# Fill up pipe and try to write more than pipe can hold
# (makes `put` block when there is no reader).
# Buffer is quite large on Windows.
gw = gevent.spawn(lambda w: w.put(LONGERTHANBUFFER), w)
gevent.sleep(SHORTTIME)
# Context manager tries to close writer first, fails,
# and must close reader nevertheless.
# Kill greenlet (free lock on writer) and close writer.
gw.kill(block=False)
gevent.sleep(-1)
w.close()
def test_lock_out_of_context_pair_2(self):
with raises(GIPCLocked):
with pipe(True) as (h1, h2):
gr = gevent.spawn(lambda h: h.get(), h2)
gevent.sleep(SHORTTIME)
# Context succeeds closing h1 reader and writer. Fails during
# closing h2 reader.
assert not h2._reader._closed
assert h1._reader._closed
assert h2._writer._closed
assert h1._writer._closed
gr.kill(block=False)
gevent.sleep(-1)
h2.close()
def fork_service(klass, concurrency=10, queue_size=None):
"""simply create service process, returns tuple like (service_process, client)
you can communicate through client's call/call_async/call_callback methods.
when you use fork_service(), please make sure to call
client.close() and service_process.join()"""
global _internal_pid
_internal_pid += 1
child_ch, parent_ch = gipc.pipe(duplex=True)
service = klass()
args = (child_ch, _internal_pid, concurrency, queue_size)
service_process = gipc.start_process(target=service, args=args)
client = IPCRPCClient(parent_ch)
return (service_process, client)
def ipc_child_c(r1, r2, m1, m2):
assert r1.get() == m1
# Test messaging between greenlets in child.
local_reader, local_writer = pipe()
testmsg = [1] * LONG
gw = gevent.spawn(lambda w: w.put(testmsg), local_writer)
gr = gevent.spawn(lambda r: r.get(), local_reader)
assert testmsg == gr.get()
gr.get()
gw.get()
local_reader.close()
local_writer.close()
# Receive second message from parent.
assert r2.get() == m2
def main():
log.info('Creating data ...')
N = 10**7
n = 80
if platform.python_implementation() == 'PyPy':
# This example seems to suffer from a severe performance problem on
# PyPy. On my machine I got 890 MBytes/s on CPython 3.6.3 / gevent
# 1.3.6, whereas with PyPy35-6.0.0 (everything else constant) I saw 3
# MB/s. Adopt to this so that this executes within reasonable time
# during CI.
N = 10**6
n = 20
if platform.python_implementation() == 'CPython' and WINDOWS:
# Temporarily work around the inability to send large messages on
# Windows. Fixing that is tracked here:
# https://github.com/jgehrcke/gipc/issues/69
N = 10**6
n = 20
# Concatenate a smaller chunk of random data multiple times (that's faster
# than creating a big chunk of random data).
data = os.urandom(N) * n
mbytes = len(data) / 1024.0 / 1024
log.info('Data size: %s MBytes' % mbytes)
checksum = hashlib.md5(data).digest()
with gipc.pipe(duplex=True) as (c, p):
log.info('Test with default pipe...')
spawn_child_transfer(c, p, data, checksum)
with gipc.pipe(duplex=True, encoder=None, decoder=None) as (c, p):
log.info('Test with raw pipe...')
spawn_child_transfer(c, p, data, checksum)
def start(self):
pipe_parent, pipe_child = gipc.pipe(duplex=True)
self.stream = GateStreamServerEndpoint(pipe_parent)
stream_child = GateStreamWorkerEndpoint(pipe_child)
self.process = gipc.start_process(
target=self._target,
kwargs={
'stream': stream_child,
'_pipe': pipe_child,
}
)
logging.debug('Started child process %s', self.process.pid)
self.stream_reader = gevent.spawn(self._stream_reader)
def test_lock_out_of_context_single(self):
r, w = pipe()
g = gevent.spawn(lambda r: r.get(), r)
gevent.sleep(SHORTTIME)
with raises(GIPCLocked):
with r:
pass
# The context manager can't close `r`, as it is locked in `g`.
g.kill(block=False)
# Ensure killing via 'context switch', i.e. yield control to other
# coroutines (otherwise the subsequent close attempt will fail with
# `GIPCLocked` error).
gevent.sleep(-1)
# Close writer first. otherwise, `os.close(r._fd)` would block on Win.
w.close()
r.close()
def start(self):
import gipc
workers = self.workers
parent_pipes = self.parent_pipes
for idx in range(self.num_workers):
child, parent = gipc.pipe(duplex=True)
with child:
worker = gipc.start_process(target=post_creator_process,
args=(child, idx) + self.args,
kwargs=self.kwargs)
workers.append(worker)
parent_pipes.append((parent, idx))
self.pool.spawn(self.child_listener, idx, parent)
self.pool.spawn(self.dispatch)
def start(self):
pipe_parent, pipe_child = gipc.pipe(
duplex=True,
encoder=lambda x: pickle.dumps(x, 2),
)
self.stream = GateStreamServerEndpoint(pipe_parent)
stream_child = GateStreamWorkerEndpoint(pipe_child)
self.process = gipc.start_process(target=self._target,
kwargs={
'stream': stream_child,
'_pipe': pipe_child,
})
logging.debug(f'Started child process {self.process.pid}')
self.stream_reader = gevent.spawn(self._stream_reader)
def test_lock_out_of_context_pair_3(self):
with raises(GIPCLocked):
with pipe(True) as (h1, h2):
gr1 = gevent.spawn(lambda h: h.get(), h1)
gr2 = gevent.spawn(lambda h: h.get(), h2)
gevent.sleep(SHORTTIME)
# Context succeeds closing h2 writer, fails upon closing h2
# reader. Proceeds closing h1 writer, succeeds, closes h1
# reader and fails.
assert not h2._reader._closed
assert not h1._reader._closed
assert h2._writer._closed
assert h1._writer._closed
gr1.kill(block=False)
gr2.kill(block=False)
gevent.sleep(-1)
h2.close()
h1.close()
def test_lock_out_of_context_single(self):
h1, h2 = pipe(True)
g = gevent.spawn(lambda h: h.get(), h1)
gevent.sleep(SHORTTIME)
with raises(GIPCLocked):
with h1:
pass
# Can't close h1 reader on exit, as it is locked in `g`.
g.kill(block=False)
# Ensure killing via 'context switch', i.e. yield control to other
# coroutines (otherwise the subsequent close attempt may fail with
# `GIPCLocked` error).
gevent.sleep(-1)
h2.close() # Closes read and write handles of h2.
assert h1._writer._closed
assert not h1._reader._closed
h1.close() # Closes read handle, ignore that writer is already closed.
assert h1._reader._closed
def __init__(self):
"""Initialize the queue and spawn the main loop thread
Upon initialization, tasks stored in the database are immediately
scheduled.
_task_queue is a priority queue ordered using Python's heapq functionality.
Elements in _task_queue are tuples of the form (datetime, task) where
datetime is the scheduled run time and task is a dictionary as defined
in the above docstring for the Scheduler class.
For concurrency safety reasons, never write to _task_queue outside the
_loop() thread.
"""
self._task_queue = [] # Never write to this outside the _loop thread
self._pending_cancels = set()
self._executor = GIPCExecutor()
# Load previously scheduled tasks from database
now = datetime.datetime.now()
with get_app().app_context():
saved_schedule = Task.query.filter_by(active=True)
for task in saved_schedule:
new_task = {
'id': task.id,
'interval': task.interval,
'code': task.code
}
# Writing directly to the _task_queue is safe since we haven't started
# the _loop yet
self._task_queue.append((now, new_task))
# Make _task_queue a priority queue
heapify(self._task_queue)
# Spawn main loop and save writer for future communication
(read, write) = gipc.pipe()
self._main_thread = gevent.spawn(self._loop, read)
self._schedule_pipe = write
atexit.register(self._interrupt)
def test_lock_out_of_context_pair(self):
with raises(GIPCLocked):
with pipe(True) as (h1, h2):
# Write more to pipe than pipe buffer can hold
# (makes `put` block when there is no reader).
# Buffer is quite large on Windows.
gw = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h1)
gevent.sleep(SHORTTIME)
# Context manager tries to close h2 reader, h2 writer, and
# h1 writer first. Fails upon latter, must still close
# h1 reader after that.
assert not h1._writer._closed
assert h1._reader._closed
assert h2._writer._closed
assert h2._reader._closed
# Kill greenlet (free lock on h1 writer), close h1 writer.
gw.kill(block=False)
gevent.sleep(-1)
h1.close()
assert h1._writer._closed
def test_lock_out_of_context_pair_4(self):
with raises(GIPCLocked):
with pipe(True) as (h1, h2):
# Write more to pipe than pipe buffer can hold
# (makes `put` block when there is no reader).
# Buffer is quite large on Windows.
gw1 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h1)
gw2 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h2)
gevent.sleep(SHORTTIME)
# Context fails closing h2 writer, succeeds upon closing h2
# reader. Proceeds closing h1 writer, fails, closes h1
# reader and succeeds.
assert h2._reader._closed
assert h1._reader._closed
assert not h2._writer._closed
assert not h1._writer._closed
gw1.kill(block=False)
gw2.kill(block=False)
gevent.sleep(-1)
h2.close()
h1.close()
def process_item(document_id):
# Do this in another greenlet to not block the main coroutine
document = redis_client.get(document_id)
redis_client.delete(
document_id) # It's only the data, so we remove it as well
process_lock.acquire()
try:
with gipc.pipe() as (reader, writer):
gipc.start_process(analyze_item, args=(writer, document))
parsed_data = reader.get()
print 'Received parsed data: {}'.format(parsed_data)
for category, word in parsed_data:
data = json.dumps({
'word': word,
'category': CATEGORY_MAP[category],
'source': document_id
})
response = requests.post(API_CREATION_URL,
data=data,
headers=API_HEADERS)
try:
response.raise_for_status()
except requests.RequestException:
if response.status_code == httplib.BAD_REQUEST and response.json(
) == {
'error': 'key already exists'
}:
continue
print 'Received bad response from API: {}: {}'.format(
response.status_code, response.json())
finally:
process_lock.release()
def hire_worker(self):
tq_worker, self.task_queue = gipc.pipe(duplex=True)
self.worker = gipc.start_process(target=process_worker, args=(tq_worker,))
self.operator = gevent.spawn(self.receive_info)
def real_start(self):
pr, pw = gipc.pipe()
self.puppet_process = gipc.start_process(target=SockServer(Puppet).run, kwargs={"pipe": pw})
port = pr.get()
self.puppet = SockClient(("localhost", port), keep_alive=False)
self.puppet.hire_worker()
