async def receive_file(self, receiver):
self.progress_update.start()
parent, child = AioPipe(False)
start_time = time()
receive_process = AioProcess(target=receiver.fetch_data,
args=(self.pipe[1], child))
receive_process.start()
await receive_process.coro_join()
end_time = time() - start_time
self.progress_update.wait()
message = QMessageBox()
message.information(
self, "Information",
f"Download complete, time taken {round(end_time / 60, 2)} minutes")
receiver.save_location = parent.recv()
self.ui.label_4.setText("Writing File")
self.ui.progressBar.setValue(0)
path = await receiver.write_data(receiver.save_location, self)
rmtree(path)
self.ui.receiveButton.setEnabled(True)
self.ui.label_4.setVisible(False)
self.ui.progressBar.setVisible(False)
self.ui.label_4.setText("Download in Progress")
self.ui.progressBar.setValue(0)
def __init__(self, name, connection_type, socket_type, url):
super().__init__()
self.input, self.child_input = AioPipe()
self.output, self.child_output = AioPipe()
self.name = name
assert connection_type == BIND or connection_type == CONNECT, \
'Invalid connection type provided.'
self.connection_type = connection_type
self.socket_type = socket_type
self.url = url
self.context = None
self.socket = None
def __init__(self,
base_url=Constants.BaseNode.BaseUrl,
sub_port=7777,
pub_port=9998):
super().__init__()
print(sub_port)
self.parent_pipe, self.child_pipe = AioPipe()
# establish base url
self.base_url = base_url
# setup subscriber constants
self.subscriber_port = sub_port
self.subscriber_url = 'tcp://{}:{}'.format(self.base_url,
self.subscriber_port)
# setup publisher constants
self.publisher_port = pub_port
self.publisher_url = 'tcp://{}:{}'.format(self.base_url,
self.publisher_port)
# set context and sockets to none until process starts because multiprocessing zmq is funky
self.context = None
self.sub_socket = None
self.pub_socket = None
def zmq_sender(socket_type, connection_type, url):
def listen(*args):
context = zmq.Context()
socket = context.socket(socket_type=args[0])
socket.bind(args[2]) if args[1] == BIND else \
socket.connect(args[2])
zmq_loop = asyncio.new_event_loop()
asyncio.set_event_loop(zmq_loop)
zmq_loop.run_until_complete(loop(socket))
async def loop(socket):
sub_loop = asyncio.get_event_loop()
await asyncio.wait([sub_loop.run_in_executor(None, receive, socket)])
def receive(socket):
while True:
socket.send(_pipe.recv())
pipe, _pipe = AioPipe()
process = Process(target=listen,
args=(
socket_type,
connection_type,
url,
))
return pipe, process
def __init__(self):
super().__init__()
self.ui = ui.receive_files.Ui_Dialog()
self.ui.setupUi(self)
self.ui.label_4.setVisible(False)
self.ui.progressBar.setVisible(False)
self.pipe = AioPipe(False)
self.progress_update = ProgressThread(self.pipe[0])
self.progress_update.progress_val.connect(self.update_progress_bar)
self.ui.receiveButton.clicked.connect(self.receive_files)
self.ui.toolButton.clicked.connect(self.open_file_dialog)
def __init__(self):
super().__init__()
self.ui = ui.send_files.Ui_Dialog()
self.ui.setupUi(self)
self.ui.labelProgress.setVisible(False)
self.ui.progressBar.setVisible(False)
self.pipe = AioPipe(False)
self.progress_update = ProgressThread(self.pipe[0])
self.progress_update.progress_val.connect(self.update_progress_bar)
self.ui.genIP.clicked.connect(self.generate_server_ip)
self.ui.sendButton.clicked.connect(self.send_files)
self.ui.toolButton.clicked.connect(self.open_file_dialog)
def zmq_two_ways(socket_type, connection_type, url):
def listen(*args):
p = args[-1]
context = zmq.Context()
socket = context.socket(socket_type=args[0])
socket.bind(args[2]) if args[1] == BIND else \
socket.connect(args[2])
if args[0] == zmq.SUB:
socket.setsockopt(zmq.SUBSCRIBE, b'')
log.debug("{} on {}".format(args[1], args[2]))
zmq_loop = asyncio.new_event_loop()
asyncio.set_event_loop(zmq_loop)
zmq_loop.run_until_complete(loop(socket, p))
async def loop(socket, p):
sub_loop = asyncio.get_event_loop()
await asyncio.wait([
sub_loop.run_in_executor(None, receive, socket),
sub_loop.run_in_executor(None, send, p, socket)
])
def receive(socket):
# log = get_logger('ZMQ_LISTENER')
log.debug("Setting up RECV on ZMQ_LISTENER")
while True:
log.debug("Waiting for a message on {}".format(url))
_pipe.send(socket.recv())
log.debug("Got a message on {}".format(url))
def send(p, socket):
while True:
socket.send(p.recv())
log = get_logger('ZMQ_LISTENER:{}'.format(url[-4:]))
pipe, _pipe = AioPipe()
process = Process(target=listen,
args=(socket_type, connection_type, url, _pipe))
return pipe, process
def pipe_listener():
def listen():
pl_loop = asyncio.new_event_loop()
asyncio.set_event_loop(pl_loop)
pl_loop.run_until_complete(loop())
async def loop():
sub_loop = asyncio.get_event_loop()
await asyncio.wait([sub_loop.run_in_executor(None, receive)])
def receive():
while True:
pipe.send(_pipe.recv())
pipe, _pipe = AioPipe()
process = Process(target=listen)
return pipe, process
def test_basic_q_with_pipe_async():
import asyncio
from aioprocessing import AioPipe
class Node(Process):
def __init__(self, pipe=None):
super().__init__()
# assert mp structures are in place
assert pipe is not None, 'A pipe must be provided.'
# set queue
self.pipe = pipe
def run(self, *args):
super().run()
self.loop()
def loop(self):
print(
'node entering loop where it will non-block check zmq and local queue'
)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
tasks = [
asyncio. async (self.get_from_pipe(self.pipe)),
]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
async def get_from_pipe(self, pipe):
while True:
pipe.send(pipe.recv())
p_parent_pipe, p_child_pipe = AioPipe()
p = Node(pipe=p_child_pipe)
p.start()
sleep(1)
p_parent_pipe.send(b'hello world')
sleep(1)
print('q is empty...' if not p_parent_pipe.poll() else
'q is not empty. got: {}'.format(p_parent_pipe.recv()))
p.terminate()
async def watch_subscriptions(ctx: Context):
logger.info('Watching for reddit submissions')
known_subreddits = set(await Subscription.all_subreddits())
recv, send = AioPipe(duplex=False)
if known_subreddits:
process = _start_watching(ctx, known_subreddits, send)
else:
process = None
asyncio.get_event_loop().create_task(_pipe_to_queue(ctx, recv))
async for op, (channel,
subreddit) in queue_to_async_gen(ctx.subscription_changes):
logger.info('Received subscription change',
op=op,
channel=channel,
subreddit=subreddit)
# if we're already watching or there are more channels watching the
# subreddit, don't do anything
if (op == 'added' and subreddit in known_subreddits or op == 'removed'
and 1 < await Subscription.for_subreddit(subreddit).count()):
continue
if op == 'removed':
known_subreddits.remove(subreddit)
elif op == 'added':
known_subreddits.add(subreddit)
logger.info('Updating subreddit stream process')
if process:
process.terminate()
if known_subreddits:
process = _start_watching(ctx, known_subreddits, send)
else:
process = None
def test_basic_q_and_zmq_with_pipe_async():
import zmq
from cilantro import Constants
import asyncio
from aioprocessing import AioPipe
class Node(Process):
def __init__(self, pipe=None, sub_port=9999, pub_port=9998):
super().__init__()
# assert mp structures are in place
assert pipe is not None, 'A pipe must be provided.'
# establish base url
self.base_url = Constants.BaseNode.BaseUrl
# setup subscriber constants
self.subscriber_port = sub_port
self.subscriber_url = 'tcp://{}:{}'.format(self.base_url,
self.subscriber_port)
# setup publisher constants
self.publisher_port = pub_port
self.publisher_url = 'tcp://{}:{}'.format(self.base_url,
self.publisher_port)
# set context and sockets to none until process starts because multiprocessing zmq is funky
self.context = None
self.sub_socket = None
self.pub_socket = None
# set pipe
self.pipe = pipe
def run(self, *args):
super().run()
self.context = zmq.Context()
self.sub_socket = self.context.socket(socket_type=zmq.SUB)
self.sub_socket.bind(self.subscriber_url)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, b'')
self.pub_socket = self.context.socket(socket_type=zmq.PUB)
self.pub_socket.connect(self.publisher_url)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self.listen())
async def listen(self):
loop = asyncio.get_event_loop()
tasks = [
loop.run_in_executor(None, self.receive_from_pipe, self.pipe),
loop.run_in_executor(None, self.receive_from_socket,
self.sub_socket)
]
await asyncio.wait(tasks)
def receive_from_socket(self, socket):
print('zmq')
while True:
self.zmq_callback(socket.recv())
def receive_from_pipe(self, pipe):
print('pipe')
while True:
self.pipe_callback(pipe.recv())
def zmq_callback(self, msg):
raise NotImplementedError
def pipe_callback(self, msg):
raise NotImplementedError
class Subscriber(Node):
def zmq_callback(self, msg):
print('received message from publisher. putting it on the queue:',
msg)
self.pipe.send(msg)
print('done')
def pipe_callback(self, msg):
pass
class Publisher(Node):
def zmq_callback(self, msg):
pass
def pipe_callback(self, msg):
print('received a message on the queue. publishing it:', msg)
self.pub_socket.send(msg)
s_parent_pipe, s_child_pipe = AioPipe()
s = Subscriber(pipe=s_child_pipe)
s.start()
sleep(1)
p_parent_pipe, p_child_pipe = AioPipe()
p = Publisher(pipe=p_child_pipe, pub_port=9999, sub_port=9997)
p.start()
sleep(1)
print('publishing message')
p_parent_pipe.send(b'hello world')
sleep(1)
print('q is empty...' if not s_parent_pipe.poll() else
'q is not empty. got: {}'.format(s_parent_pipe.recv()))
p.terminate()
s.terminate()
def __init__(self):
super().__init__()
self.test_pipe_parent, self.test_pipe_child = AioPipe()
FACE_DESCR_PATH = "../resources/cascades/haarcascade_frontalface_default.xml"
EYE_DESCR_PATH = "../resources/cascades/haarcascade_eye.xml"
# Start camera videostream
# WARNING: VideoStream doesn't originally accept a "name" parameter. See
# the README in the misc directory or simply remove the "name" kwarg for
# it to work normally
videostream = VideoStream(usePiCamera=True, name="CameraThread").start()
time.sleep(0.1)
# Set the thread's name
prctl.set_name("Cascade(Main)")
# Initialise conection objects and processes
parentConn1, childConn1 = AioPipe()
parentConn2, childConn2 = AioPipe()
faceDetector = FaceDetector(conn=childConn1,
width=FRAME_WIDTH,
descriptor=FACE_DESCR_PATH,
name="Face",
log=args.log,
display=args.display)
eyeDetector = EyeDetector(conn=childConn2,
width=FRAME_WIDTH,
descriptor=EYE_DESCR_PATH,
name="Eye",
log=args.log,
display=args.display,