def send_event(topic, user, data): # Get back the thread local storage try: context = thread_local.context socket = thread_local.socket except AttributeError: # Create the context and socket thread_local.context = context = zmq.Context.instance() thread_local.socket = socket = context.socket(zmq.PUSH) socket.connect(settings.INTERNAL_EVENT_SOCKET) try: # The format is [topic, uuid, datetime, username, data as json] msg = [ b(settings.EVENT_TOPIC + topic), b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b(user), b(simplejson.dumps(data)) ] # Send the message in the non-blockng mode. # If the consumer (lava-publisher) is not active, the message will be lost. socket.send_multipart(msg, zmq.DONTWAIT) except (TypeError, ValueError, zmq.ZMQError): # The event can't be send, just skip it print("Unable to send the zmq event %s" % (settings.EVENT_TOPIC + topic))
def main(): # Get the arguments if len(sys.argv) != 4 and len(sys.argv) != 6: print("%d arguments" % len(sys.argv)) print("Usage: push.py url topic num_messages [master_cert slave_cert]") sys.exit(1) url = sys.argv[1] topic = sys.argv[2] num_messages = int(sys.argv[3]) username = pwd.getpwuid(os.geteuid()).pw_name # Create the socket context = zmq.Context() sock = context.socket(zmq.PUSH) if len(sys.argv) > 4: # Configure encryption (server_public, _) = zmq.auth.load_certificate(sys.argv[4]) sock.curve_serverkey = server_public (client_public, client_private) = zmq.auth.load_certificate(sys.argv[5]) sock.curve_publickey = client_public sock.curve_secretkey = client_private sock.connect(url) for i in range(0, num_messages): sock.send_multipart([b(topic), b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b(username), b(json.dumps({'id': i}))])
def test_init_iface(self): logger = self.logger ctx = self.context handler = handlers.PUBHandler(self.iface) self.assertFalse(handler.ctx is ctx) self.sockets.append(handler.socket) # handler.ctx.term() handler = handlers.PUBHandler(self.iface, self.context) self.sockets.append(handler.socket) self.assertTrue(handler.ctx is ctx) handler.setLevel(logging.DEBUG) handler.root_topic = self.topic logger.addHandler(handler) sub = ctx.socket(zmq.SUB) self.sockets.append(sub) sub.setsockopt(zmq.SUBSCRIBE, b(self.topic)) sub.connect(self.iface) import time; time.sleep(0.25) msg1 = 'message' logger.info(msg1) (topic, msg2) = sub.recv_multipart() self.assertEquals(topic, b'zmq.INFO') self.assertEquals(msg2, b(msg1)+b'\n') logger.removeHandler(handler)
def configure_curve(self, domain='*', location=''): try: domain = b(domain, self.encoding) location = b(location, self.encoding) self.pipe.send([b'CURVE', domain, location]) except Exception as e: log.exception('configure-curve', e=e)
def test_frame_more(self): """test Frame.more attribute""" frame = zmq.Frame(b("hello")) self.assertFalse(frame.more) sa,sb = self.create_bound_pair(zmq.PAIR, zmq.PAIR) sa.send_multipart([b('hi'), b('there')]) frame = self.recv(sb, copy=False) self.assertTrue(frame.more) frame = self.recv(sb, copy=False) self.assertFalse(frame.more)
def test_unicode_message(self): logger, handler, sub = self.connect_handler() base_topic = b(self.topic + '.INFO') for msg, expected in [ (u('hello'), [base_topic, b('hello\n')]), (u('héllo'), [base_topic, b('héllo\n')]), (u('tøpic::héllo'), [base_topic + b('.tøpic'), b('héllo\n')]), ]: logger.info(msg) received = sub.recv_multipart() self.assertEqual(received, expected)
def configure_curve(self, domain='*', location=''): ''' Configure CURVE authentication for a given domain. CURVE authentication uses a directory that holds all public client certificates, i.e. their public keys. The certificates must be in zcert_save () format. To cover all domains, use "*". You can add and remove certificates in that directory at any time. To allow all client keys without checking, specify CURVE_ALLOW_ANY for the location. ''' domain = b(domain, self.encoding) location = b(location, self.encoding) self.pipe.send_multipart([b'CURVE', domain, location])
def test_lifecycle1(self): """Run through a ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i)*x rc = 2 self.assertEquals(grc(s), rc) m = zmq.Frame(s) rc += 2 self.assertEquals(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEquals(grc(s), rc) buf = m2.buffer rc += view_rc self.assertEquals(grc(s), rc) self.assertEquals(s, b(str(m))) self.assertEquals(s, bytes(m2)) self.assertEquals(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del m2 rc -= 1 self.assertEquals(grc(s), rc) rc -= view_rc del buf self.assertEquals(grc(s), rc) del m rc -= 2 self.assertEquals(grc(s), rc) self.assertEquals(rc, 2) del s
def send_multipart_u(sock, data): """ Wrapper around send_multipart that encode data as bytes. :param sock: The socket to use :param data: Data to convert to byte strings """ return sock.send_multipart([b(d) for d in data])
def test_lifecycle2(self): """Run through a different ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i)*x rc = 2 self.assertEqual(grc(s), rc) m = zmq.Frame(s) rc += 2 self.assertEqual(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEqual(grc(s), rc) buf = m.buffer rc += view_rc self.assertEqual(grc(s), rc) self.assertEqual(s, b(str(m))) self.assertEqual(s, bytes(m2)) self.assertEqual(s, m2.bytes) self.assertEqual(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del buf self.assertEqual(grc(s), rc) del m # m.buffer is kept until m is del'd rc -= view_rc rc -= 1 self.assertEqual(grc(s), rc) del m2 rc -= 2 await_gc(s, rc) self.assertEqual(grc(s), rc) self.assertEqual(rc, 2) del s
def configure_plain(self, domain='*', passwords=None): ''' Configure PLAIN authentication for a given domain. PLAIN authentication uses a plain-text password file. To cover all domains, use "*". You can modify the password file at any time; it is reloaded automatically. ''' self.pipe.send_multipart([b'PLAIN', b(domain, self.encoding), jsonapi.dumps(passwords or {})])
def test_buffer_out(self): """receiving buffered output""" ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√") m = zmq.Frame(ins) outb = m.buffer self.assertTrue(isinstance(outb, view)) self.assert_(outb is m.buffer) self.assert_(m.buffer is m.buffer)
def deny(self, address): ''' Deny (blacklist) a single IP address. For all security mechanisms, this rejects the connection without any further authentication. Use either a whitelist, or a blacklist, not not both. If you define both a whitelist and a blacklist, only the whitelist takes effect. ''' self.pipe.send_multipart([b'DENY', b(address, self.encoding)])
def test_str(self): """Test the str representations of the Frames.""" for i in range(16): s = (2**i)*x m = zmq.Frame(s) m_str = str(m) m_str_b = b(m_str) # py3compat self.assertEqual(s, m_str_b)
def allow(self, address): ''' Allow (whitelist) a single IP address. For NULL, all clients from this address will be accepted. For PLAIN and CURVE, they will be allowed to continue with authentication. You can call this method multiple times to whitelist multiple IP addresses. If you whitelist a single address, any non-whitelisted addresses are treated as blacklisted. ''' self.pipe.send_multipart([b'ALLOW', b(address, self.encoding)])
def notify_event(self, topic, msg): """Publish a message on the event publisher channel""" name = bytestring(self.res_name) multipart_msg = [b("watcher.%s.%s" % (name, topic)), json.dumps(msg)] if self.evpub_socket is not None and not self.evpub_socket.closed: self.evpub_socket.send_multipart(multipart_msg)
def test_retry_term(self): push = self.socket(zmq.PUSH) push.linger = self.timeout_ms push.connect('tcp://127.0.0.1:5555') push.send(b('ping')) time.sleep(0.1) self.alarm() self.context.destroy() assert self.timer_fired assert self.context.closed
def test_init_socket(self): pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB) logger = self.logger handler = handlers.PUBHandler(pub) handler.setLevel(logging.DEBUG) handler.root_topic = self.topic logger.addHandler(handler) self.assertTrue(handler.socket is pub) self.assertTrue(handler.ctx is pub.context) self.assertTrue(handler.ctx is self.context) sub.setsockopt(zmq.SUBSCRIBE, b(self.topic)) import time; time.sleep(0.1) msg1 = 'message' logger.info(msg1) (topic, msg2) = sub.recv_multipart() self.assertEquals(topic, b'zmq.INFO') self.assertEquals(msg2, b(msg1)+b'\n') logger.removeHandler(handler)
def connect_handler(self, topic=None): topic = self.topic if topic is None else topic logger = self.logger pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB) handler = handlers.PUBHandler(pub) handler.setLevel(logging.DEBUG) handler.root_topic = topic logger.addHandler(handler) sub.setsockopt(zmq.SUBSCRIBE, b(topic)) time.sleep(0.1) return logger, handler, sub
def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint self._id = b(uuid.uuid4().hex) self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) get_connection(self.socket, endpoint, ssh_server, ssh_keyfile) self._timeout = timeout self.timeout = timeout * 1000 self.stream = ZMQStream(self.socket, tornado.ioloop.IOLoop.instance())
def __init__(self, topics, context=None, endpoint=DEFAULT_ENDPOINT_SUB, ssh_server=None, timeout=1.): self.topics = topics self.keep_context = context is not None self._init_context(context) self.endpoint = endpoint self.pubsub_socket = self.context.socket(zmq.SUB) get_connection(self.pubsub_socket, self.endpoint, ssh_server) for topic in self.topics: self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, b(topic)) self._init_poller() self.timeout = timeout
def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint self._id = b(uuid.uuid4().hex) self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) get_connection(self.socket, endpoint, ssh_server, ssh_keyfile) self._init_poller() self._timeout = timeout self.timeout = timeout * 1000
def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server=None, **config): self.daemon = True self.config = config self.active = to_bool(config.get('active', True)) self.context = zmq.Context() self.pubsub_endpoint = pubsub_endpoint self.endpoint = endpoint self.check_delay = check_delay self.ssh_server = ssh_server self.loop = ioloop.IOLoop() self._id = b(uuid.uuid4().hex) self.running = False
def test_memoryview_shape(self): """memoryview shape info""" if sys.version_info < (3,): raise SkipTest("only test memoryviews on Python 3") data = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√") n = len(data) f = zmq.Frame(data) view1 = f.buffer self.assertEqual(view1.ndim, 1) self.assertEqual(view1.shape, (n,)) self.assertEqual(view1.tobytes(), data) view2 = memoryview(f) self.assertEqual(view2.ndim, 1) self.assertEqual(view2.shape, (n,)) self.assertEqual(view2.tobytes(), data)
def test_sockopts(self): """setting socket options with ctx attributes""" ctx = zmq.Context() ctx.linger = 5 self.assertEquals(ctx.linger, 5) s = ctx.socket(zmq.REQ) self.assertEquals(s.linger, 5) self.assertEquals(s.getsockopt(zmq.LINGER), 5) s.close() # check that subscribe doesn't get set on sockets that don't subscribe: ctx.subscribe = b('') s = ctx.socket(zmq.REQ) s.close() ctx.term()
def test_root_topic(self): logger, handler, sub = self.connect_handler() handler.socket.bind(self.iface) sub2 = sub.context.socket(zmq.SUB) self.sockets.append(sub2) sub2.connect(self.iface) sub2.setsockopt(zmq.SUBSCRIBE, b'') handler.root_topic = b'twoonly' msg1 = 'ignored' logger.info(msg1) self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK) topic,msg2 = sub2.recv_multipart() self.assertEquals(topic, b'twoonly.INFO') self.assertEquals(msg2, b(msg1)+b'\n') logger.removeHandler(handler)
def run_ctl(args, queue=None, stdin=''): cmd = '%s -m circus.circusctl' % sys.executable proc = subprocess.Popen(cmd.split() + shlex.split(args), stdin=subprocess.PIPE if stdin else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate(b(stdin) if stdin else None) stdout = u(stdout) stderr = u(stderr) if queue: queue.put(stderr) queue.put(stdout) try: import gevent if hasattr(gevent, 'shutdown'): gevent.shutdown() except ImportError: pass return stdout, stderr
def test_poller_events(self): """Tornado poller implementation maps events correctly""" req,rep = self.create_bound_pair(zmq.REQ, zmq.REP) poller = ioloop.ZMQPoller() poller.register(req, ioloop.IOLoop.READ) poller.register(rep, ioloop.IOLoop.READ) events = dict(poller.poll(0)) self.assertEquals(events.get(rep), None) self.assertEquals(events.get(req), None) poller.register(req, ioloop.IOLoop.WRITE) poller.register(rep, ioloop.IOLoop.WRITE) events = dict(poller.poll(1)) self.assertEquals(events.get(req), ioloop.IOLoop.WRITE) self.assertEquals(events.get(rep), None) poller.register(rep, ioloop.IOLoop.READ) req.send(b('hi')) events = dict(poller.poll(1)) self.assertEquals(events.get(rep), ioloop.IOLoop.READ) self.assertEquals(events.get(req), None)
def run(self): self.setup() while True: msg = self.sub.recv_multipart() try: (topic, uuid, datetime, username, data) = (u(m) for m in msg) except (IndexError, ValueError): LOG.error("Invalid message: %s", msg) continue variables = {"topic": topic, "uuid": uuid, "datetime": datetime, "username": username} for (i, m) in enumerate(self.matchers): if m.match(variables, data): LOG.debug("%s matching %s", msg, m.name) self.ctrl.send_multipart([b(str(i)), b(topic), b(uuid), b(datetime), b(username), b(data)])
ports = [] sockets = [] for i in range(n): sock = socket.socket() sock.bind(('', 0)) ports.append(sock.getsockname()[1]) sockets.append(sock) for sock in sockets: sock.close() return ports #----------------------------------------------------------------------------- # Check for passwordless login #----------------------------------------------------------------------------- _password_pat = re.compile(b(r'pass(word|phrase):'), re.IGNORECASE) def try_passwordless_ssh(server, keyfile, paramiko=None): """Attempt to make an ssh connection without a password. This is mainly used for requiring password input only once when many tunnels may be connected to the same server. If paramiko is None, the default for the platform is chosen. """ if paramiko is None: paramiko = sys.platform == 'win32' if not paramiko: f = _try_passwordless_openssh else: f = _try_passwordless_paramiko
def wake(self): try: self.writer.send(b("x")) except IOError: pass
def allow(self, *addresses): self.send([b'ALLOW'] + [b(a, self.encoding) for a in addresses])
def configure_curve(self, domain='*', location=''): domain = b(domain, self.encoding) location = b(location, self.encoding) self.send([b'CURVE', domain, location])
def configure_plain(self, domain='*', passwords=None): self.pipe.send_multipart([ b'PLAIN', b(domain, self.encoding), jsonapi.dumps(passwords or {}) ])
def configure_curve(self, domain='*', location=''): domain = b(domain, self.encoding) location = b(location, self.encoding) self.pipe.send_multipart([b'CURVE', domain, location])
def handle(self, *args, **options): # Initialize logging. self.setup_logging("lava-master", options["level"], options["log_file"], FORMAT) self.logger.info("[INIT] Starting lava-master") self.logger.info("[INIT] Version %s", __version__) self.logger.info("[INIT] Using protocol version %d", PROTOCOL_VERSION) self.logger.info("[INIT] Dropping privileges") if not self.drop_privileges(options["user"], options["group"]): self.logger.error("[INIT] Unable to drop privileges") return filename = os.path.join(settings.MEDIA_ROOT, "lava-master-config.yaml") self.logger.debug("[INIT] Dumping config to %s", filename) with open(filename, "w") as output: yaml_dump(options, output) self.logger.info("[INIT] Marking all workers as offline") with transaction.atomic(): for worker in Worker.objects.select_for_update().all(): worker.go_state_offline() worker.save() # Create the sockets context = zmq.Context() self.controler = context.socket(zmq.ROUTER) self.event_socket = context.socket(zmq.SUB) if options["ipv6"]: self.logger.info("[INIT] Enabling IPv6") self.controler.setsockopt(zmq.IPV6, 1) self.event_socket.setsockopt(zmq.IPV6, 1) encryption_settings = get_encryption_settings(options) if encryption_settings["encrypt"]: self.logger.info("[INIT] Starting encryption") try: self.auth = ThreadAuthenticator(context) self.auth.start() self.logger.debug( "[INIT] Opening master certificate: %s", encryption_settings["master_cert"], ) master_public, master_secret = zmq.auth.load_certificate( encryption_settings["master_cert"]) self.logger.debug( "[INIT] Using slaves certificates from: %s", encryption_settings["slaves_certs"], ) self.auth.configure_curve( domain="*", location=encryption_settings["slaves_certs"]) except OSError as err: self.logger.error(err) self.auth.stop() return self.controler.curve_publickey = master_public self.controler.curve_secretkey = master_secret self.controler.curve_server = True self.logger.debug("[INIT] Watching %s", encryption_settings["slaves_certs"]) self.inotify_fd = watch_directory( encryption_settings["slaves_certs"]) if self.inotify_fd is None: self.logger.error("[INIT] Unable to start inotify") self.controler.setsockopt(zmq.IDENTITY, b"master") # From http://api.zeromq.org/4-2:zmq-setsockopt#toc42 # "If two clients use the same identity when connecting to a ROUTER # [...] the ROUTER socket shall hand-over the connection to the new # client and disconnect the existing one." self.controler.setsockopt(zmq.ROUTER_HANDOVER, 1) self.controler.bind(options["master_socket"]) # Set the topic and connect self.event_socket.setsockopt(zmq.SUBSCRIBE, b(settings.EVENT_TOPIC)) self.event_socket.connect(options["event_url"]) # Poll on the sockets. This allow to have a # nice timeout along with polling. self.poller = zmq.Poller() self.poller.register(self.controler, zmq.POLLIN) self.poller.register(self.event_socket, zmq.POLLIN) if self.inotify_fd is not None: self.poller.register(os.fdopen(self.inotify_fd), zmq.POLLIN) # Translate signals into zmq messages (self.pipe_r, _) = self.setup_zmq_signal_handler() self.poller.register(self.pipe_r, zmq.POLLIN) # Send master upgrade notifications. send_upgraded_master_notifications(__version__, self.logger) # Main loop self.logger.info("[INIT] Starting main loop") try: self.main_loop(options) except BaseException as exc: self.logger.error("[CLOSE] Unknown exception raised, leaving!") self.logger.exception(exc) finally: # Drop controler socket: the protocol does handle lost messages self.logger.info( "[CLOSE] Closing the controler socket and dropping messages") self.controler.close(linger=0) self.event_socket.close(linger=0) if encryption_settings["encrypt"]: self.auth.stop() context.term()
def allow(self, *addresses): self.pipe.send_multipart([b'ALLOW'] + [b(a, self.encoding) for a in addresses])
def deny(self, *addresses): self.pipe.send_multipart([b'DENY'] + [b(a, self.encoding) for a in addresses])
def emit(self, record): msg = [b(self.job_id), b(self.formatter.format(record))] self.socket.send_multipart(msg)
def test_simple_forward(tmpdir): conf_filename = str(tmpdir.join("conf.yaml")) pull_url = tmpdir.join("input.socket") push_url = tmpdir.join("output.socket") db_url = tmpdir.join("db.sqlite3") inbound = tmpdir.join("inbound") outbound = tmpdir.join("outbound") stdout = tmpdir.join("stdout") stderr = tmpdir.join("stderr") with open(conf_filename, "w") as f: f.write("inputs:\n") f.write("- class: ZMQPull\n") f.write(" name: testing-input\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % pull_url) f.write("db:\n") f.write(" url: sqlite:///%s\n" % db_url) f.write("core:\n") f.write(" inbound: ipc://%s\n" % inbound) f.write(" outbound: ipc://%s\n" % outbound) f.write("outputs:\n") f.write("- class: ZMQPush\n") f.write(" name: testing-out\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % push_url) args = [ "python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG", "--log-file", "-" ] proc = subprocess.Popen(args, stdout=open(str(stdout), "w"), stderr=open(str(stderr), "w")) ctx = zmq.Context.instance() in_sock = ctx.socket(zmq.PUSH) in_sock.connect("ipc://%s" % pull_url) out_sock = ctx.socket(zmq.PULL) out_sock.bind("ipc://%s" % push_url) # Allow the process sometime to setup and connect time.sleep(1) # Send many messages for i in range(0, 1000): data = [ b"org.reactobus.test.py", b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b"py.test", b(json.dumps({'pipeline': True})) ] in_sock.send_multipart(data) assert out_sock.recv_multipart() == data time.sleep(3) proc.terminate() proc.wait() # Check the database Base = declarative_base() engine = create_engine("sqlite:///%s" % db_url) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine) session = sessions() assert session.query(Message).count() == 1000 assert session.query(Message).filter_by(topic="org.reactobus.test.py")\ .count() == 1000
def test_buffer_in(self): """test using a buffer as input""" ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√") m = zmq.Frame(view(ins))
def test_reactor(tmpdir): conf_filename = str(tmpdir.join("conf.yaml")) pull_url = tmpdir.join("input.socket") inbound = tmpdir.join("inbound") outbound = tmpdir.join("outbound") stdout = tmpdir.join("stdout") stderr = tmpdir.join("stderr") script = tmpdir.join("script.sh") script_stdin = tmpdir.join("script.stdin") script_args = tmpdir.join("script.args") with open(str(script), "w") as f: f.write("#!/bin/sh\n") f.write("cat /dev/stdin > %s\n" % script_stdin) f.write("echo $@ > %s\n" % script_args) os.chmod(str(script), 0o755) with open(conf_filename, "w") as f: f.write("inputs:\n") f.write("- class: ZMQPull\n") f.write(" name: testing-input\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % pull_url) f.write("reactor:\n") f.write(" workers: 2\n") f.write(" rules:\n") f.write(" - name: org.videolan.git\n") f.write(" match:\n") f.write(" field: topic\n") f.write(" pattern: ^org\.videolan\.git$\n") f.write(" exec:\n") f.write(" path: %s\n" % script) f.write(" timeout: 1\n") f.write(" args:\n") f.write(" - topic\n") f.write(" - $topic\n") f.write(" - data.url\n") f.write(" - $data.url\n") f.write(" - data\n") f.write(" - $data\n") f.write(" - stdin:user\n") f.write(" - stdin:$username\n") f.write(" - stdin:url\n") f.write(" - stdin:$data.url\n") f.write("core:\n") f.write(" inbound: ipc://%s\n" % inbound) f.write(" outbound: ipc://%s\n" % outbound) args = [ "python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG", "--log-file", "-" ] proc = subprocess.Popen(args, stdout=open(str(stdout), "w"), stderr=open(str(stderr), "w")) ctx = zmq.Context.instance() in_sock = ctx.socket(zmq.PUSH) in_sock.connect("ipc://%s" % pull_url) # Allow the process sometime to setup and connect time.sleep(1) data = {"url": "https://code.videolan.org/éêï", "username": "******"} in_sock.send_multipart([ b"org.videolan.git", b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b("vidéolan-git"), b(json.dumps(data)) ]) time.sleep(1) proc.terminate() proc.wait() with open(str(script_args), "r") as f: line = f.readlines()[0] (begin, data_recv) = line.split("{") data_recv = json.loads("{" + data_recv) assert data == data_recv assert begin == "topic org.videolan.git data.url https://code.videolan.org/éêï data " with open(str(script_stdin), "r") as f: lines = f.readlines() assert len(lines) == 4 assert lines[0] == "user\n" assert lines[1] == "vidéolan-git\n" assert lines[2] == "url\n" assert lines[3] == "https://code.videolan.org/éêï"
def test_encryption(tmpdir): # Create the tmp names conf_filename = str(tmpdir.join("conf.yaml")) pull_url = tmpdir.join("input.pull.socket") pull_cert_dir = tmpdir.mkdir("input.pull") pull_clients_cert_dir = pull_cert_dir.mkdir("clients") sub_url = tmpdir.join("input.sub.socket") sub_cert_dir = tmpdir.mkdir("input.sub") push_url = tmpdir.join("output.push.socket") inbound = tmpdir.join("inbound") outbound = tmpdir.join("outbound") stdout = tmpdir.join("stdout") stderr = tmpdir.join("stderr") # Create the certificates create_certificates(str(pull_cert_dir), "pull") create_certificates(str(pull_clients_cert_dir), "client1") create_certificates(str(pull_clients_cert_dir), "client2") create_certificates(str(sub_cert_dir), "sub") create_certificates(str(sub_cert_dir), "sub-server") with open(conf_filename, "w") as f: f.write("inputs:\n") f.write("- class: ZMQPull\n") f.write(" name: in-pull\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % pull_url) f.write(" encryption:\n") f.write(" self: %s\n" % pull_cert_dir.join("pull.key_secret")) f.write(" clients: %s\n" % pull_clients_cert_dir) f.write("- class: ZMQSub\n") f.write(" name: in-sub\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % sub_url) f.write(" encryption:\n") f.write(" self: %s\n" % sub_cert_dir.join("sub.key_secret")) f.write(" server: %s\n" % sub_cert_dir.join("sub-server.key")) f.write("core:\n") f.write(" inbound: ipc://%s\n" % inbound) f.write(" outbound: ipc://%s\n" % outbound) f.write("outputs:\n") f.write("- class: ZMQPush\n") f.write(" name: out-push\n") f.write(" options:\n") f.write(" url: ipc://%s\n" % push_url) args = [ "python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG", "--log-file", "-" ] proc = subprocess.Popen(args, stdout=open(str(stdout), "w"), stderr=open(str(stderr), "w")) # Create the input sockets ctx = zmq.Context.instance() in_sock = ctx.socket(zmq.PUSH) (server_public, _) = load_certificate(str(pull_cert_dir.join("pull.key"))) in_sock.curve_serverkey = server_public (client_public, client_private) = load_certificate( str(pull_clients_cert_dir.join("client1.key_secret"))) in_sock.curve_publickey = client_public in_sock.curve_secretkey = client_private in_sock.connect("ipc://%s" % pull_url) out_sock = ctx.socket(zmq.PULL) out_sock.bind("ipc://%s" % push_url) pub_sock = ctx.socket(zmq.PUB) auth = ThreadAuthenticator(ctx) auth.start() auth.configure_curve(domain="*", location=str(sub_cert_dir)) (server_public, server_secret) = load_certificate( str(sub_cert_dir.join("sub-server.key_secret"))) pub_sock.curve_publickey = server_public pub_sock.curve_secretkey = server_secret pub_sock.curve_server = True pub_sock.bind("ipc://%s" % sub_url) # Allow the process sometime to setup and connect time.sleep(1) # Send some data data = [ b"org.videolan.git", b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b("videolan-git"), b( json.dumps({ "url": "https://code.videolan.org/éêï", "username": "******" })) ] in_sock.send_multipart(data) msg = out_sock.recv_multipart() assert msg == data data = [ b"org.videolan.git", b(str(uuid.uuid1())), b(datetime.datetime.utcnow().isoformat()), b("videolan-git"), b( json.dumps({ "url": "https://code.videolan.org/éêï", "username": "******" })) ] pub_sock.send_multipart(data) msg = out_sock.recv_multipart() assert msg == data # End the process proc.terminate() proc.wait()