def send_event(topic, user, data):
# Get back the thread local storage
try:
context = thread_local.context
socket = thread_local.socket
except AttributeError:
# Create the context and socket
thread_local.context = context = zmq.Context.instance()
thread_local.socket = socket = context.socket(zmq.PUSH)
socket.connect(settings.INTERNAL_EVENT_SOCKET)
try:
# The format is [topic, uuid, datetime, username, data as json]
msg = [
b(settings.EVENT_TOPIC + topic),
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()),
b(user),
b(simplejson.dumps(data))
]
# Send the message in the non-blockng mode.
# If the consumer (lava-publisher) is not active, the message will be lost.
socket.send_multipart(msg, zmq.DONTWAIT)
except (TypeError, ValueError, zmq.ZMQError):
# The event can't be send, just skip it
print("Unable to send the zmq event %s" % (settings.EVENT_TOPIC + topic))
def main():
# Get the arguments
if len(sys.argv) != 4 and len(sys.argv) != 6:
print("%d arguments" % len(sys.argv))
print("Usage: push.py url topic num_messages [master_cert slave_cert]")
sys.exit(1)
url = sys.argv[1]
topic = sys.argv[2]
num_messages = int(sys.argv[3])
username = pwd.getpwuid(os.geteuid()).pw_name
# Create the socket
context = zmq.Context()
sock = context.socket(zmq.PUSH)
if len(sys.argv) > 4:
# Configure encryption
(server_public, _) = zmq.auth.load_certificate(sys.argv[4])
sock.curve_serverkey = server_public
(client_public, client_private) = zmq.auth.load_certificate(sys.argv[5])
sock.curve_publickey = client_public
sock.curve_secretkey = client_private
sock.connect(url)
for i in range(0, num_messages):
sock.send_multipart([b(topic),
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()),
b(username),
b(json.dumps({'id': i}))])
def test_init_iface(self):
logger = self.logger
ctx = self.context
handler = handlers.PUBHandler(self.iface)
self.assertFalse(handler.ctx is ctx)
self.sockets.append(handler.socket)
# handler.ctx.term()
handler = handlers.PUBHandler(self.iface, self.context)
self.sockets.append(handler.socket)
self.assertTrue(handler.ctx is ctx)
handler.setLevel(logging.DEBUG)
handler.root_topic = self.topic
logger.addHandler(handler)
sub = ctx.socket(zmq.SUB)
self.sockets.append(sub)
sub.setsockopt(zmq.SUBSCRIBE, b(self.topic))
sub.connect(self.iface)
import time; time.sleep(0.25)
msg1 = 'message'
logger.info(msg1)
(topic, msg2) = sub.recv_multipart()
self.assertEquals(topic, b'zmq.INFO')
self.assertEquals(msg2, b(msg1)+b'\n')
logger.removeHandler(handler)
def configure_curve(self, domain='*', location=''):
try:
domain = b(domain, self.encoding)
location = b(location, self.encoding)
self.pipe.send([b'CURVE', domain, location])
except Exception as e:
log.exception('configure-curve', e=e)
def test_frame_more(self):
"""test Frame.more attribute"""
frame = zmq.Frame(b("hello"))
self.assertFalse(frame.more)
sa,sb = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
sa.send_multipart([b('hi'), b('there')])
frame = self.recv(sb, copy=False)
self.assertTrue(frame.more)
frame = self.recv(sb, copy=False)
self.assertFalse(frame.more)
def test_unicode_message(self):
logger, handler, sub = self.connect_handler()
base_topic = b(self.topic + '.INFO')
for msg, expected in [
(u('hello'), [base_topic, b('hello\n')]),
(u('héllo'), [base_topic, b('héllo\n')]),
(u('tøpic::héllo'), [base_topic + b('.tøpic'), b('héllo\n')]),
]:
logger.info(msg)
received = sub.recv_multipart()
self.assertEqual(received, expected)
def configure_curve(self, domain='*', location=''):
'''
Configure CURVE authentication for a given domain. CURVE authentication
uses a directory that holds all public client certificates, i.e. their
public keys. The certificates must be in zcert_save () format.
To cover all domains, use "*".
You can add and remove certificates in that directory at any time.
To allow all client keys without checking, specify CURVE_ALLOW_ANY for
the location.
'''
domain = b(domain, self.encoding)
location = b(location, self.encoding)
self.pipe.send_multipart([b'CURVE', domain, location])
def test_lifecycle1(self):
"""Run through a ref counting cycle with a copy."""
for i in range(5, 16): # 32, 64,..., 65536
s = (2**i)*x
rc = 2
self.assertEquals(grc(s), rc)
m = zmq.Frame(s)
rc += 2
self.assertEquals(grc(s), rc)
m2 = copy.copy(m)
rc += 1
self.assertEquals(grc(s), rc)
buf = m2.buffer
rc += view_rc
self.assertEquals(grc(s), rc)
self.assertEquals(s, b(str(m)))
self.assertEquals(s, bytes(m2))
self.assertEquals(s, m.bytes)
# self.assert_(s is str(m))
# self.assert_(s is str(m2))
del m2
rc -= 1
self.assertEquals(grc(s), rc)
rc -= view_rc
del buf
self.assertEquals(grc(s), rc)
del m
rc -= 2
self.assertEquals(grc(s), rc)
self.assertEquals(rc, 2)
del s
def send_multipart_u(sock, data):
""" Wrapper around send_multipart that encode data as bytes.
:param sock: The socket to use
:param data: Data to convert to byte strings
"""
return sock.send_multipart([b(d) for d in data])
def test_lifecycle2(self):
"""Run through a different ref counting cycle with a copy."""
for i in range(5, 16): # 32, 64,..., 65536
s = (2**i)*x
rc = 2
self.assertEqual(grc(s), rc)
m = zmq.Frame(s)
rc += 2
self.assertEqual(grc(s), rc)
m2 = copy.copy(m)
rc += 1
self.assertEqual(grc(s), rc)
buf = m.buffer
rc += view_rc
self.assertEqual(grc(s), rc)
self.assertEqual(s, b(str(m)))
self.assertEqual(s, bytes(m2))
self.assertEqual(s, m2.bytes)
self.assertEqual(s, m.bytes)
# self.assert_(s is str(m))
# self.assert_(s is str(m2))
del buf
self.assertEqual(grc(s), rc)
del m
# m.buffer is kept until m is del'd
rc -= view_rc
rc -= 1
self.assertEqual(grc(s), rc)
del m2
rc -= 2
await_gc(s, rc)
self.assertEqual(grc(s), rc)
self.assertEqual(rc, 2)
del s
def configure_plain(self, domain='*', passwords=None):
'''
Configure PLAIN authentication for a given domain. PLAIN authentication
uses a plain-text password file. To cover all domains, use "*".
You can modify the password file at any time; it is reloaded automatically.
'''
self.pipe.send_multipart([b'PLAIN', b(domain, self.encoding), jsonapi.dumps(passwords or {})])
def test_buffer_out(self):
"""receiving buffered output"""
ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√")
m = zmq.Frame(ins)
outb = m.buffer
self.assertTrue(isinstance(outb, view))
self.assert_(outb is m.buffer)
self.assert_(m.buffer is m.buffer)
def deny(self, address):
'''
Deny (blacklist) a single IP address. For all security mechanisms, this
rejects the connection without any further authentication. Use either a
whitelist, or a blacklist, not not both. If you define both a whitelist
and a blacklist, only the whitelist takes effect.
'''
self.pipe.send_multipart([b'DENY', b(address, self.encoding)])
def test_str(self):
"""Test the str representations of the Frames."""
for i in range(16):
s = (2**i)*x
m = zmq.Frame(s)
m_str = str(m)
m_str_b = b(m_str) # py3compat
self.assertEqual(s, m_str_b)
def allow(self, address):
'''
Allow (whitelist) a single IP address. For NULL, all clients from this
address will be accepted. For PLAIN and CURVE, they will be allowed to
continue with authentication. You can call this method multiple times
to whitelist multiple IP addresses. If you whitelist a single address,
any non-whitelisted addresses are treated as blacklisted.
'''
self.pipe.send_multipart([b'ALLOW', b(address, self.encoding)])
def notify_event(self, topic, msg):
"""Publish a message on the event publisher channel"""
name = bytestring(self.res_name)
multipart_msg = [b("watcher.%s.%s" % (name, topic)), json.dumps(msg)]
if self.evpub_socket is not None and not self.evpub_socket.closed:
self.evpub_socket.send_multipart(multipart_msg)
def test_retry_term(self):
push = self.socket(zmq.PUSH)
push.linger = self.timeout_ms
push.connect('tcp://127.0.0.1:5555')
push.send(b('ping'))
time.sleep(0.1)
self.alarm()
self.context.destroy()
assert self.timer_fired
assert self.context.closed
def test_init_socket(self):
pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB)
logger = self.logger
handler = handlers.PUBHandler(pub)
handler.setLevel(logging.DEBUG)
handler.root_topic = self.topic
logger.addHandler(handler)
self.assertTrue(handler.socket is pub)
self.assertTrue(handler.ctx is pub.context)
self.assertTrue(handler.ctx is self.context)
sub.setsockopt(zmq.SUBSCRIBE, b(self.topic))
import time; time.sleep(0.1)
msg1 = 'message'
logger.info(msg1)
(topic, msg2) = sub.recv_multipart()
self.assertEquals(topic, b'zmq.INFO')
self.assertEquals(msg2, b(msg1)+b'\n')
logger.removeHandler(handler)
def connect_handler(self, topic=None):
topic = self.topic if topic is None else topic
logger = self.logger
pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB)
handler = handlers.PUBHandler(pub)
handler.setLevel(logging.DEBUG)
handler.root_topic = topic
logger.addHandler(handler)
sub.setsockopt(zmq.SUBSCRIBE, b(topic))
time.sleep(0.1)
return logger, handler, sub
def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER,
timeout=5.0, ssh_server=None, ssh_keyfile=None):
self._init_context(context)
self.endpoint = endpoint
self._id = b(uuid.uuid4().hex)
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, self._id)
self.socket.setsockopt(zmq.LINGER, 0)
get_connection(self.socket, endpoint, ssh_server, ssh_keyfile)
self._timeout = timeout
self.timeout = timeout * 1000
self.stream = ZMQStream(self.socket, tornado.ioloop.IOLoop.instance())
def __init__(self, topics, context=None, endpoint=DEFAULT_ENDPOINT_SUB,
ssh_server=None, timeout=1.):
self.topics = topics
self.keep_context = context is not None
self._init_context(context)
self.endpoint = endpoint
self.pubsub_socket = self.context.socket(zmq.SUB)
get_connection(self.pubsub_socket, self.endpoint, ssh_server)
for topic in self.topics:
self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, b(topic))
self._init_poller()
self.timeout = timeout
def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER,
timeout=5.0, ssh_server=None, ssh_keyfile=None):
self._init_context(context)
self.endpoint = endpoint
self._id = b(uuid.uuid4().hex)
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, self._id)
self.socket.setsockopt(zmq.LINGER, 0)
get_connection(self.socket, endpoint, ssh_server, ssh_keyfile)
self._init_poller()
self._timeout = timeout
self.timeout = timeout * 1000
def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server=None,
**config):
self.daemon = True
self.config = config
self.active = to_bool(config.get('active', True))
self.context = zmq.Context()
self.pubsub_endpoint = pubsub_endpoint
self.endpoint = endpoint
self.check_delay = check_delay
self.ssh_server = ssh_server
self.loop = ioloop.IOLoop()
self._id = b(uuid.uuid4().hex)
self.running = False
def test_memoryview_shape(self):
"""memoryview shape info"""
if sys.version_info < (3,):
raise SkipTest("only test memoryviews on Python 3")
data = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√")
n = len(data)
f = zmq.Frame(data)
view1 = f.buffer
self.assertEqual(view1.ndim, 1)
self.assertEqual(view1.shape, (n,))
self.assertEqual(view1.tobytes(), data)
view2 = memoryview(f)
self.assertEqual(view2.ndim, 1)
self.assertEqual(view2.shape, (n,))
self.assertEqual(view2.tobytes(), data)
def test_sockopts(self):
"""setting socket options with ctx attributes"""
ctx = zmq.Context()
ctx.linger = 5
self.assertEquals(ctx.linger, 5)
s = ctx.socket(zmq.REQ)
self.assertEquals(s.linger, 5)
self.assertEquals(s.getsockopt(zmq.LINGER), 5)
s.close()
# check that subscribe doesn't get set on sockets that don't subscribe:
ctx.subscribe = b('')
s = ctx.socket(zmq.REQ)
s.close()
ctx.term()
def test_root_topic(self):
logger, handler, sub = self.connect_handler()
handler.socket.bind(self.iface)
sub2 = sub.context.socket(zmq.SUB)
self.sockets.append(sub2)
sub2.connect(self.iface)
sub2.setsockopt(zmq.SUBSCRIBE, b'')
handler.root_topic = b'twoonly'
msg1 = 'ignored'
logger.info(msg1)
self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK)
topic,msg2 = sub2.recv_multipart()
self.assertEquals(topic, b'twoonly.INFO')
self.assertEquals(msg2, b(msg1)+b'\n')
logger.removeHandler(handler)
def run_ctl(args, queue=None, stdin=''):
cmd = '%s -m circus.circusctl' % sys.executable
proc = subprocess.Popen(cmd.split() + shlex.split(args),
stdin=subprocess.PIPE if stdin else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(b(stdin) if stdin else None)
stdout = u(stdout)
stderr = u(stderr)
if queue:
queue.put(stderr)
queue.put(stdout)
try:
import gevent
if hasattr(gevent, 'shutdown'):
gevent.shutdown()
except ImportError:
pass
return stdout, stderr
def test_poller_events(self):
"""Tornado poller implementation maps events correctly"""
req,rep = self.create_bound_pair(zmq.REQ, zmq.REP)
poller = ioloop.ZMQPoller()
poller.register(req, ioloop.IOLoop.READ)
poller.register(rep, ioloop.IOLoop.READ)
events = dict(poller.poll(0))
self.assertEquals(events.get(rep), None)
self.assertEquals(events.get(req), None)
poller.register(req, ioloop.IOLoop.WRITE)
poller.register(rep, ioloop.IOLoop.WRITE)
events = dict(poller.poll(1))
self.assertEquals(events.get(req), ioloop.IOLoop.WRITE)
self.assertEquals(events.get(rep), None)
poller.register(rep, ioloop.IOLoop.READ)
req.send(b('hi'))
events = dict(poller.poll(1))
self.assertEquals(events.get(rep), ioloop.IOLoop.READ)
self.assertEquals(events.get(req), None)
def run(self):
self.setup()
while True:
msg = self.sub.recv_multipart()
try:
(topic, uuid, datetime, username, data) = (u(m) for m in msg)
except (IndexError, ValueError):
LOG.error("Invalid message: %s", msg)
continue
variables = {"topic": topic,
"uuid": uuid,
"datetime": datetime,
"username": username}
for (i, m) in enumerate(self.matchers):
if m.match(variables, data):
LOG.debug("%s matching %s", msg, m.name)
self.ctrl.send_multipart([b(str(i)),
b(topic),
b(uuid),
b(datetime),
b(username),
b(data)])
ports = []
sockets = []
for i in range(n):
sock = socket.socket()
sock.bind(('', 0))
ports.append(sock.getsockname()[1])
sockets.append(sock)
for sock in sockets:
sock.close()
return ports
#-----------------------------------------------------------------------------
# Check for passwordless login
#-----------------------------------------------------------------------------
_password_pat = re.compile(b(r'pass(word|phrase):'), re.IGNORECASE)
def try_passwordless_ssh(server, keyfile, paramiko=None):
"""Attempt to make an ssh connection without a password.
This is mainly used for requiring password input only once
when many tunnels may be connected to the same server.
If paramiko is None, the default for the platform is chosen.
"""
if paramiko is None:
paramiko = sys.platform == 'win32'
if not paramiko:
f = _try_passwordless_openssh
else:
f = _try_passwordless_paramiko
def wake(self):
try:
self.writer.send(b("x"))
except IOError:
pass
def allow(self, *addresses):
self.send([b'ALLOW'] + [b(a, self.encoding) for a in addresses])
def configure_curve(self, domain='*', location=''):
domain = b(domain, self.encoding)
location = b(location, self.encoding)
self.send([b'CURVE', domain, location])
def configure_plain(self, domain='*', passwords=None):
self.pipe.send_multipart([
b'PLAIN',
b(domain, self.encoding),
jsonapi.dumps(passwords or {})
])
def configure_curve(self, domain='*', location=''):
domain = b(domain, self.encoding)
location = b(location, self.encoding)
self.pipe.send_multipart([b'CURVE', domain, location])
def handle(self, *args, **options):
# Initialize logging.
self.setup_logging("lava-master", options["level"],
options["log_file"], FORMAT)
self.logger.info("[INIT] Starting lava-master")
self.logger.info("[INIT] Version %s", __version__)
self.logger.info("[INIT] Using protocol version %d", PROTOCOL_VERSION)
self.logger.info("[INIT] Dropping privileges")
if not self.drop_privileges(options["user"], options["group"]):
self.logger.error("[INIT] Unable to drop privileges")
return
filename = os.path.join(settings.MEDIA_ROOT, "lava-master-config.yaml")
self.logger.debug("[INIT] Dumping config to %s", filename)
with open(filename, "w") as output:
yaml_dump(options, output)
self.logger.info("[INIT] Marking all workers as offline")
with transaction.atomic():
for worker in Worker.objects.select_for_update().all():
worker.go_state_offline()
worker.save()
# Create the sockets
context = zmq.Context()
self.controler = context.socket(zmq.ROUTER)
self.event_socket = context.socket(zmq.SUB)
if options["ipv6"]:
self.logger.info("[INIT] Enabling IPv6")
self.controler.setsockopt(zmq.IPV6, 1)
self.event_socket.setsockopt(zmq.IPV6, 1)
encryption_settings = get_encryption_settings(options)
if encryption_settings["encrypt"]:
self.logger.info("[INIT] Starting encryption")
try:
self.auth = ThreadAuthenticator(context)
self.auth.start()
self.logger.debug(
"[INIT] Opening master certificate: %s",
encryption_settings["master_cert"],
)
master_public, master_secret = zmq.auth.load_certificate(
encryption_settings["master_cert"])
self.logger.debug(
"[INIT] Using slaves certificates from: %s",
encryption_settings["slaves_certs"],
)
self.auth.configure_curve(
domain="*", location=encryption_settings["slaves_certs"])
except OSError as err:
self.logger.error(err)
self.auth.stop()
return
self.controler.curve_publickey = master_public
self.controler.curve_secretkey = master_secret
self.controler.curve_server = True
self.logger.debug("[INIT] Watching %s",
encryption_settings["slaves_certs"])
self.inotify_fd = watch_directory(
encryption_settings["slaves_certs"])
if self.inotify_fd is None:
self.logger.error("[INIT] Unable to start inotify")
self.controler.setsockopt(zmq.IDENTITY, b"master")
# From http://api.zeromq.org/4-2:zmq-setsockopt#toc42
# "If two clients use the same identity when connecting to a ROUTER
# [...] the ROUTER socket shall hand-over the connection to the new
# client and disconnect the existing one."
self.controler.setsockopt(zmq.ROUTER_HANDOVER, 1)
self.controler.bind(options["master_socket"])
# Set the topic and connect
self.event_socket.setsockopt(zmq.SUBSCRIBE, b(settings.EVENT_TOPIC))
self.event_socket.connect(options["event_url"])
# Poll on the sockets. This allow to have a
# nice timeout along with polling.
self.poller = zmq.Poller()
self.poller.register(self.controler, zmq.POLLIN)
self.poller.register(self.event_socket, zmq.POLLIN)
if self.inotify_fd is not None:
self.poller.register(os.fdopen(self.inotify_fd), zmq.POLLIN)
# Translate signals into zmq messages
(self.pipe_r, _) = self.setup_zmq_signal_handler()
self.poller.register(self.pipe_r, zmq.POLLIN)
# Send master upgrade notifications.
send_upgraded_master_notifications(__version__, self.logger)
# Main loop
self.logger.info("[INIT] Starting main loop")
try:
self.main_loop(options)
except BaseException as exc:
self.logger.error("[CLOSE] Unknown exception raised, leaving!")
self.logger.exception(exc)
finally:
# Drop controler socket: the protocol does handle lost messages
self.logger.info(
"[CLOSE] Closing the controler socket and dropping messages")
self.controler.close(linger=0)
self.event_socket.close(linger=0)
if encryption_settings["encrypt"]:
self.auth.stop()
context.term()
def allow(self, *addresses):
self.pipe.send_multipart([b'ALLOW'] +
[b(a, self.encoding) for a in addresses])
def deny(self, *addresses):
self.pipe.send_multipart([b'DENY'] +
[b(a, self.encoding) for a in addresses])
def emit(self, record):
msg = [b(self.job_id), b(self.formatter.format(record))]
self.socket.send_multipart(msg)
def test_simple_forward(tmpdir):
conf_filename = str(tmpdir.join("conf.yaml"))
pull_url = tmpdir.join("input.socket")
push_url = tmpdir.join("output.socket")
db_url = tmpdir.join("db.sqlite3")
inbound = tmpdir.join("inbound")
outbound = tmpdir.join("outbound")
stdout = tmpdir.join("stdout")
stderr = tmpdir.join("stderr")
with open(conf_filename, "w") as f:
f.write("inputs:\n")
f.write("- class: ZMQPull\n")
f.write(" name: testing-input\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % pull_url)
f.write("db:\n")
f.write(" url: sqlite:///%s\n" % db_url)
f.write("core:\n")
f.write(" inbound: ipc://%s\n" % inbound)
f.write(" outbound: ipc://%s\n" % outbound)
f.write("outputs:\n")
f.write("- class: ZMQPush\n")
f.write(" name: testing-out\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % push_url)
args = [
"python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG",
"--log-file", "-"
]
proc = subprocess.Popen(args,
stdout=open(str(stdout), "w"),
stderr=open(str(stderr), "w"))
ctx = zmq.Context.instance()
in_sock = ctx.socket(zmq.PUSH)
in_sock.connect("ipc://%s" % pull_url)
out_sock = ctx.socket(zmq.PULL)
out_sock.bind("ipc://%s" % push_url)
# Allow the process sometime to setup and connect
time.sleep(1)
# Send many messages
for i in range(0, 1000):
data = [
b"org.reactobus.test.py",
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()), b"py.test",
b(json.dumps({'pipeline': True}))
]
in_sock.send_multipart(data)
assert out_sock.recv_multipart() == data
time.sleep(3)
proc.terminate()
proc.wait()
# Check the database
Base = declarative_base()
engine = create_engine("sqlite:///%s" % db_url)
Base.metadata.create_all(engine)
sessions = sessionmaker(bind=engine)
session = sessions()
assert session.query(Message).count() == 1000
assert session.query(Message).filter_by(topic="org.reactobus.test.py")\
.count() == 1000
def test_buffer_in(self):
"""test using a buffer as input"""
ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√")
m = zmq.Frame(view(ins))
def test_reactor(tmpdir):
conf_filename = str(tmpdir.join("conf.yaml"))
pull_url = tmpdir.join("input.socket")
inbound = tmpdir.join("inbound")
outbound = tmpdir.join("outbound")
stdout = tmpdir.join("stdout")
stderr = tmpdir.join("stderr")
script = tmpdir.join("script.sh")
script_stdin = tmpdir.join("script.stdin")
script_args = tmpdir.join("script.args")
with open(str(script), "w") as f:
f.write("#!/bin/sh\n")
f.write("cat /dev/stdin > %s\n" % script_stdin)
f.write("echo $@ > %s\n" % script_args)
os.chmod(str(script), 0o755)
with open(conf_filename, "w") as f:
f.write("inputs:\n")
f.write("- class: ZMQPull\n")
f.write(" name: testing-input\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % pull_url)
f.write("reactor:\n")
f.write(" workers: 2\n")
f.write(" rules:\n")
f.write(" - name: org.videolan.git\n")
f.write(" match:\n")
f.write(" field: topic\n")
f.write(" pattern: ^org\.videolan\.git$\n")
f.write(" exec:\n")
f.write(" path: %s\n" % script)
f.write(" timeout: 1\n")
f.write(" args:\n")
f.write(" - topic\n")
f.write(" - $topic\n")
f.write(" - data.url\n")
f.write(" - $data.url\n")
f.write(" - data\n")
f.write(" - $data\n")
f.write(" - stdin:user\n")
f.write(" - stdin:$username\n")
f.write(" - stdin:url\n")
f.write(" - stdin:$data.url\n")
f.write("core:\n")
f.write(" inbound: ipc://%s\n" % inbound)
f.write(" outbound: ipc://%s\n" % outbound)
args = [
"python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG",
"--log-file", "-"
]
proc = subprocess.Popen(args,
stdout=open(str(stdout), "w"),
stderr=open(str(stderr), "w"))
ctx = zmq.Context.instance()
in_sock = ctx.socket(zmq.PUSH)
in_sock.connect("ipc://%s" % pull_url)
# Allow the process sometime to setup and connect
time.sleep(1)
data = {"url": "https://code.videolan.org/éêï", "username": "******"}
in_sock.send_multipart([
b"org.videolan.git",
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()),
b("vidéolan-git"),
b(json.dumps(data))
])
time.sleep(1)
proc.terminate()
proc.wait()
with open(str(script_args), "r") as f:
line = f.readlines()[0]
(begin, data_recv) = line.split("{")
data_recv = json.loads("{" + data_recv)
assert data == data_recv
assert begin == "topic org.videolan.git data.url https://code.videolan.org/éêï data "
with open(str(script_stdin), "r") as f:
lines = f.readlines()
assert len(lines) == 4
assert lines[0] == "user\n"
assert lines[1] == "vidéolan-git\n"
assert lines[2] == "url\n"
assert lines[3] == "https://code.videolan.org/éêï"
def test_encryption(tmpdir):
# Create the tmp names
conf_filename = str(tmpdir.join("conf.yaml"))
pull_url = tmpdir.join("input.pull.socket")
pull_cert_dir = tmpdir.mkdir("input.pull")
pull_clients_cert_dir = pull_cert_dir.mkdir("clients")
sub_url = tmpdir.join("input.sub.socket")
sub_cert_dir = tmpdir.mkdir("input.sub")
push_url = tmpdir.join("output.push.socket")
inbound = tmpdir.join("inbound")
outbound = tmpdir.join("outbound")
stdout = tmpdir.join("stdout")
stderr = tmpdir.join("stderr")
# Create the certificates
create_certificates(str(pull_cert_dir), "pull")
create_certificates(str(pull_clients_cert_dir), "client1")
create_certificates(str(pull_clients_cert_dir), "client2")
create_certificates(str(sub_cert_dir), "sub")
create_certificates(str(sub_cert_dir), "sub-server")
with open(conf_filename, "w") as f:
f.write("inputs:\n")
f.write("- class: ZMQPull\n")
f.write(" name: in-pull\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % pull_url)
f.write(" encryption:\n")
f.write(" self: %s\n" % pull_cert_dir.join("pull.key_secret"))
f.write(" clients: %s\n" % pull_clients_cert_dir)
f.write("- class: ZMQSub\n")
f.write(" name: in-sub\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % sub_url)
f.write(" encryption:\n")
f.write(" self: %s\n" % sub_cert_dir.join("sub.key_secret"))
f.write(" server: %s\n" % sub_cert_dir.join("sub-server.key"))
f.write("core:\n")
f.write(" inbound: ipc://%s\n" % inbound)
f.write(" outbound: ipc://%s\n" % outbound)
f.write("outputs:\n")
f.write("- class: ZMQPush\n")
f.write(" name: out-push\n")
f.write(" options:\n")
f.write(" url: ipc://%s\n" % push_url)
args = [
"python3", "reactobus", "--conf", conf_filename, "--level", "DEBUG",
"--log-file", "-"
]
proc = subprocess.Popen(args,
stdout=open(str(stdout), "w"),
stderr=open(str(stderr), "w"))
# Create the input sockets
ctx = zmq.Context.instance()
in_sock = ctx.socket(zmq.PUSH)
(server_public, _) = load_certificate(str(pull_cert_dir.join("pull.key")))
in_sock.curve_serverkey = server_public
(client_public, client_private) = load_certificate(
str(pull_clients_cert_dir.join("client1.key_secret")))
in_sock.curve_publickey = client_public
in_sock.curve_secretkey = client_private
in_sock.connect("ipc://%s" % pull_url)
out_sock = ctx.socket(zmq.PULL)
out_sock.bind("ipc://%s" % push_url)
pub_sock = ctx.socket(zmq.PUB)
auth = ThreadAuthenticator(ctx)
auth.start()
auth.configure_curve(domain="*", location=str(sub_cert_dir))
(server_public, server_secret) = load_certificate(
str(sub_cert_dir.join("sub-server.key_secret")))
pub_sock.curve_publickey = server_public
pub_sock.curve_secretkey = server_secret
pub_sock.curve_server = True
pub_sock.bind("ipc://%s" % sub_url)
# Allow the process sometime to setup and connect
time.sleep(1)
# Send some data
data = [
b"org.videolan.git",
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()),
b("videolan-git"),
b(
json.dumps({
"url": "https://code.videolan.org/éêï",
"username": "******"
}))
]
in_sock.send_multipart(data)
msg = out_sock.recv_multipart()
assert msg == data
data = [
b"org.videolan.git",
b(str(uuid.uuid1())),
b(datetime.datetime.utcnow().isoformat()),
b("videolan-git"),
b(
json.dumps({
"url": "https://code.videolan.org/éêï",
"username": "******"
}))
]
pub_sock.send_multipart(data)
msg = out_sock.recv_multipart()
assert msg == data
# End the process
proc.terminate()
proc.wait()
