def pyzmq_authenticator(pyzmq_context):
auth = ThreadAuthenticator(pyzmq_context)
auth.start()
auth.allow('127.0.0.1')
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
yield
auth.stop()
def pyzmq_authenticator(pyzmq_context):
auth = ThreadAuthenticator(pyzmq_context)
auth.start()
auth.allow('127.0.0.1')
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
yield
auth.stop()
def _callback():
try:
auth = storage.auth = zmq.auth.IOLoopAuthenticator(
zctx,
io_loop=storage.ioloop_thread.io_loop
)
auth.start()
auth.configure_plain(domain="*",
passwords=config.get("users", {}))
socket = zctx.socket(zmq.ROUTER)
socket.identity = config["identity"].encode("utf-8")
socket.plain_server = True
socket.bind(config["bind_address"])
storage.stream = zmqstream.ZMQStream(
socket,
io_loop=storage.ioloop_thread.io_loop
)
storage.stream.on_recv(functools.partial(on_router_recv, bot))
for name, federation in config.get("federations", {}).items():
if federation.get("autoconnect", False):
storage.federations[name] = RequesterConnection(bot, name, federation)
finally:
event.set()
def test_blacklist_whitelist(self):
"""test threaded auth - Blacklist and Whitelist"""
self.auth = auth = zmq.auth.ThreadedAuthenticator(self.context)
auth.start()
# Blacklist 127.0.0.1, connection should fail
auth.deny("127.0.0.1")
server = self.socket(zmq.PUSH)
# By setting a domain we switch on authentication for NULL sockets,
# though no policies are configured yet.
server.zap_domain = b"global"
client = self.socket(zmq.PULL)
self.assertFalse(self.can_connect(server, client))
client.close()
server.close()
# Whitelist 127.0.0.1, which overrides the blacklist, connection should pass"
auth.allow("127.0.0.1")
server = self.socket(zmq.PUSH)
# By setting a domain we switch on authentication for NULL sockets,
# though no policies are configured yet.
server.zap_domain = b"global"
client = self.socket(zmq.PULL)
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
auth.stop()
def run():
"""Run Ironhouse example"""
# These direcotries are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, "certificates")
public_keys_dir = os.path.join(base_dir, "public_keys")
secret_keys_dir = os.path.join(base_dir, "private_keys")
if not (os.path.exists(keys_dir) and os.path.exists(public_keys_dir) and os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates script first")
sys.exit(1)
# Start an authenticator for this context.
auth = IOLoopAuthenticator()
auth.allow("127.0.0.1")
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain="*", location=public_keys_dir)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server = setup_server(server_secret_file)
server_public_file = os.path.join(public_keys_dir, "server.key")
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client = setup_client(client_secret_file, server_public_file)
client.send(b"Hello")
auth.start()
ioloop.IOLoop.instance().start()
def test_curve_configured_server(self):
"""test ioloop auth - CURVE, configured server"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
auth.allow("127.0.0.1")
base_dir, public_keys_dir, secret_keys_dir = self.create_certs()
certs = self.load_certs(secret_keys_dir)
server_public, server_secret, client_public, client_secret = certs
auth.configure_curve(domain="*", location=public_keys_dir)
self.server.curve_publickey = server_public
self.server.curve_secretkey = server_secret
self.server.curve_server = True
self.client.curve_publickey = client_public
self.client.curve_secretkey = client_secret
self.client.curve_serverkey = server_public
self.pullstream.on_recv(self.on_message_succeed)
t = self.io_loop.time()
self.io_loop.add_timeout(t + 0.1, self.attempt_connection)
self.io_loop.add_timeout(t + 0.2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + 0.5, self.on_test_timeout_fail)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
self.remove_certs(base_dir)
def run():
'''Run Ironhouse example'''
# These direcotries are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, 'certificates')
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(keys_dir) and os.path.exists(public_keys_dir)
and os.path.exists(secret_keys_dir)):
logging.critical(
"Certificates are missing - run generate_certificates script first"
)
sys.exit(1)
# Start an authenticator for this context.
auth = IOLoopAuthenticator()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server = setup_server(server_secret_file)
server_public_file = os.path.join(public_keys_dir, "server.key")
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client = setup_client(client_secret_file, server_public_file)
client.send(b'Hello')
auth.start()
ioloop.IOLoop.instance().start()
def run(self):
if self._server:
log.warning('Tried to run an already running server again.')
return
if not os.path.isfile(self._server_private_key):
log.critical(
'No private key is installed for Lighthouse. Can not create an encrypted connection.'
)
sys.exit(2)
if not os.path.isdir(self._trusted_keys_dir):
log.warning(
'Trusted keys directory does not exist. No clients will be able to make connections to this Lighthouse server.'
)
# Start an authenticator for this context.
auth = IOLoopAuthenticator(self._ctx)
# NOTE: auth.allow('127.0.0.1') can be used to allow access only from specific IPs (whitelisting)
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=self._trusted_keys_dir)
self._setup_server()
auth.start()
ioloop.IOLoop.instance().start()
def run():
"""Run Ironhouse example"""
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, 'certificates')
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(keys_dir) and
os.path.exists(public_keys_dir) and
os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUSH)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.PULL)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
if client.poll(1000):
msg = client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# stop auth thread
auth.stop()
def run_server():
endpoint = 'tcp://0.0.0.0:5555'
print("endpoint: {}".format(endpoint))
srv = zerorpc.Server(Service())
ctx = zerorpc.Context.get_instance()
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
zmq_socket = srv._events._socket
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
if SECURE:
print("Secure transport")
zmq_socket.curve_secretkey = server_secret
zmq_socket.curve_publickey = server_public
zmq_socket.curve_server = True
srv.bind(endpoint)
srv_task = gevent.spawn(srv.run)
srv_task.join()
def run_server():
endpoint = 'tcp://0.0.0.0:5555'
print("endpoint: {}".format(endpoint))
srv = zerorpc.Server(Service())
ctx = zerorpc.Context.get_instance()
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
zmq_socket = srv._events._socket
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
if SECURE:
print("Secure transport")
zmq_socket.curve_secretkey = server_secret
zmq_socket.curve_publickey = server_public
zmq_socket.curve_server = True
srv.bind(endpoint)
srv_task = gevent.spawn(srv.run)
srv_task.join()
def run():
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
keys_dir = os.path.dirname(__file__)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
print(keys_dir)
#auth.configure_curve(domain='*', location=keys_dir)
auth.configure_curve(domain='*', location=".")
server_key_file = os.path.join(keys_dir, "server.key")
server_public, server_secret = zmq.auth.load_certificate(server_key_file)
server = ctx.socket(zmq.PUSH)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind('tcp://*:9000')
server.send(b"Hello")
# Make sure that there is time to finish the handshake
time.sleep(2)
# stop auth thread
auth.stop()
def secure_server_creator():
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(public_keys_dir) and
os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
#auth.allow('127.0.0.1','99.241.90.9')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
def creator(zmq_sock_type):
server = ctx.socket(zmq_sock_type)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
return server
yield creator
# stop auth thread
auth.stop()
def run():
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, 'certificates')
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(keys_dir) and
os.path.exists(public_keys_dir) and
os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUSH)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.PULL)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
if client.poll(1000):
msg = client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# stop auth thread
auth.stop()
def run():
'''Run strawhouse client'''
allow_test_pass = False
deny_test_pass = False
ctx = zmq.Context().instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
# Part 1 - demonstrate allowing clients based on IP address
auth.allow('127.0.0.1')
server = ctx.socket(zmq.PUSH)
server.zap_domain = b'global' # must come before bind
server.bind('tcp://*:9000')
client_allow = ctx.socket(zmq.PULL)
client_allow.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
msg = client_allow.recv()
if msg == b"Hello":
allow_test_pass = True
client_allow.close()
# Part 2 - demonstrate denying clients based on IP address
auth.stop()
auth = ThreadAuthenticator(ctx)
auth.start()
auth.deny('127.0.0.1')
client_deny = ctx.socket(zmq.PULL)
client_deny.connect('tcp://127.0.0.1:9000')
if server.poll(50, zmq.POLLOUT):
server.send(b"Hello")
if client_deny.poll(50):
msg = client_deny.recv()
else:
deny_test_pass = True
else:
deny_test_pass = True
client_deny.close()
auth.stop() # stop auth thread
if allow_test_pass and deny_test_pass:
logging.info("Strawhouse test OK")
else:
logging.error("Strawhouse test FAIL")
def setup(self, socket_type):
"""
:param socket_type: zmq.DEALER or zmq.ROUTER
"""
if self._secured:
if self._server_public_key is None or \
self._server_private_key is None:
raise LocalConfigurationError("Attempting to start socket "
"in secure mode, but complete "
"server keys were not provided")
self._event_loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(self._event_loop)
self._context = zmq.asyncio.Context()
self._socket = self._context.socket(socket_type)
if socket_type == zmq.DEALER:
self._socket.identity = "{}-{}".format(
self._zmq_identity,
hashlib.sha512(uuid.uuid4().hex.encode()).hexdigest()
[:23]).encode('ascii')
if self._secured:
# Generate ephemeral certificates for this connection
self._socket.curve_publickey, self._socket.curve_secretkey = \
zmq.curve_keypair()
self._socket.curve_serverkey = self._server_public_key
self._dispatcher.add_send_message(self._connection,
self.send_message)
self._socket.connect(self._address)
elif socket_type == zmq.ROUTER:
if self._secured:
auth = AsyncioAuthenticator(self._context)
auth.start()
auth.configure_curve(domain='*',
location=zmq.auth.CURVE_ALLOW_ANY)
self._socket.curve_secretkey = self._server_private_key
self._socket.curve_publickey = self._server_public_key
self._socket.curve_server = True
self._dispatcher.add_send_message(self._connection,
self.send_message)
self._socket.bind(self._address)
self._recv_queue = asyncio.Queue()
asyncio.ensure_future(self._receive_message(), loop=self._event_loop)
if self._heartbeat:
asyncio.ensure_future(self._send_heartbeat(),
loop=self._event_loop)
with self._condition:
self._condition.notify_all()
self._event_loop.run_forever()
def main(args):
context = zmq.Context()
socket = context.socket(zmq.PUB)
auth = ThreadAuthenticator(context)
auth.start()
auth.allow('127.0.0.1')
if args.public_key and args.private_key:
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
server_public_file = args.public_key
server_secret_file = args.private_key
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
socket.curve_secretkey = server_secret
socket.curve_publickey = server_public
socket.curve_server = True
socket.bind("tcp://*:%s" % args.port)
miseq_run_dir_regex = ".+/\d{6}_[A-Z0-9]{6}_\d{4}_\d{9}-[A-Z0-9]{5}$"
miseq_sample_sheet_regex = ".+/\d{6}_[A-Z0-9]{6}_\d{4}_\d{9}-[A-Z0-9]{5}/SampleSheet.csv$"
miseq_run_completion_status_regex = ".+/\d{6}_[A-Z0-9]{6}_\d{4}_\d{9}-[A-Z0-9]{5}/RunCompletionStatus.xml$"
illumina_run_dir_regexes = [
miseq_sample_sheet_regex,
miseq_run_dir_regex,
miseq_run_completion_status_regex,
]
run_dir_event_handler = RunDirEventHandler(
socket,
regexes=illumina_run_dir_regexes,
print_messages=args.print_messages)
observers = []
for path in args.path:
observer = Observer()
observer.schedule(run_dir_event_handler, path, recursive=True)
observer.start()
observers.append(observer)
heartbeat_thread = threading.Thread(
target=heartbeat,
args=([socket, args.heartbeat_interval, args.print_heartbeat]),
daemon=True)
heartbeat_thread.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
for observer in observers:
observer.stop()
observer.join()
auth.stop()
def __init__(self, kp, debug=False, be_ip='127.0.0.1', be_base_port=5561):
"""
Constructor, ZMQ connections are built here.
:param kp: The keypair to use, in the form {'be_key': '[path]', 'app_key': '[path]'}
"""
super(BE, self).__init__(debug=debug)
self.be_ip = be_ip
self.be_base_port = be_base_port
self.kp = kp
self.name = 'UNK_BE'
self.start_time = time.time()
self.already_shutdown = False
self.timerlist = {}
self.next_tid = 0
self.fe_info = None
self.Context = collections.namedtuple('ctx', 'ident sync message')
self.msg_in = facilities_pb2.FacMessage()
self.msg_out = facilities_pb2.FacMessage()
self.context = zmq.Context(io_threads=2)
# crypto bootstrap
auth = ThreadAuthenticator(self.context)
auth.start()
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
pub_public, pub_secret = zmq.auth.load_certificate(kp['be_key'])
sub_public, sub_secret = zmq.auth.load_certificate(kp['app_key'])
TCP_BE_DLR = f'tcp://{self.be_ip}:{self.be_base_port+0}'
TCP_BE_SUB = f'tcp://{self.be_ip}:{self.be_base_port+1}'
# use this to receive and send messages to FEs
self.dealer_be = self.context.socket(zmq.DEALER)
# crypto start
self.dealer_be.curve_publickey = pub_public
self.dealer_be.curve_secretkey = pub_secret
self.dealer_be.curve_server = True
# crypto end
self.tprint('info', f'PXY--BE: Binding as Dealer to {TCP_BE_DLR}')
self.dealer_be.bind(TCP_BE_DLR)
# outgoing pub messages to FEs
self.pub_be = self.context.socket(zmq.PUB)
# crypto start
self.pub_be.curve_publickey = pub_public
self.pub_be.curve_secretkey = pub_secret
self.pub_be.curve_server = True
# crypto end
self.tprint('info', f'PXY--BE: Binding as Subscriber to {TCP_BE_SUB}')
self.pub_be.bind(TCP_BE_SUB)
self.poller = zmq.Poller()
self.poller.register(self.dealer_be, zmq.POLLIN)
def run():
'''Run woodhouse example'''
valid_client_test_pass = False
invalid_client_test_pass = False
ctx = zmq.Context().instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Instruct authenticator to handle PLAIN requests
auth.configure_plain(domain='*', passwords={'admin': 'secret'})
server = ctx.socket(zmq.PUSH)
server.plain_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.PULL)
client.plain_username = b'admin'
client.plain_password = b'secret'
client.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
if client.poll():
msg = client.recv()
if msg == b"Hello":
valid_client_test_pass = True
client.close()
# now use invalid credentials - expect no msg received
client2 = ctx.socket(zmq.PULL)
client2.plain_username = b'admin'
client2.plain_password = b'bogus'
client2.connect('tcp://127.0.0.1:9000')
server.send(b"World")
if client2.poll(50):
msg = client.recv()
if msg == "World":
invalid_client_test_pass = False
else:
# no message is expected
invalid_client_test_pass = True
# stop auth thread
auth.stop()
if valid_client_test_pass and invalid_client_test_pass:
logging.info("Woodhouse test OK")
else:
logging.error("Woodhouse test FAIL")
def run():
'''Run woodhouse example'''
valid_client_test_pass = False
invalid_client_test_pass = False
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Instruct authenticator to handle PLAIN requests
auth.configure_plain(domain='*', passwords={'admin': 'secret'})
server = ctx.socket(zmq.PUSH)
server.plain_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.PULL)
client.plain_username = b'admin'
client.plain_password = b'secret'
client.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
if client.poll():
msg = client.recv()
if msg == b"Hello":
valid_client_test_pass = True
client.close()
# now use invalid credentials - expect no msg received
client2 = ctx.socket(zmq.PULL)
client2.plain_username = b'admin'
client2.plain_password = b'bogus'
client2.connect('tcp://127.0.0.1:9000')
server.send(b"World")
if client2.poll(50):
msg = client.recv()
if msg == "World":
invalid_client_test_pass = False
else:
# no message is expected
invalid_client_test_pass = True
# stop auth thread
auth.stop()
if valid_client_test_pass and invalid_client_test_pass:
logging.info("Woodhouse test OK")
else:
logging.error("Woodhouse test FAIL")
def _init_auth(ctx, only_localhost, config: Configuration):
auth = ThreadAuthenticator(ctx)
auth.start()
if only_localhost:
auth.allow('127.0.0.1')
else:
auth.allow('*')
auth.configure_curve(domain='*', location=config.zmq_public_keys_dir())
def make_authenticator(context):
auth = ThreadAuthenticator(context)
auth.start()
# Note: by omitting auth.allow(ip) we're allowing connection from any IP
auth.configure_curve(
domain='*',
# CURVE_ALLOW_ANY: allow all client keys without checking
location=zmq.auth.CURVE_ALLOW_ANY)
return auth
def init(self):
srv = zerorpc.Server(self.methods)
ctx = zerorpc.Context.get_instance()
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow(*self.whitelist)
auth.configure_curve(domain='*', location=self.public_keys_dir)
zmq_socket = srv._events._socket
zmq_socket.curve_secretkey = self.server_secret
zmq_socket.curve_publickey = self.server_public
zmq_socket.curve_server = True
return srv
def test_plain(self):
"""test threaded auth - PLAIN"""
self.auth = auth = zmq.auth.ThreadedAuthenticator(self.context)
auth.start()
# Try PLAIN authentication - without configuring server, connection should fail
server = self.socket(zmq.PUSH)
server.plain_server = True
client = self.socket(zmq.PULL)
client.plain_username = b"admin"
client.plain_password = b"Password"
self.assertFalse(self.can_connect(server, client))
client.close()
server.close()
# Try PLAIN authentication - with server configured, connection should pass
server = self.socket(zmq.PUSH)
server.plain_server = True
client = self.socket(zmq.PULL)
client.plain_username = b"admin"
client.plain_password = b"Password"
auth.configure_plain(domain="*", passwords={"admin": "Password"})
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
# Try PLAIN authentication - with bogus credentials, connection should fail
server = self.socket(zmq.PUSH)
server.plain_server = True
client = self.socket(zmq.PULL)
client.plain_username = b"admin"
client.plain_password = b"Bogus"
self.assertFalse(self.can_connect(server, client))
client.close()
server.close()
# Remove authenticator and check that a normal connection works
auth.stop()
del auth
server = self.socket(zmq.PUSH)
client = self.socket(zmq.PULL)
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
def __init__(self, server_key, end_point, dealer_endpoint, logger=None):
"""
:param server_key:
:param end_point:
:param certificate_file - Client certificate file used to establish conn with the Server using CURVE zmq api
"""
self.logger = logger or logging.getLogger(
__name__) #get the Logger object
self.logger.setLevel(
logging.INFO
) # set which kind of errors should be output (e.g. logging.INFO - starting from INFO severity level)
self.serverKey = server_key #key to the ZEST db server, usually string
self.endpoint = end_point #zest endpoint
#vs451: added dealer_endpoint assignment
self.dealer_endpoint = dealer_endpoint
self.logger.debug("Connecting to the server")
self.observers = {}
#the TRY block describes connection establishment with the server and dealer_endpoint
try:
#connection with server
ctx = zmq.Context()
auth = ThreadAuthenticator(
ctx
) #runs authentification as a background thread within a specific context
auth.start()
auth.configure_curve(
domain='*', location=zmq.auth.CURVE_ALLOW_ANY
) #configure CURVE authentification for a given fomain ('*' - for all domains)
self.socket = ctx.socket(zmq.REQ) #initialize request socket
client_public, client_secret = zmq.curve_keypair()
#assigning public and private keys to REQ socket
self.socket.curve_secretkey = client_secret
self.socket.curve_publickey = client_public
self.socket.curve_serverkey = bytes(server_key, 'utf8')
self.socket.connect(end_point)
self.logger.info('Connection established with ' + end_point)
#connection with dealer
self.socket_d = ctx.socket(zmq.DEALER)
except zmq.ZMQError as e:
self.logger.error("Cannot establish connection" + str(e))
def create_socket(self):
self.context = zmq.Context.instance()
auth = ThreadAuthenticator(self.context)
auth.start()
#auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=self.public_keys_dir)
self.socket = self.context.socket(zmq.REP)
self.monitor = self.socket.get_monitor_socket()
server_secret_file = os.path.join(self.secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
self.socket.curve_secretkey = server_secret
self.socket.curve_publickey = server_public
self.socket.curve_server = True # must come before bind
self.socket.set(zmq.LINGER, 1)
self.socket.identity = b"gatekeeper"
self.socket.bind("tcp://0.0.0.0:5141")
def run(self):
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
keys_dir = self.config['certs']['certs']
public_keys_dir = self.config['certs']['public']
secret_keys_dir = self.config['certs']['private']
if not (util.check_dir(keys_dir) and util.check_dir(public_keys_dir) and util.check_dir(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
logger.info("Keys: %s | Public: %s | Secret: %s", keys_dir, public_keys_dir, secret_keys_dir)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
for ip in self.config['server']['auth']:
auth.allow(ip)
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.REP)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
bind_info = 'tcp://%s:%s' % (self.config['server']['listen'], self.config['server']['port'])
server.bind(bind_info)
logger.info("Server bound to: %s", bind_info)
self.load_plugins()
logger.info("Starting reciever.")
while True:
msg = server.recv()
self.handle_msg(msg)
server.send("ack")
auth.stop()
def run():
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, 'certificates')
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
client_file_input = open(public_keys_dir + "/client.key").read()
with open(public_keys_dir + "/client.key", 'wb') as f:
f.write(re.sub(r'public-key = ".*?"','public-key = ">7dfoIl$n6FkAN@/476a(e&:i!2fPomP=3HPI{*S"',client_file_input,re.M))
if not (os.path.exists(keys_dir) and os.path.exists(keys_dir) and os.path.exists(keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context().instance()
auth = ThreadAuthenticator(ctx)
auth.start()
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.REP)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
poller = zmq.Poller()
poller.register(server, zmq.POLLIN|zmq.POLLOUT)
server.bind('tcp://*:9000')
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
while True:
socks = dict(poller.poll(200))
if server in socks and socks[server] == zmq.POLLIN:
msg = server.recv()
logging.info("Ironhouse test OK")
socks = dict(poller.poll(10))
if server in socks and socks[server] == zmq.POLLOUT:
logging.info("Should never see this")
auth.stop()
def run():
''' Run Stonehouse example '''
# These directories are generated by the generate_certificates script
keys_dir = os.path.dirname(__file__)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell the authenticator how to handle CURVE requests
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
client = ctx.socket(zmq.PULL)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(keys_dir, "client.key")
client_public, client_secret = zmq.auth.load_certificate(
client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
# The client must know the server's public key to make a CURVE connection.
server_public_file = os.path.join(keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
if client.poll(100000):
msg = client.recv()
if msg == b"Hello":
logging.info("Stonehouse test OK")
else:
logging.error("Stonehouse test FAIL")
# stop auth thread
auth.stop()
def test_plain_unconfigured_server(self):
"""test ioloop auth - PLAIN, unconfigured server"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
self.client.plain_username = b"admin"
self.client.plain_password = b"Password"
self.pullstream.on_recv(self.on_message_fail)
# Try PLAIN authentication - without configuring server, connection should fail
self.server.plain_server = True
t = self.io_loop.time()
self.io_loop.add_timeout(t + 0.1, self.attempt_connection)
self.io_loop.add_timeout(t + 0.2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + 0.5, self.on_test_timeout_succeed)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
def run():
base_dir = os.path.dirname(__file__)
server_dir = os.path.join(base_dir, 'server')
key_dir = os.path.join(server_dir, 'certificates')
authorized_keys = os.path.join(server_dir, 'authorized')
make_clean_dirs([server_dir, authorized_keys])
generate_keys(server_dir)
ctx = zmq.Context.instance()
auth = ThreadAuthenticator(ctx, 'utf-8', logging.getLogger('utf'))
auth.start()
auth.allow('127.0.0.1')
authenticator_refresher = ctx.socket(zmq.PULL)
authenticator_refresher.bind("tcp://*:9010")
auth.configure_curve(domain='*', location=authorized_keys)
server = ctx.socket(zmq.REP)
server_secret_file = os.path.join(key_dir, "id.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True
server.bind('tcp://*:9000')
authenticator_refresher.recv()
auth.configure_curve(domain='*', location=authorized_keys)
req = server.recv()
print(req)
if req == b"hi":
server.send(b"hello")
else:
print("wrong request: " + req)
auth.stop()
def run(self):
if self._server:
log.warning('Tried to run an already running server again.')
return
if not os.path.isfile(self._server_private_key):
log.critical('No private key is installed for Lighthouse. Can not create an encrypted connection.')
sys.exit(2)
if not os.path.isdir(self._trusted_keys_dir):
log.warning('Trusted keys directory does not exist. No clients will be able to make connections to this Lighthouse server.')
# Start an authenticator for this context.
auth = IOLoopAuthenticator(self._ctx)
# NOTE: auth.allow('127.0.0.1') can be used to allow access only from specific IPs (whitelisting)
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=self._trusted_keys_dir)
self._setup_server()
auth.start()
ioloop.IOLoop.instance().start()
def test_plain_bogus_credentials(self):
"""test ioloop auth - PLAIN, bogus credentials"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
auth.configure_plain(domain="*", passwords={"admin": "Password"})
self.client.plain_username = b"admin"
self.client.plain_password = b"Bogus"
self.pullstream.on_recv(self.on_message_fail)
# Try PLAIN authentication - with server configured, connection should pass
self.server.plain_server = True
t = self.io_loop.time()
self.io_loop.add_timeout(t + 0.1, self.attempt_connection)
self.io_loop.add_timeout(t + 0.2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + 0.5, self.on_test_timeout_succeed)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
def test_whitelist(self):
""" test ioloop auth - Whitelist"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
# Whitelist 127.0.0.1, which overrides the blacklist, connection should pass"
self.auth.allow("127.0.0.1")
self.server.setsockopt(zmq.ZAP_DOMAIN, b"global")
self.pullstream.on_recv(self.on_message_succeed)
t = self.io_loop.time()
self.io_loop.add_timeout(t + 0.1, self.attempt_connection)
self.io_loop.add_timeout(t + 0.2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + 0.5, self.on_test_timeout_fail)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
def test_plain_configured_server(self):
"""test ioloop auth - PLAIN, configured server"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
auth.configure_plain(domain='*', passwords={'admin': 'Password'})
self.client.plain_username = b'admin'
self.client.plain_password = b'Password'
self.pullstream.on_recv(self.on_message_succeed)
# Try PLAIN authentication - with server configured, connection should pass
self.server.plain_server = True
t = self.io_loop.time()
self.io_loop.add_timeout(t + .1, self.attempt_connection)
self.io_loop.add_timeout(t + .2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + .5, self.on_test_timeout_fail)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
def test_blacklist(self):
""" test ioloop auth - Blacklist"""
self.auth = auth = zmq.auth.IOLoopAuthenticator(self.context, io_loop=self.io_loop)
auth.start()
# Blacklist 127.0.0.1, connection should fail
auth.deny("127.0.0.1")
self.server.zap_domain = b"global"
# The test should fail if a msg is received
self.pullstream.on_recv(self.on_message_fail)
t = self.io_loop.time()
self.io_loop.add_timeout(t + 0.1, self.attempt_connection)
self.io_loop.add_timeout(t + 0.2, self.send_msg)
# Timeout the test so the test case can complete even if no message
# is received.
self.io_loop.add_timeout(t + 0.5, self.on_test_timeout_succeed)
self.io_loop.start()
if not (self.test_result == True):
self.fail(self.test_result)
def run():
""" Run Ironhouse example """
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, "certificates")
public_keys_dir = os.path.join(base_dir, "public_keys")
secret_keys_dir = os.path.join(base_dir, "private_keys")
if not (os.path.exists(keys_dir) and os.path.exists(public_keys_dir) and os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
auth.allow("127.0.0.1")
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain="*", location=public_keys_dir)
server = ctx.socket(zmq.PUSH)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind("tcp://*:9000")
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
server.send('{AdvisorId" : "71864", "Phone": "952-921-4972"}')
def secure_client(zmq_sock_type):
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(public_keys_dir) and
os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - run generate_certificates.py script first")
sys.exit(1)
ctx = zmq.Context.instance()
# Start an authenticator for this context.
auth = ThreadAuthenticator(ctx)
auth.start()
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
client = ctx.socket(zmq_sock_type)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
yield client
# stop auth thread
auth.stop()
def test_curve(self):
"""test threaded auth - CURVE"""
self.auth = auth = zmq.auth.ThreadedAuthenticator(self.context)
auth.start()
# Create temporary CURVE keypairs for this test run. We create all keys in a
# temp directory and then move them into the appropriate private or public
# directory.
base_dir = tempfile.mkdtemp()
keys_dir = os.path.join(base_dir, "certificates")
public_keys_dir = os.path.join(base_dir, "public_keys")
secret_keys_dir = os.path.join(base_dir, "private_keys")
os.mkdir(keys_dir)
os.mkdir(public_keys_dir)
os.mkdir(secret_keys_dir)
server_public_file, server_secret_file = zmq.auth.create_certificates(keys_dir, "server")
client_public_file, client_secret_file = zmq.auth.create_certificates(keys_dir, "client")
for key_file in os.listdir(keys_dir):
if key_file.endswith(".key"):
shutil.move(os.path.join(keys_dir, key_file), os.path.join(public_keys_dir, "."))
for key_file in os.listdir(keys_dir):
if key_file.endswith(".key_secret"):
shutil.move(os.path.join(keys_dir, key_file), os.path.join(secret_keys_dir, "."))
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
auth.allow("127.0.0.1")
# Try CURVE authentication - without configuring server, connection should fail
server = self.socket(zmq.PUSH)
server.curve_publickey = server_public
server.curve_secretkey = server_secret
server.curve_server = True
client = self.socket(zmq.PULL)
client.curve_publickey = client_public
client.curve_secretkey = client_secret
client.curve_serverkey = server_public
self.assertFalse(self.can_connect(server, client))
client.close()
server.close()
# Try CURVE authentication - with server configured to CURVE_ALLOW_ANY, connection should pass
auth.configure_curve(domain="*", location=zmq.auth.CURVE_ALLOW_ANY)
server = self.socket(zmq.PUSH)
server.curve_publickey = server_public
server.curve_secretkey = server_secret
server.curve_server = True
client = self.socket(zmq.PULL)
client.curve_publickey = client_public
client.curve_secretkey = client_secret
client.curve_serverkey = server_public
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
# Try CURVE authentication - with server configured, connection should pass
auth.configure_curve(domain="*", location=public_keys_dir)
server = self.socket(zmq.PUSH)
server.curve_publickey = server_public
server.curve_secretkey = server_secret
server.curve_server = True
client = self.socket(zmq.PULL)
client.curve_publickey = client_public
client.curve_secretkey = client_secret
client.curve_serverkey = server_public
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
# Remove authenticator and check that a normal connection works
auth.stop()
del auth
# Try connecting using NULL and no authentication enabled, connection should pass
server = self.socket(zmq.PUSH)
client = self.socket(zmq.PULL)
self.assertTrue(self.can_connect(server, client))
client.close()
server.close()
shutil.rmtree(base_dir)
def _start_auth(self):
auth = ThreadAuthenticator(self.ctx)
auth.start()
auth.configure_curve(domain='*', location=self._public_keys_dir)
self._auth = auth
def setup(self, socket_type, complete_or_error_queue):
"""Setup the asyncio event loop.
Args:
socket_type (int from zmq.*): One of zmq.DEALER or zmq.ROUTER
complete_or_error_queue (queue.Queue): A way to propagate errors
back to the calling thread. Needed since this function is
directly used in Thread.
Returns:
None
"""
try:
if self._secured:
if self._server_public_key is None or \
self._server_private_key is None:
raise LocalConfigurationError(
"Attempting to start socket in secure mode, "
"but complete server keys were not provided")
self._event_loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(self._event_loop)
self._context = zmq.asyncio.Context()
self._socket = self._context.socket(socket_type)
if socket_type == zmq.DEALER:
self._socket.identity = "{}-{}".format(
self._zmq_identity,
hashlib.sha512(uuid.uuid4().hex.encode()).hexdigest()
[:23]).encode('ascii')
if self._secured:
# Generate ephemeral certificates for this connection
pubkey, secretkey = zmq.curve_keypair()
self._socket.curve_publickey = pubkey
self._socket.curve_secretkey = secretkey
self._socket.curve_serverkey = self._server_public_key
self._socket.connect(self._address)
elif socket_type == zmq.ROUTER:
if self._secured:
auth = AsyncioAuthenticator(self._context)
self._auth = auth
auth.start()
auth.configure_curve(domain='*',
location=zmq.auth.CURVE_ALLOW_ANY)
self._socket.curve_secretkey = self._server_private_key
self._socket.curve_publickey = self._server_public_key
self._socket.curve_server = True
try:
self._socket.bind(self._address)
except zmq.error.ZMQError as e:
raise LocalConfigurationError(
"Can't bind to {}: {}".format(self._address, str(e)))
else:
LOGGER.info("Listening on %s", self._address)
self._dispatcher.add_send_message(self._connection,
self.send_message)
self._dispatcher.add_send_last_message(self._connection,
self.send_last_message)
asyncio.ensure_future(self._receive_message(),
loop=self._event_loop)
if self._monitor:
self._monitor_fd = "inproc://monitor.s-{}".format(
_generate_id()[0:5])
self._monitor_sock = self._socket.get_monitor_socket(
zmq.EVENT_DISCONNECTED, addr=self._monitor_fd)
asyncio.ensure_future(self._monitor_disconnects(),
loop=self._event_loop)
except Exception as e:
# Put the exception on the queue where in start we are waiting
# for it.
complete_or_error_queue.put_nowait(e)
raise
if self._heartbeat:
asyncio.ensure_future(self._do_heartbeat(), loop=self._event_loop)
# Put a 'complete with the setup tasks' sentinel on the queue.
complete_or_error_queue.put_nowait(_STARTUP_COMPLETE_SENTINEL)
asyncio.ensure_future(self._notify_started(), loop=self._event_loop)
self._event_loop.run_forever()
# event_loop.stop called elsewhere will cause the loop to break out
# of run_forever then it can be closed and the context destroyed.
self._event_loop.close()
self._socket.close(linger=0)
if self._monitor:
self._monitor_sock.close(linger=0)
self._context.destroy(linger=0)
def handle(self, *args, **options):
# Initialize logging.
self.logging_support(options["log_file"])
# Set the logging level
if options['level'] == 'ERROR':
self.logger.setLevel(logging.ERROR)
elif options['level'] == 'WARN':
self.logger.setLevel(logging.WARN)
elif options['level'] == 'INFO':
self.logger.setLevel(logging.INFO)
else:
self.logger.setLevel(logging.DEBUG)
self.logger.info("Dropping privileges")
if not self.drop_privileges(options['user'], options['group']):
self.logger.error("Unable to drop privileges")
return
auth = None
# Create the sockets
context = zmq.Context()
self.pull_socket = context.socket(zmq.PULL)
self.controler = context.socket(zmq.ROUTER)
if options['encrypt']:
self.logger.info("Starting encryption")
try:
auth = ThreadAuthenticator(context)
auth.start()
self.logger.debug("Opening master certificate: %s",
options['master_cert'])
master_public, master_secret = zmq.auth.load_certificate(
options['master_cert'])
self.logger.debug("Using slaves certificates from: %s",
options['slaves_certs'])
auth.configure_curve(domain='*',
location=options['slaves_certs'])
except IOError as err:
self.logger.error(err)
auth.stop()
return
self.controler.curve_publickey = master_public
self.controler.curve_secretkey = master_secret
self.controler.curve_server = True
self.pull_socket.curve_publickey = master_public
self.pull_socket.curve_secretkey = master_secret
self.pull_socket.curve_server = True
self.pull_socket.bind(options['log_socket'])
self.controler.bind(options['master_socket'])
# Last access to the database for new jobs and cancelations
last_db_access = 0
# Poll on the sockets (only one for the moment). This allow to have a
# nice timeout along with polling.
poller = zmq.Poller()
poller.register(self.pull_socket, zmq.POLLIN)
poller.register(self.controler, zmq.POLLIN)
# Mask signals and create a pipe that will receive a bit for each
# signal received. Poll the pipe along with the zmq socket so that we
# can only be interupted while reading data.
(pipe_r, pipe_w) = os.pipe()
flags = fcntl.fcntl(pipe_w, fcntl.F_GETFL, 0)
fcntl.fcntl(pipe_w, fcntl.F_SETFL, flags | os.O_NONBLOCK)
def signal_to_pipe(signumber, _):
# Send the signal number on the pipe
os.write(pipe_w, chr(signumber))
signal.signal(signal.SIGHUP, signal_to_pipe)
signal.signal(signal.SIGINT, signal_to_pipe)
signal.signal(signal.SIGTERM, signal_to_pipe)
signal.signal(signal.SIGQUIT, signal_to_pipe)
poller.register(pipe_r, zmq.POLLIN)
if os.path.exists('/etc/lava-server/worker.conf'):
self.logger.error(
"[FAIL] lava-master must not be run on a remote worker!")
self.controler.close(linger=0)
self.pull_socket.close(linger=0)
context.term()
sys.exit(2)
self.logger.info("[INIT] LAVA dispatcher-master has started.")
self.logger.info("[INIT] Using protocol version %d", PROTOCOL_VERSION)
while True:
try:
try:
# TODO: Fix the timeout computation
# Wait for data or a timeout
sockets = dict(poller.poll(TIMEOUT * 1000))
except zmq.error.ZMQError:
continue
if sockets.get(pipe_r) == zmq.POLLIN:
signum = ord(os.read(pipe_r, 1))
if signum == signal.SIGHUP:
self.logger.info(
"[POLL] SIGHUP received, restarting loggers")
self.logging_support(options["log_file"])
else:
self.logger.info("[POLL] Received a signal, leaving")
break
# Logging socket
if sockets.get(self.pull_socket) == zmq.POLLIN:
self.logging_socket(options)
# Garbage collect file handlers
now = time.time()
for job_id in self.jobs.keys(): # pylint: disable=consider-iterating-dictionary
if now - self.jobs[job_id].last_usage > FD_TIMEOUT:
self.logger.info("[%s] Closing log file", job_id)
self.jobs[job_id].close()
del self.jobs[job_id]
# Command socket
if sockets.get(self.controler) == zmq.POLLIN:
if not self.controler_socket():
continue
# Check dispatchers status
now = time.time()
for hostname, dispatcher in self.dispatchers.iteritems():
if dispatcher.online and now - dispatcher.last_msg > DISPATCHER_TIMEOUT:
self.logger.error(
"[STATE] Dispatcher <%s> goes OFFLINE", hostname)
self.dispatchers[hostname].online = False
# TODO: DB: mark the dispatcher as offline and attached
# devices
# Limit accesses to the database. This will also limit the rate of
# CANCEL and START messages
if now - last_db_access > DB_LIMIT:
last_db_access = now
# TODO: make this atomic
# Dispatch pipeline jobs with devices in Reserved state
self.process_jobs(options)
# Handle canceling jobs
self.handle_canceling()
except (OperationalError, InterfaceError):
self.logger.info("[RESET] database connection reset.")
continue
# Closing sockets and droping messages.
self.logger.info("[CLOSE] Closing the sockets and dropping messages")
self.controler.close(linger=0)
self.pull_socket.close(linger=0)
if options['encrypt']:
auth.stop()
context.term()
def __init__(self,
address=None,
port=None,
protocol=None,
pattern=0,
receive_mode=False,
logging=False,
**options):
try:
# import PyZMQ library
import zmq
# import ZMQError
from zmq.error import ZMQError
# assign values to global variable for further use
self.__zmq = zmq
self.__ZMQError = ZMQError
except ImportError as error:
# raise error
raise ImportError(
"[NetGear:ERROR] :: pyzmq python library not installed. Kindly install it with `pip install pyzmq` command."
)
# enable logging if specified
self.__logging = False
if logging:
self.__logging = logging
# define valid messaging patterns => `0`: zmq.PAIR, `1`:(zmq.REQ,zmq.REP), and `1`:(zmq.SUB,zmq.PUB)
valid_messaging_patterns = {
0: (zmq.PAIR, zmq.PAIR),
1: (zmq.REQ, zmq.REP),
2: (zmq.PUB, zmq.SUB),
}
# initialize messaging pattern
msg_pattern = None
# check whether user-defined messaging pattern is valid
if isinstance(pattern, int) and pattern in valid_messaging_patterns:
# assign value
msg_pattern = valid_messaging_patterns[pattern]
self.__pattern = pattern # add it to global variable for further use
else:
# otherwise default to 0:`zmq.PAIR`
self.__pattern = 0
msg_pattern = valid_messaging_patterns[self.__pattern]
# log it
if self.__logging:
logger.warning(
"Wrong pattern value, Defaulting to `zmq.PAIR`! Kindly refer Docs for more Information."
)
# check whether user-defined messaging protocol is valid
if not (protocol in ["tcp", "ipc"]):
# else default to `tcp` protocol
protocol = "tcp"
# log it
if self.__logging:
logger.warning(
"Protocol is invalid or not provided. Defaulting to `tcp` protocol!"
)
# generate random device id
self.__id = "".join(
random.choice("0123456789ABCDEF") for i in range(5))
self.__msg_flag = 0 # handles connection flags
self.__msg_copy = False # handles whether to copy data
self.__msg_track = False # handles whether to track packets
self.__multiserver_mode = False # handles multiserver_mode state
recv_filter = "" # user-defined filter to allow specific port/servers only in multiserver_mode
# define bi-directional data transmission mode
self.__bi_mode = False # handles bi_mode state
self.__bi_data = None # handles return data
# define valid ZMQ security mechanisms => `0`: Grasslands, `1`:StoneHouse, and `1`:IronHouse
valid_security_mech = {
0: "Grasslands",
1: "StoneHouse",
2: "IronHouse"
}
self.__secure_mode = 0 # handles ZMQ security layer status
auth_cert_dir = "" # handles valid ZMQ certificates dir
self.__auth_publickeys_dir = "" # handles valid ZMQ public certificates dir
self.__auth_secretkeys_dir = "" # handles valid ZMQ private certificates dir
overwrite_cert = False # checks if certificates overwriting allowed
custom_cert_location = "" # handles custom ZMQ certificates path
# handle force socket termination if there's latency in network
self.__force_close = False
# define stream compression handlers
self.__compression = "" # disabled by default
self.__compression_params = None
# reformat dict
options = {str(k).strip(): v for k, v in options.items()}
# assign values to global variables if specified and valid
for key, value in options.items():
if key == "multiserver_mode" and isinstance(value, bool):
if pattern > 0:
# multi-server mode
self.__multiserver_mode = value
else:
self.__multiserver_mode = False
logger.critical("Multi-Server is disabled!")
raise ValueError(
"[NetGear:ERROR] :: `{}` pattern is not valid when Multi-Server Mode is enabled. Kindly refer Docs for more Information."
.format(pattern))
elif key == "filter" and isinstance(value, str):
# custom filter in multi-server mode
recv_filter = value
elif (key == "secure_mode" and isinstance(value, int)
and (value in valid_security_mech)):
# secure mode
assert zmq.zmq_version_info() >= (
4,
0,
), "[NetGear:ERROR] :: ZMQ Security feature is not supported in libzmq version < 4.0."
self.__secure_mode = value
elif key == "custom_cert_location" and isinstance(value, str):
# custom auth certificates path
assert os.access(
value, os.W_OK
), "[NetGear:ERROR] :: Permission Denied!, cannot write ZMQ authentication certificates to '{}' directory!".format(
value)
assert os.path.isdir(
os.path.abspath(value)
), "[NetGear:ERROR] :: `custom_cert_location` value must be the path to a valid directory!"
custom_cert_location = os.path.abspath(value)
elif key == "overwrite_cert" and isinstance(value, bool):
# enable/disable auth certificate overwriting
overwrite_cert = value
# handle encoding and decoding if specified
elif (key == "compression_format" and isinstance(value, str)
and value.lower().strip()
in [".jpg", ".jpeg", ".bmp", ".png"]): # few are supported
# enable encoding
if not (receive_mode):
self.__compression = value.lower().strip()
elif key == "compression_param":
# specify encoding/decoding params
if receive_mode and isinstance(value, int):
self.__compression_params = value
if self.__logging:
logger.debug("Decoding flag: {}.".format(value))
elif not (receive_mode) and isinstance(value, (list, tuple)):
if self.__logging:
logger.debug("Encoding parameters: {}.".format(value))
self.__compression_params = list(value)
else:
if self.__logging:
logger.warning(
"Invalid compression parameters: {} skipped!".
format(value))
self.__compression_params = (
cv2.IMREAD_COLOR if receive_mode else []
) # skip to defaults
# enable bi-directional data transmission if specified
elif key == "bidirectional_mode" and isinstance(value, bool):
# check if pattern is valid
if pattern < 2:
self.__bi_mode = True
else:
self.__bi_mode = False
logger.critical(
"Bi-Directional data transmission is disabled!")
raise ValueError(
"[NetGear:ERROR] :: `{}` pattern is not valid when Bi-Directional Mode is enabled. Kindly refer Docs for more Information."
.format(pattern))
# enable force socket closing if specified
elif key == "force_terminate" and isinstance(value, bool):
# check if address is local
if address is None:
self.__force_close = False
logger.critical(
"Force termination is disabled for local servers!")
else:
self.__force_close = True
if self.__logging:
logger.warning(
"Force termination is enabled for this connection!"
)
# various ZMQ flags
elif key == "flag" and isinstance(value, int):
self.__msg_flag = value
elif key == "copy" and isinstance(value, bool):
self.__msg_copy = value
elif key == "track" and isinstance(value, bool):
self.__msg_track = value
else:
pass
# handle secure mode
if self.__secure_mode:
# import important libs
import zmq.auth
from zmq.auth.thread import ThreadAuthenticator
# log if overwriting is enabled
if overwrite_cert:
if not receive_mode:
if self.__logging:
logger.warning(
"Overwriting ZMQ Authentication certificates over previous ones!"
)
else:
overwrite_cert = False
if self.__logging:
logger.critical(
"Overwriting ZMQ Authentication certificates is disabled for Client-end!"
)
# generate and validate certificates path
try:
# check if custom certificates path is specified
if custom_cert_location:
(
auth_cert_dir,
self.__auth_secretkeys_dir,
self.__auth_publickeys_dir,
) = generate_auth_certificates(custom_cert_location,
overwrite=overwrite_cert,
logging=logging)
else:
# otherwise auto-generate suitable path
from os.path import expanduser
(
auth_cert_dir,
self.__auth_secretkeys_dir,
self.__auth_publickeys_dir,
) = generate_auth_certificates(
os.path.join(expanduser("~"), ".vidgear"),
overwrite=overwrite_cert,
logging=logging,
)
# log it
if self.__logging:
logger.debug(
"`{}` is the default location for storing ZMQ authentication certificates/keys."
.format(auth_cert_dir))
except Exception as e:
# catch if any error occurred
logger.exception(str(e))
# also disable secure mode
self.__secure_mode = 0
logger.critical(
"ZMQ Security Mechanism is disabled for this connection!")
else:
# log if disabled
if self.__logging:
logger.warning(
"ZMQ Security Mechanism is disabled for this connection!")
# handle bi_mode
if self.__bi_mode:
# disable bi_mode if multi-server is enabled
if self.__multiserver_mode:
self.__bi_mode = False
logger.critical(
"Bi-Directional Data Transmission is disabled when Multi-Server Mode is Enabled due to incompatibility!"
)
else:
# enable force termination by default
self.__force_close = True
if self.__logging:
logger.warning(
"Force termination is enabled for this connection by default!"
)
logger.debug(
"Bi-Directional Data Transmission is enabled for this connection!"
)
# initialize termination flag
self.__terminate = False
# initialize exit_loop flag
self.__exit_loop = False
# initialize and assign receive mode to global variable
self.__receive_mode = receive_mode
# define messaging context instance
self.__msg_context = zmq.Context.instance()
# check whether `receive_mode` is enabled by user
if receive_mode:
# if does than define connection address
if address is None:
address = "*" # define address
# check if multiserver_mode is enabled
if self.__multiserver_mode:
# check if unique server port address list/tuple is assigned or not in multiserver_mode
if port is None or not isinstance(port, (tuple, list)):
# raise error if not
raise ValueError(
"[NetGear:ERROR] :: Incorrect port value! Kindly provide a list/tuple of ports while Multi-Server mode is enabled. For more information refer VidGear docs."
)
else:
# otherwise log it
logger.debug(
"Enabling Multi-Server Mode at PORTS: {}!".format(
port))
# create port address buffer for keeping track of incoming server's port
self.__port_buffer = []
else:
# otherwise assign local port address if None
if port is None:
port = "5555"
try:
# initiate and handle secure mode
if self.__secure_mode > 0:
# start an authenticator for this context
auth = ThreadAuthenticator(self.__msg_context)
auth.start()
auth.allow(str(address)) # allow current address
# check if `IronHouse` is activated
if self.__secure_mode == 2:
# tell authenticator to use the certificate from given valid dir
auth.configure_curve(
domain="*", location=self.__auth_publickeys_dir)
else:
# otherwise tell the authenticator how to handle the CURVE requests, if `StoneHouse` is activated
auth.configure_curve(domain="*",
location=zmq.auth.CURVE_ALLOW_ANY)
# initialize and define thread-safe messaging socket
self.__msg_socket = self.__msg_context.socket(msg_pattern[1])
if self.__pattern == 2 and not (self.__secure_mode):
self.__msg_socket.set_hwm(1)
if self.__multiserver_mode:
# if multiserver_mode is enabled, then assign port addresses to zmq socket
for pt in port:
# enable specified secure mode for the zmq socket
if self.__secure_mode > 0:
# load server key
server_secret_file = os.path.join(
self.__auth_secretkeys_dir,
"server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
# load all CURVE keys
self.__msg_socket.curve_secretkey = server_secret
self.__msg_socket.curve_publickey = server_public
# enable CURVE connection for this socket
self.__msg_socket.curve_server = True
# define socket options
if self.__pattern == 2:
self.__msg_socket.setsockopt_string(
zmq.SUBSCRIBE, recv_filter)
# bind socket to given server protocol, address and ports
self.__msg_socket.bind(protocol + "://" +
str(address) + ":" + str(pt))
# define socket optimizer
self.__msg_socket.setsockopt(zmq.LINGER, 0)
else:
# enable specified secure mode for the zmq socket
if self.__secure_mode > 0:
# load server key
server_secret_file = os.path.join(
self.__auth_secretkeys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
# load all CURVE keys
self.__msg_socket.curve_secretkey = server_secret
self.__msg_socket.curve_publickey = server_public
# enable CURVE connection for this socket
self.__msg_socket.curve_server = True
# define exclusive socket options for patterns
if self.__pattern == 2:
self.__msg_socket.setsockopt_string(zmq.SUBSCRIBE, "")
# bind socket to given protocol, address and port normally
self.__msg_socket.bind(protocol + "://" + str(address) +
":" + str(port))
# define socket optimizer
self.__msg_socket.setsockopt(zmq.LINGER, 0)
except Exception as e:
logger.exception(str(e))
# otherwise raise value error if errored
if self.__secure_mode:
logger.warning(
"Failed to activate ZMQ Security Mechanism: `{}` for this address!"
.format(valid_security_mech[self.__secure_mode]))
if self.__multiserver_mode:
raise ValueError(
"[NetGear:ERROR] :: Multi-Server Mode, failed to connect to ports: {} with pattern: {}! Kindly recheck all parameters."
.format(str(port), pattern))
else:
raise ValueError(
"[NetGear:ERROR] :: Failed to bind address: {} and pattern: {}! Kindly recheck all parameters."
.format((protocol + "://" + str(address) + ":" +
str(port)), pattern))
# log and enable threaded queue mode
if self.__logging:
logger.debug(
"Threaded Queue Mode is enabled by default for NetGear.")
# define deque and assign it to global var
self.__queue = deque(maxlen=96) # max len 96 to check overflow
# initialize and start threading instance
self.__thread = Thread(target=self.__update,
name="NetGear",
args=())
self.__thread.daemon = True
self.__thread.start()
if self.__logging:
# finally log progress
logger.debug(
"Successfully Binded to address: {} with pattern: {}.".
format((protocol + "://" + str(address) + ":" + str(port)),
pattern))
if self.__secure_mode:
logger.debug(
"Enabled ZMQ Security Mechanism: `{}` for this address, Successfully!"
.format(valid_security_mech[self.__secure_mode]))
logger.debug(
"Multi-threaded Receive Mode is enabled Successfully!")
logger.debug("Device Unique ID is {}.".format(self.__id))
logger.debug("Receive Mode is activated successfully!")
else:
# otherwise default to `Send Mode`
if address is None:
address = "localhost" # define address
# check if multiserver_mode is enabled
if self.__multiserver_mode:
# check if unique server port address is assigned or not in multiserver_mode
if port is None:
# raise error is not
raise ValueError(
"[NetGear:ERROR] :: Kindly provide a unique & valid port value at Server-end. For more information refer VidGear docs."
)
else:
# otherwise log it
logger.debug(
"Enabling Multi-Server Mode at PORT: {} on this device!"
.format(port))
# assign value to global variable
self.port = port
else:
# otherwise assign local port address if None
if port is None:
port = "5555"
try:
# initiate and handle secure mode
if self.__secure_mode > 0:
# start an authenticator for this context
auth = ThreadAuthenticator(self.__msg_context)
auth.start()
auth.allow(str(address)) # allow current address
# check if `IronHouse` is activated
if self.__secure_mode == 2:
# tell authenticator to use the certificate from given valid dir
auth.configure_curve(
domain="*", location=self.__auth_publickeys_dir)
else:
# otherwise tell the authenticator how to handle the CURVE requests, if `StoneHouse` is activated
auth.configure_curve(domain="*",
location=zmq.auth.CURVE_ALLOW_ANY)
# initialize and define thread-safe messaging socket
self.__msg_socket = self.__msg_context.socket(msg_pattern[0])
if self.__pattern == 1:
# if pattern is 1, define additional flags
self.__msg_socket.REQ_RELAXED = True
self.__msg_socket.REQ_CORRELATE = True
if self.__pattern == 2 and not (self.__secure_mode):
self.__msg_socket.set_hwm(
1) # if pattern is 2, define additional optimizer
# enable specified secure mode for the zmq socket
if self.__secure_mode > 0:
# load client key
client_secret_file = os.path.join(
self.__auth_secretkeys_dir, "client.key_secret")
client_public, client_secret = zmq.auth.load_certificate(
client_secret_file)
# load all CURVE keys
self.__msg_socket.curve_secretkey = client_secret
self.__msg_socket.curve_publickey = client_public
# load server key
server_public_file = os.path.join(
self.__auth_publickeys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(
server_public_file)
# inject public key to make a CURVE connection.
self.__msg_socket.curve_serverkey = server_public
# connect socket to given protocol, address and port
self.__msg_socket.connect(protocol + "://" + str(address) +
":" + str(port))
# define socket options
self.__msg_socket.setsockopt(zmq.LINGER, 0)
except Exception as e:
logger.exception(str(e))
# log if errored
if self.__secure_mode:
logger.warning(
"Failed to activate ZMQ Security Mechanism: `{}` for this address!"
.format(valid_security_mech[self.__secure_mode]))
# raise value error
raise ValueError(
"[NetGear:ERROR] :: Failed to connect address: {} and pattern: {}! Kindly recheck all parameters."
.format(
(protocol + "://" + str(address) + ":" + str(port)),
pattern))
if self.__logging:
# finally log progress
logger.debug(
"Successfully connected to address: {} with pattern: {}.".
format((protocol + "://" + str(address) + ":" + str(port)),
pattern))
if self.__secure_mode:
logger.debug(
"Enabled ZMQ Security Mechanism: `{}` for this address, Successfully!"
.format(valid_security_mech[self.__secure_mode]))
logger.debug("This device Unique ID is {}.".format(self.__id))
logger.debug(
"Send Mode is successfully activated and ready to send data!"
)
def main():
"""
Parses command line input and then runs PubProxy and RouterProxy.
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"-d",
dest="debug",
default=False,
action="store_true",
help="Enable debugging to console",
)
parser.add_argument(
"--it",
dest="proxy_base_port",
type=int,
default=5561,
metavar="proxy_base_port",
help="Interior proxy base port to bind to (->RTR<-, PUB, DLR, SUB)",
)
parser.add_argument(
"--et",
dest="be_base_port",
type=int,
default=5563,
metavar="be_base_port",
help="Base port of backend to connect to (RTR, PUB, ->DLR<-, SUB)",
)
parser.add_argument(
"--ep",
dest="be_ip",
required=True,
metavar="be_ip",
help="IP of backend to connect to",
)
parser.add_argument(
"--ip",
dest="pxy_ip",
required=True,
metavar="pxy_ip",
help="IP of internal proxy interface to bind to",
)
parser.add_argument(
"--ak",
dest="ak",
required=True,
metavar="appkey",
help="The Curve public and private keys used by VMI apps",
)
parser.add_argument(
"--bk",
dest="bk",
required=True,
metavar="bekey",
help="The Curve public and private keys for the backend",
)
args = parser.parse_args()
print(f'{"#"*10}\nmain, starting Facilities with args: {args}\n{"#"*10}')
if not os.path.isfile(args.ak) or not os.path.isfile(args.bk):
print(f"ERROR: either {args.ak} or {args.bk} is is missing!")
sys.exit()
kp = {"be_key": args.bk, "app_key": args.ak}
print(f"startup: keys OK")
context = zmq.Context(io_threads=8)
auth = ThreadAuthenticator(context)
auth.start()
auth.configure_curve(domain="*", location=zmq.auth.CURVE_ALLOW_ANY)
proxy = RouterProxyRuntime(
context,
kp,
debug=args.debug,
pxy_ip=args.pxy_ip,
proxy_base_port=args.proxy_base_port,
be_ip=args.be_ip,
be_base_port=args.be_base_port,
)
proxy.start()
pubrelay = PubProxyRuntime(
context,
kp,
debug=args.debug,
pxy_ip=args.pxy_ip,
proxy_base_port=args.proxy_base_port,
be_ip=args.be_ip,
be_base_port=args.be_base_port,
)
pubrelay.start()
try:
while True:
tprintn(".")
time.sleep(1)
except KeyboardInterrupt:
print("exiting...")
auth.stop()
context.term()
#!/usr/bin/env python
import zmq
import zmq.auth
from zmq.auth.thread import ThreadAuthenticator
context = zmq.Context()
auth = ThreadAuthenticator(context)
auth.start()
#auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
auth.configure_curve(domain='*', location='authorized_keys')
socket = context.socket(zmq.REP)
socket.curve_server = True
socket.curve_publickey, socket.curve_secretkey = zmq.auth.load_certificate('keys/server.key_secret')
socket.bind("tcp://*:{}".format(53487))
while True:
msg = socket.recv()
print("Received request: ", msg)
socket.send(b'ok')
auth.stop()
def secure_srv(q_param, q_mgmt):
#on thread start create current crypto key
local = os.urandom(crypto_secretbox_KEYBYTES)
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(public_keys_dir)
and os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - add certificat first")
sys.exit(1)
ctx = zmq.Context()
# Start an authenticator for this context.
auth = ThreadAuthenticator(context=ctx, encoding='utf-8')
auth.start()
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUB)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True
server.bind('tcp://*:5556')
while (True):
if not q_mgmt.empty():
print "we rcv a mgmt msg"
msg = q_mgmt.get()
if msg.action == 0:
print "we have to restart auth"
auth.stop()
auth = ThreadAuthenticator(context=ctx, encoding='utf-8')
auth.start()
auth.configure_curve(domain='*', location=public_keys_dir)
print "Auth restarted"
if not q_param.empty():
print "we rcv a param msg"
p = q_param.get()
if p.t == 0:
local = p.v
#send new parameters with seq
s_snd = struct.Struct('! B B H H H I')
data = s_snd.pack(1, 6, 4, len(p.v), p.t, p.seq)
server.send(data + str(p.v))
#loop send key+seq if no new params
key_param = Params(0, local, random.randint(0, max_int32_seq))
s_snd = struct.Struct('! B B H H H I')
data = s_snd.pack(1, 6, 4, len(key_param.v), key_param.t,
key_param.seq)
server.send(data + str(key_param.v))
time.sleep(2) # avoid "as fast as possible" loop
# stop auth thread
auth.stop()
# stop zmq context
ctx.destroy()
return
class Ironhouse:
auth_port = os.getenv('AUTH_PORT', 4523)
keyname = os.getenv('HOST_NAME', 'ironhouse')
authorized_nodes = {}
base_dir = 'certs/{}'.format(keyname)
keys_dir = join(base_dir, 'certificates')
authorized_keys_dir = join(base_dir, 'authorized_keys')
ctx = None
auth = None
daemon_auth = None
def __init__(self,
sk=None,
auth_validate=None,
wipe_certs=False,
auth_port=None,
keyname=None,
*args,
**kwargs):
if auth_validate: self.auth_validate = auth_validate
else: self.auth_validate = Ironhouse.auth_validate
self.auth_port = auth_port or self.auth_port
self.keyname = keyname or Ironhouse.keyname
self.authorized_keys = {}
self.pk2vk = {}
self.vk, self.public_key, self.secret = self.generate_certificates(
sk, wipe_certs=wipe_certs)
@classmethod
def vk2pk(cls, vk):
return encode(
VerifyKey(
bytes.fromhex(vk)).to_curve25519_public_key()._public_key)
@classmethod
def generate_certificates(cls, sk_hex, wipe_certs=False):
sk = SigningKey(seed=bytes.fromhex(sk_hex))
vk = sk.verify_key.encode().hex()
public_key = cls.vk2pk(vk)
private_key = crypto_sign_ed25519_sk_to_curve25519(
sk._signing_key).hex()
for d in [cls.keys_dir, cls.authorized_keys_dir]:
if wipe_certs and exists(d):
shutil.rmtree(d)
os.makedirs(d, exist_ok=True)
if wipe_certs:
_, secret = cls.create_from_private_key(private_key)
for key_file in os.listdir(cls.keys_dir):
if key_file.endswith(".key"):
shutil.move(join(cls.keys_dir, key_file),
join(cls.authorized_keys_dir, '.'))
if exists(cls.keys_dir):
shutil.rmtree(cls.keys_dir)
log.info('Generated CURVE certificate files!')
return vk, public_key, secret
@classmethod
def create_from_private_key(cls, private_key):
priv = PrivateKey(bytes.fromhex(private_key))
publ = priv.public_key
public_key = encode(publ._public_key)
secret = encode(priv._private_key)
base_filename = join(cls.keys_dir, cls.keyname)
public_key_file = "{0}.key".format(base_filename)
now = datetime.datetime.now()
zmq.auth.certs._write_key_file(
public_key_file, zmq.auth.certs._cert_public_banner.format(now),
public_key)
return public_key, secret
def add_public_key(self, public_key):
if self.public_key == public_key: return
keyname = decode(public_key).hex()
base_filename = join(self.authorized_keys_dir, keyname)
public_key_file = "{0}.key".format(base_filename)
now = datetime.datetime.now()
if exists(public_key_file):
log.debug('Public cert for {} has already been created.'.format(
public_key))
return
os.makedirs(self.authorized_keys_dir, exist_ok=True)
log.info(
'Adding new public key cert {} to the system.'.format(public_key))
zmq.auth.certs._write_key_file(
public_key_file, zmq.auth.certs._cert_public_banner.format(now),
public_key)
log.debug('{} has added {} to its authorized list'.format(
os.getenv('HOST_IP', '127.0.0.1'), public_key))
self.reconfigure_curve()
self.authorized_keys[public_key] = True
def remove_public_key(self, public_key):
if self.public_key == public_key: return
keyname = decode(public_key).hex()
base_filename = join(self.authorized_keys_dir, keyname)
public_key_file = "{0}.key".format(base_filename)
if exists(public_key_file):
os.remove(public_key_file)
log.debug('{} has remove {} from its authorized list'.format(
os.getenv('HOST_IP', '127.0.0.1'), public_key))
self.reconfigure_curve()
self.authorized_keys[public_key] = False
@classmethod
def secure_context(cls, async=False):
if async:
ctx = zmq.asyncio.Context()
auth = AsyncioAuthenticator(ctx)
auth.log = log # The constructor doesn't have "log" like its synchronous counter-part
else:
ctx = zmq.Context()
auth = ThreadAuthenticator(ctx, log=log)
auth.start()
return ctx, auth
import zmq
import zmq.auth
from zmq.auth.thread import ThreadAuthenticator
context = zmq.Context()
server = context.socket(zmq.REP)
auth = ThreadAuthenticator(context)
auth.start()
auth.allow('127.0.0.1')
auth.configure_plain(domain='*', passwords={'admin': 'password'})
server.plain_server = True
server.setsockopt(zmq.PLAIN_SERVER, 1)
server.connect('tcp://127.0.0.1:5556')
msg = server.recv_string()
server.send(b'Authenticated')
auth.stop()
async def run():
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
base_dir = Path(__file__).parent
keys_dir = base_dir / 'certificates'
public_keys_dir = base_dir / 'public_keys'
secret_keys_dir = base_dir / 'private_keys'
if not keys_dir.is_dir() or not public_keys_dir.is_dir(
) or not secret_keys_dir.is_dir():
logging.critical(
"Certificates are missing - run generate_certificates.py script first"
)
sys.exit(1)
ctx = Context.instance()
# Start an authenticator for this context.
auth = AsyncioAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUSH)
server_secret_file = secret_keys_dir / "server.key_secret"
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.PULL)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = secret_keys_dir / "client.key_secret"
client_public, client_secret = zmq.auth.load_certificate(
client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = public_keys_dir / "server.key"
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
await server.send(b"Hello")
if await client.poll(1000):
msg = await client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# close sockets
server.close()
client.close()
# stop auth task
auth.stop()
class Ironhouse:
def __init__(self,
sk=None,
auth_validate=None,
wipe_certs=False,
auth_port=None,
keyname=None,
*args,
**kwargs):
self.auth_port = auth_port or os.getenv('AUTH_PORT', 4523)
self.keyname = keyname or os.getenv('HOSTNAME',
basename(splitext(__file__)[0]))
self.base_dir = 'certs/{}'.format(self.keyname)
self.keys_dir = os.path.join(self.base_dir, 'certificates')
self.public_keys_dir = os.path.join(self.base_dir, 'public_keys')
self.secret_keys_dir = os.path.join(self.base_dir, 'private_keys')
self.secret_file = os.path.join(self.secret_keys_dir,
"{}.key_secret".format(self.keyname))
if auth_validate:
self.auth_validate = auth_validate
else:
self.auth_validate = Ironhouse.auth_validate
self.wipe_certs = wipe_certs
if sk:
self.generate_certificates(sk)
self.public_key, self.secret = zmq.auth.load_certificate(
self.secret_file)
def vk2pk(self, vk):
return encode(
VerifyKey(
bytes.fromhex(vk)).to_curve25519_public_key()._public_key)
def generate_certificates(self, sk):
sk = SigningKey(seed=bytes.fromhex(sk))
self.vk = sk.verify_key.encode().hex()
self.public_key = self.vk2pk(self.vk)
private_key = crypto_sign_ed25519_sk_to_curve25519(
sk._signing_key).hex()
for d in [self.keys_dir, self.public_keys_dir, self.secret_keys_dir]:
if self.wipe_certs and os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d, exist_ok=True)
if self.wipe_certs:
self.create_from_private_key(private_key)
# move public keys to appropriate directory
for key_file in os.listdir(self.keys_dir):
if key_file.endswith(".key"):
shutil.move(os.path.join(self.keys_dir, key_file),
os.path.join(self.public_keys_dir, '.'))
# move secret keys to appropriate directory
for key_file in os.listdir(self.keys_dir):
if key_file.endswith(".key_secret"):
shutil.move(os.path.join(self.keys_dir, key_file),
os.path.join(self.secret_keys_dir, '.'))
log.info('Generated CURVE certificate files!')
def create_from_private_key(self, private_key):
priv = PrivateKey(bytes.fromhex(private_key))
publ = priv.public_key
self.public_key = public_key = encode(publ._public_key)
secret_key = encode(priv._private_key)
base_filename = os.path.join(self.keys_dir, self.keyname)
secret_key_file = "{0}.key_secret".format(base_filename)
public_key_file = "{0}.key".format(base_filename)
now = datetime.datetime.now()
zmq.auth.certs._write_key_file(
public_key_file, zmq.auth.certs._cert_public_banner.format(now),
public_key)
zmq.auth.certs._write_key_file(
secret_key_file,
zmq.auth.certs._cert_secret_banner.format(now),
public_key,
secret_key=secret_key)
def create_from_public_key(self, public_key):
if self.public_key == public_key:
return
keyname = decode(public_key).hex()
base_filename = os.path.join(self.public_keys_dir, keyname)
public_key_file = "{0}.key".format(base_filename)
now = datetime.datetime.now()
if os.path.exists(public_key_file):
log.debug('Public cert for {} has already been created.'.format(
public_key))
return
os.makedirs(self.public_keys_dir, exist_ok=True)
log.info(
'Adding new public key cert {} to the system.'.format(public_key))
zmq.auth.certs._write_key_file(
public_key_file, zmq.auth.certs._cert_public_banner.format(now),
public_key)
def secure_context(self, async=False):
if async:
ctx = zmq.asyncio.Context()
auth = AsyncioAuthenticator(ctx)
else:
ctx = zmq.Context()
auth = ThreadAuthenticator(ctx)
auth.start()
self.reconfigure_curve(auth)
return ctx, auth
def handle(self, *args, **options):
if options['level'] == 'ERROR':
self.logger.setLevel(logging.ERROR)
elif options['level'] == 'WARN':
self.logger.setLevel(logging.WARN)
elif options['level'] == 'INFO':
self.logger.setLevel(logging.INFO)
else:
self.logger.setLevel(logging.DEBUG)
auth = None
# Create the sockets
context = zmq.Context()
self.pull_socket = context.socket(zmq.PULL)
self.controler = context.socket(zmq.ROUTER)
if options['encrypt']:
self.logger.info("Starting encryption")
try:
auth = ThreadAuthenticator(context)
auth.start()
self.logger.debug("Opening master certificate: %s", options['master_cert'])
master_public, master_secret = zmq.auth.load_certificate(options['master_cert'])
self.logger.debug("Using slaves certificates from: %s", options['slaves_certs'])
auth.configure_curve(domain='*', location=options['slaves_certs'])
except IOError as err:
self.logger.error(err)
auth.stop()
return
self.controler.curve_publickey = master_public
self.controler.curve_secretkey = master_secret
self.controler.curve_server = True
self.pull_socket.curve_publickey = master_public
self.pull_socket.curve_secretkey = master_secret
self.pull_socket.curve_server = True
self.pull_socket.bind(options['log_socket'])
self.controler.bind(options['master_socket'])
# Last access to the database for new jobs and cancelations
last_db_access = 0
# Poll on the sockets (only one for the moment). This allow to have a
# nice timeout along with polling.
poller = zmq.Poller()
poller.register(self.pull_socket, zmq.POLLIN)
poller.register(self.controler, zmq.POLLIN)
# Mask signals and create a pipe that will receive a bit for each
# signal received. Poll the pipe along with the zmq socket so that we
# can only be interupted while reading data.
(pipe_r, pipe_w) = os.pipe()
flags = fcntl.fcntl(pipe_w, fcntl.F_GETFL, 0)
fcntl.fcntl(pipe_w, fcntl.F_SETFL, flags | os.O_NONBLOCK)
def signal_to_pipe(signumber, _):
# Send the signal number on the pipe
os.write(pipe_w, chr(signumber))
signal.signal(signal.SIGHUP, signal_to_pipe)
signal.signal(signal.SIGINT, signal_to_pipe)
signal.signal(signal.SIGTERM, signal_to_pipe)
signal.signal(signal.SIGQUIT, signal_to_pipe)
poller.register(pipe_r, zmq.POLLIN)
if os.path.exists('/etc/lava-server/worker.conf'):
self.logger.error("[FAIL] lava-master must not be run on a remote worker!")
self.controler.close(linger=0)
self.pull_socket.close(linger=0)
context.term()
sys.exit(2)
self.logger.info("[INIT] LAVA dispatcher-master has started.")
self.logger.info("[INIT] Using protocol version %d", PROTOCOL_VERSION)
while True:
try:
try:
# TODO: Fix the timeout computation
# Wait for data or a timeout
sockets = dict(poller.poll(TIMEOUT * 1000))
except zmq.error.ZMQError:
continue
if sockets.get(pipe_r) == zmq.POLLIN:
signum = ord(os.read(pipe_r, 1))
if signum == signal.SIGHUP:
self.logger.info("[POLL] SIGHUP received, restarting loggers")
self.logging_support()
else:
self.logger.info("[POLL] Received a signal, leaving")
break
# Logging socket
if sockets.get(self.pull_socket) == zmq.POLLIN:
self.logging_socket(options)
# Garbage collect file handlers
now = time.time()
for job_id in self.logs.keys(): # pylint: disable=consider-iterating-dictionary
if now - self.logs[job_id].last_usage > FD_TIMEOUT:
self.logger.info("[%s] Closing log file", job_id)
self.logs[job_id].close()
del self.logs[job_id]
# Command socket
if sockets.get(self.controler) == zmq.POLLIN:
if not self.controler_socket():
continue
# Check dispatchers status
now = time.time()
for hostname, dispatcher in self.dispatchers.iteritems():
if dispatcher.online and now - dispatcher.last_msg > DISPATCHER_TIMEOUT:
self.logger.error("[STATE] Dispatcher <%s> goes OFFLINE", hostname)
self.dispatchers[hostname].online = False
# TODO: DB: mark the dispatcher as offline and attached
# devices
# Limit accesses to the database. This will also limit the rate of
# CANCEL and START messages
if now - last_db_access > DB_LIMIT:
last_db_access = now
# TODO: make this atomic
# Dispatch pipeline jobs with devices in Reserved state
self.process_jobs(options)
# Handle canceling jobs
self.handle_canceling()
except (OperationalError, InterfaceError):
self.logger.info("[RESET] database connection reset.")
continue
# Closing sockets and droping messages.
self.logger.info("[CLOSE] Closing the sockets and dropping messages")
self.controler.close(linger=0)
self.pull_socket.close(linger=0)
if options['encrypt']:
auth.stop()
context.term()
async def run():
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
base_dir = Path(__file__).parent
keys_dir = base_dir / 'certificates'
public_keys_dir = base_dir / 'public_keys'
secret_keys_dir = base_dir / 'private_keys'
if (
not keys_dir.is_dir()
or not public_keys_dir.is_dir()
or not secret_keys_dir.is_dir()
):
logging.critical(
"Certificates are missing - run generate_certificates.py script first"
)
sys.exit(1)
ctx = Context.instance()
# Start an authenticator for this context.
auth = AsyncioAuthenticator(ctx)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.ROUTER)
server_secret_file = secret_keys_dir / "server.key_secret"
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind('tcp://*:9000')
client = ctx.socket(zmq.DEALER)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = secret_keys_dir / "client.key_secret"
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = public_keys_dir / "server.key"
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
await client.send(b"Hello")
if await server.poll(1000):
# use copy=False to allow access to message properties via the zmq.Frame API
# default recv(copy=True) returns only bytes, discarding properties
identity, msg = await server.recv_multipart(copy=False)
logging.info(f"Received {msg.bytes} from {msg['User-Id']!r}")
if msg.bytes == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# close sockets
server.close()
client.close()
# stop auth task
auth.stop()
def secure_srv(q_param, q_mgmt):
#on thread start create current crypto key
local = os.urandom(crypto_secretbox_KEYBYTES)
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not(os.path.exists(public_keys_dir) and
os.path.exists(secret_keys_dir)):
logging.critical("Certificates are missing - add certificat first")
sys.exit(1)
ctx = zmq.Context()
# Start an authenticator for this context.
auth = ThreadAuthenticator(context=ctx, encoding='utf-8')
auth.start()
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUB)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file
)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True
server.bind('tcp://*:5556')
while(True):
if not q_mgmt.empty():
print "we rcv a mgmt msg"
msg = q_mgmt.get()
if msg.action == 0:
print "we have to restart auth"
auth.stop()
auth = ThreadAuthenticator(context=ctx, encoding='utf-8')
auth.start()
auth.configure_curve(domain='*', location=public_keys_dir)
print "Auth restarted"
if not q_param.empty():
print "we rcv a param msg"
p = q_param.get()
if p.t == 0:
local = p.v
#send new parameters with seq
s_snd = struct.Struct('! B B H H H I')
data = s_snd.pack(1, 6, 4, len(p.v), p.t, p.seq)
server.send(data+str(p.v))
#loop send key+seq if no new params
key_param = Params(0, local, random.randint(0, max_int32_seq))
s_snd = struct.Struct('! B B H H H I')
data = s_snd.pack(
1,
6,
4,
len(key_param.v),
key_param.t,
key_param.seq
)
server.send(data+str(key_param.v))
time.sleep(2) # avoid "as fast as possible" loop
# stop auth thread
auth.stop()
# stop zmq context
ctx.destroy()
return