def _run(self):
"""
Start a loop to process the ZMQ requests from the signaler client.
"""
logger.debug("Running SignalerQt loop")
context = zmq.Context()
socket = context.socket(zmq.REP)
# Start an authenticator for this context.
auth = ThreadAuthenticator(context)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
public, secret = get_frontend_certificates()
socket.curve_publickey = public
socket.curve_secretkey = secret
socket.curve_server = True # must come before bind
socket.bind(self.BIND_ADDR)
while self._do_work.is_set():
# Wait for next request from client
try:
request = socket.recv(zmq.NOBLOCK)
# logger.debug("Received request: '{0}'".format(request))
socket.send("OK")
self._process_request(request)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
raise
time.sleep(0.01)
logger.debug("SignalerQt thread stopped.")
def _init_zmq(self):
"""
Configure the zmq components and connection.
"""
context = zmq.Context()
socket = context.socket(zmq.REP)
if flags.ZMQ_HAS_CURVE:
# Start an authenticator for this context.
auth = ThreadAuthenticator(context)
auth.start()
# XXX do not hardcode this here.
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
public, secret = get_backend_certificates()
socket.curve_publickey = public
socket.curve_secretkey = secret
socket.curve_server = True # must come before bind
socket.bind(self.BIND_ADDR)
if not flags.ZMQ_HAS_CURVE:
os.chmod(self.SOCKET_FILE, 0600)
self._zmq_socket = socket
def _run(self):
"""
Start a loop to process the ZMQ requests from the signaler client.
"""
logger.debug("Running SignalerQt loop")
context = zmq.Context()
socket = context.socket(zmq.REP)
# Start an authenticator for this context.
auth = ThreadAuthenticator(context)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
public, secret = get_frontend_certificates()
socket.curve_publickey = public
socket.curve_secretkey = secret
socket.curve_server = True # must come before bind
socket.bind(self.BIND_ADDR)
while self._do_work.is_set():
# Wait for next request from client
try:
request = socket.recv(zmq.NOBLOCK)
logger.debug("Received request: '{0}'".format(request))
socket.send("OK")
self._process_request(request)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
raise
time.sleep(0.01)
logger.debug("SignalerQt thread stopped.")
def main():
localhost = socket_m.getfqdn()
port = "5556"
# ip = "*"
ip = socket_m.gethostbyaddr(localhost)[2][0]
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.zap_domain = b'global'
socket.bind("tcp://" + ip + ":%s" % port)
auth = ThreadAuthenticator(context)
host = localhost
# host = asap3-p00
whitelist = socket_m.gethostbyaddr(host)[2][0]
# whitelist = None
auth.start()
if whitelist is None:
auth.auth = None
else:
auth.allow(whitelist)
try:
while True:
message = socket.recv_multipart()
print("received reply ", message)
except KeyboardInterrupt:
pass
finally:
auth.stop()
class Authenticator(object):
_authenticators = {}
@classmethod
def instance(cls, public_keys_dir):
'''Please avoid create multi instance'''
if public_keys_dir in cls._authenticators:
return cls._authenticators[public_keys_dir]
new_instance = cls(public_keys_dir)
cls._authenticators[public_keys_dir] = new_instance
return new_instance
def __init__(self, public_keys_dir):
self._auth = ThreadAuthenticator(zmq.Context.instance())
self._auth.start()
self._auth.allow('*')
self._auth.configure_curve(domain='*', location=public_keys_dir)
def set_server_key(self, zmq_socket, server_secret_key_path):
'''must call before bind'''
load_and_set_key(zmq_socket, server_secret_key_path)
zmq_socket.curve_server = True
def set_client_key(self, zmq_socket, client_secret_key_path, server_public_key_path):
'''must call before bind'''
load_and_set_key(zmq_socket, client_secret_key_path)
server_public, _ = zmq.auth.load_certificate(server_public_key_path)
zmq_socket.curve_serverkey = server_public
def stop(self):
self._auth.stop()
def _init_txzmq(self):
"""
Configure the txzmq components and connection.
"""
self._zmq_factory = txzmq.ZmqFactory()
self._zmq_factory.registerForShutdown()
self._zmq_connection = txzmq.ZmqREPConnection(self._zmq_factory)
context = self._zmq_factory.context
socket = self._zmq_connection.socket
def _gotMessage(messageId, messageParts):
self._zmq_connection.reply(messageId, "OK")
self._process_request(messageParts)
self._zmq_connection.gotMessage = _gotMessage
if flags.ZMQ_HAS_CURVE:
# Start an authenticator for this context.
auth = ThreadAuthenticator(context)
auth.start()
# XXX do not hardcode this here.
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
public, secret = get_backend_certificates()
socket.curve_publickey = public
socket.curve_secretkey = secret
socket.curve_server = True # must come before bind
proto, addr = self._server_address.split('://') # tcp/ipc, ip/socket
socket.bind(self._server_address)
if proto == 'ipc':
os.chmod(addr, 0600)
def main():
auth = ThreadAuthenticator(zmq.Context.instance())
auth.start()
auth.allow('127.0.0.1')
# Tell the authenticator how to handle CURVE requests
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
key = Key.load('example/broker.key_secret')
broker = SecureMajorDomoBroker(key, sys.argv[1])
try:
broker.serve_forever()
except KeyboardInterrupt:
auth.stop()
raise
def main():
auth = ThreadAuthenticator(zmq.Context.instance())
auth.start()
auth.allow('127.0.0.1')
# Tell the authenticator how to handle CURVE requests
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
key = Key.load('example/broker.key_secret')
broker = SecureMajorDomoBroker(key, sys.argv[1])
try:
broker.serve_forever()
except KeyboardInterrupt:
auth.stop()
raise
def _start_thread_auth(self, socket):
"""
Start the zmq curve thread authenticator.
:param socket: The socket in which to configure the authenticator.
:type socket: zmq.Socket
"""
authenticator = ThreadAuthenticator(self._factory.context)
authenticator.start()
# XXX do not hardcode this here.
authenticator.allow('127.0.0.1')
# tell authenticator to use the certificate in a directory
public_keys_dir = os.path.join(self._config_prefix, PUBLIC_KEYS_PREFIX)
authenticator.configure_curve(domain="*", location=public_keys_dir)
socket.curve_server = True # must come before bind
def _start_thread_auth(self, socket):
"""
Start the zmq curve thread authenticator.
:param socket: The socket in which to configure the authenticator.
:type socket: zmq.Socket
"""
authenticator = ThreadAuthenticator(self._factory.context)
authenticator.start()
# XXX do not hardcode this here.
authenticator.allow('127.0.0.1')
# tell authenticator to use the certificate in a directory
public_keys_dir = os.path.join(self._config_prefix, PUBLIC_KEYS_PREFIX)
authenticator.configure_curve(domain="*", location=public_keys_dir)
socket.curve_server = True # must come before bind
def main():
port = "5556"
socket_ip = "*"
# ip = socket.getfqdn()
context = zmq.Context()
auth = ThreadAuthenticator(context)
auth.start()
whitelist = [socket.getfqdn()]
for host in whitelist:
hostname, tmp, ip = socket.gethostbyaddr(host)
auth.allow(ip[0])
zmq_socket = context.socket(zmq.PUSH)
zmq_socket.zap_domain = b'global'
zmq_socket.bind("tcp://" + socket_ip + ":%s" % port)
try:
for i in range(5):
message = ["World"]
print("Send: ", message)
res = zmq_socket.send_multipart(message, copy=False, track=True)
if res.done:
print("res: done")
else:
print("res: waiting")
res.wait()
print("res: waiting...")
print("sleeping...")
if i == 1:
auth.stop()
zmq_socket.close(0)
auth.start()
# ip = socket.gethostbyaddr(socket.getfqdn())[2]
# auth.allow(ip[0])
ip = socket.gethostbyaddr(socket.getfqdn())[2]
auth.deny(ip[0])
zmq_socket = context.socket(zmq.PUSH)
zmq_socket.zap_domain = b'global'
zmq_socket.bind("tcp://" + socket_ip + ":%s" % port)
time.sleep(1)
print("sleeping...done")
i += 1
finally:
auth.stop()
def auth_init():
"""Start an authenticator for this context."""
from zmq.auth.thread import ThreadAuthenticator
from jomiel.log import lg
auth = ThreadAuthenticator(ctx, log=lg())
auth.start()
auth.allow(opts.curve_allow)
# Tell the authenticator to use the client certificates in the
# specified directory.
#
from os.path import abspath
pubdir = abspath(opts.curve_public_key_dir)
auth.configure_curve(domain=opts.curve_domain, location=pubdir)
return auth
def _start_thread_auth(self, socket):
"""
Start the zmq curve thread authenticator.
:param socket: The socket in which to configure the authenticator.
:type socket: zmq.Socket
"""
authenticator = ThreadAuthenticator(self._factory.context)
# Temporary fix until we understand what the problem is
# See https://leap.se/code/issues/7536
time.sleep(0.5)
authenticator.start()
# XXX do not hardcode this here.
authenticator.allow('127.0.0.1')
# tell authenticator to use the certificate in a directory
public_keys_dir = os.path.join(self._config_prefix, PUBLIC_KEYS_PREFIX)
authenticator.configure_curve(domain="*", location=public_keys_dir)
socket.curve_server = True # must come before bind
def _init_zmq(self):
"""
Configure the zmq components and connection.
"""
context = zmq.Context()
socket = context.socket(zmq.REP)
# Start an authenticator for this context.
auth = ThreadAuthenticator(context)
auth.start()
auth.allow('127.0.0.1')
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
public, secret = get_backend_certificates()
socket.curve_publickey = public
socket.curve_secretkey = secret
socket.curve_server = True # must come before bind
socket.bind(self.BIND_ADDR)
self._zmq_socket = socket
def setup_auth():
global _auth
assert _options is not None
auth = _options.get('auth',None)
if auth is None:
return
base_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),'..'))
try:
_auth = ThreadAuthenticator(_zctx)
_auth.start()
whitelist = auth.get('whitelist',None)
if whitelist is not None:
_auth.allow(whitelist)
public_path = auth.get('public_key_dir','public_keys')
_auth.configure_curve(domain='*',location=getExistsPath(base_dir,public_path))
private_dir = getExistsPath(base_dir,auth.get('private_key_dir','private_keys'))
private_key = os.path.join(private_dir,auth.get('private_key_file','server.key_secret'))
server_public,server_private = zmq.auth.load_certificate(private_key)
_sock.curve_secretkey = server_private
_sock.curve_publickey = server_public
_sock.curve_server = True
except:
_auth.stop()
_auth = None
class Actor(object):
'''The actor class implements all the management and control functions over its components
:param gModel: the JSON-based dictionary holding the model for the app this actor belongs to.
:type gModel: dict
:param gModelName: the name of the top-level model for the app
:type gModelName: str
:param aName: name of the actor. It is an index into the gModel that points to the part of the model specific to the actor
:type aName: str
:param sysArgv: list of arguments for the actor: -key1 value1 -key2 value2 ...
:type list:
'''
def __init__(self, gModel, gModelName, aName, sysArgv):
'''
Constructor
'''
self.logger = logging.getLogger(__name__)
self.inst_ = self
self.appName = gModel["name"]
self.modelName = gModelName
self.name = aName
self.pid = os.getpid()
self.uuid = None
self.setupIfaces()
# Assumption : pid is a 4 byte int
self.actorID = ipaddress.IPv4Address(
self.globalHost).packed + self.pid.to_bytes(4, 'big')
self.suffix = ""
if aName not in gModel["actors"]:
raise BuildError('Actor "%s" unknown' % aName)
self.model = gModel["actors"][
aName] # Fetch the relevant content from the model
self.INT_RE = re.compile(r"^[-]?\d+$")
self.parseParams(sysArgv)
# Use czmq's context
czmq_ctx = Zsys.init()
self.context = zmq.Context.shadow(czmq_ctx.value)
Zsys.handler_reset() # Reset previous signal handler
# Context for app sockets
self.appContext = zmq.Context()
if Config.SECURITY:
(self.public_key,
self.private_key) = zmq.auth.load_certificate(const.appCertFile)
hosts = ['127.0.0.1']
try:
with open(const.appDescFile, 'r') as f:
content = yaml.load(f)
hosts += content.hosts
except:
pass
self.auth = ThreadAuthenticator(self.appContext)
self.auth.start()
self.auth.allow(*hosts)
self.auth.configure_curve(domain='*',
location=zmq.auth.CURVE_ALLOW_ANY)
else:
(self.public_key, self.private_key) = (None, None)
self.auth = None
self.appContext = self.context
try:
if os.path.isfile(const.logConfFile) and os.access(
const.logConfFile, os.R_OK):
spdlog_setup.from_file(const.logConfFile)
except Exception as e:
self.logger.error("error while configuring componentLogger: %s" %
repr(e))
messages = gModel[
"messages"] # Global message types (global on the network)
self.messageNames = []
for messageSpec in messages:
self.messageNames.append(messageSpec["name"])
locals_ = self.model[
"locals"] # Local message types (local to the host)
self.localNames = []
for messageSpec in locals_:
self.localNames.append(messageSpec["type"])
internals = self.model[
"internals"] # Internal message types (internal to the actor process)
self.internalNames = []
for messageSpec in internals:
self.internalNames.append(messageSpec["type"])
self.components = {}
instSpecs = self.model["instances"]
compSpecs = gModel["components"]
ioSpecs = gModel["devices"]
for instName in instSpecs: # Create the component instances: the 'parts'
instSpec = instSpecs[instName]
instType = instSpec['type']
if instType in compSpecs:
typeSpec = compSpecs[instType]
ioComp = False
elif instType in ioSpecs:
typeSpec = ioSpecs[instType]
ioComp = True
else:
raise BuildError(
'Component type "%s" for instance "%s" is undefined' %
(instType, instName))
instFormals = typeSpec['formals']
instActuals = instSpec['actuals']
instArgs = self.buildInstArgs(instName, instFormals, instActuals)
# Check whether the component is C++ component
ccComponentFile = 'lib' + instType.lower() + '.so'
ccComp = os.path.isfile(ccComponentFile)
try:
if not ioComp:
if ccComp:
modObj = importlib.import_module('lib' +
instType.lower())
self.components[instName] = modObj.create_component_py(
self, self.model, typeSpec, instName, instType,
instArgs, self.appName, self.name)
else:
self.components[instName] = Part(
self, typeSpec, instName, instType, instArgs)
else:
self.components[instName] = Peripheral(
self, typeSpec, instName, instType, instArgs)
except Exception as e:
traceback.print_exc()
self.logger.error("Error while constructing part '%s.%s': %s" %
(instType, instName, str(e)))
def getParameterValueType(self, param, defaultType):
''' Infer the type of a parameter from its value unless a default type is provided. \
In the latter case the parameter's value is converted to that type.
:param param: a parameter value
:type param: one of bool,int,float,str
:param defaultType:
:type defaultType: one of bool,int,float,str
:return: a pair (value,type)
:rtype: tuple
'''
paramValue, paramType = None, None
if defaultType != None:
if defaultType == str:
paramValue, paramType = param, str
elif defaultType == int:
paramValue, paramType = int(param), int
elif defaultType == float:
paramValue, paramType = float(param), float
elif defaultType == bool:
paramType = bool
paramValue = False if param == "False" else True if param == "True" else None
paramValue, paramType = bool(param), float
else:
if param == 'True':
paramValue, paramType = True, bool
elif param == 'False':
paramValue, paramType = True, bool
elif self.INT_RE.match(param) is not None:
paramValue, paramType = int(param), int
else:
try:
paramValue, paramType = float(param), float
except:
paramValue, paramType = str(param), str
return (paramValue, paramType)
def parseParams(self, sysArgv):
'''Parse actor arguments from the command line
Compares the actual arguments to the formal arguments (from the model) and
fills out the local parameter table accordingly. Generates a warning on
extra arguments and raises an exception on required but missing ones.
'''
self.params = {}
formals = self.model["formals"]
optList = []
for formal in formals:
key = formal["name"]
default = None if "default" not in formal else formal["default"]
self.params[key] = default
optList.append("%s=" % key)
try:
opts, _args = getopt.getopt(sysArgv, '', optList)
except:
self.logger.info("Error parsing actor options %s" % str(sysArgv))
return
for opt in opts:
optName2, optValue = opt
optName = optName2[2:] # Drop two leading dashes
if optName in self.params:
defaultType = None if self.params[optName] == None else type(
self.params[optName])
paramValue, paramType = self.getParameterValueType(
optValue, defaultType)
if self.params[optName] != None:
if paramType != type(self.params[optName]):
raise BuildError(
"Type of default value does not match type of argument %s"
% str((optName, optValue)))
self.params[optName] = paramValue
else:
self.logger.info("Unknown argument %s - ignored" % optName)
for param in self.params:
if self.params[param] == None:
raise BuildError("Required parameter %s missing" % param)
def buildInstArgs(self, instName, formals, actuals):
args = {}
for formal in formals:
argName = formal['name']
argValue = None
actual = next(
(actual for actual in actuals if actual['name'] == argName),
None)
defaultValue = None
if 'default' in formal:
defaultValue = formal['default']
if actual != None:
assert (actual['name'] == argName)
if 'param' in actual:
paramName = actual['param']
if paramName in self.params:
argValue = self.params[paramName]
else:
raise BuildError(
"Unspecified parameter %s referenced in %s" %
(paramName, instName))
elif 'value' in actual:
argValue = actual['value']
else:
raise BuildError("Actual parameter %s has no value" %
argName)
elif defaultValue != None:
argValue = defaultValue
else:
raise BuildError("Argument %s in %s has no defined value" %
(argName, instName))
args[argName] = argValue
return args
def isLocalMessage(self, msgTypeName):
'''Return True if the message type is local
'''
return msgTypeName in self.localNames
def isInnerMessage(self, msgTypeName):
'''Return True if the message type is internal
'''
return msgTypeName in self.internalNames
def getLocalIface(self):
'''Return the IP address of the host-local network interface (usually 127.0.0.1)
'''
return self.localHost
def getGlobalIface(self):
'''Return the IP address of the global network interface
'''
return self.globalHost
def getActorName(self):
'''Return the name of this actor (as defined in the app model)
'''
return self.name
def getAppName(self):
'''Return the name of the app this actor belongs to
'''
return self.appName
def getActorID(self):
'''Returns an ID for this actor.
The actor's id constructed from the host's IP address the actor's process id.
The id is unique for a given host and actor run.
'''
return self.actorID
def setUUID(self, uuid):
'''Sets the UUID for this actor.
The UUID is dynamically generated (by the peer-to-peer network system)
and is unique.
'''
self.uuid = uuid
def getUUID(self):
'''Return the UUID for this actor.
'''
return self.uuid
def setupIfaces(self):
'''Find the IP addresses of the (host-)local and network(-global) interfaces
'''
(globalIPs, globalMACs, _globalNames, localIP) = getNetworkInterfaces()
try:
assert len(globalIPs) > 0 and len(globalMACs) > 0
except:
self.logger.error("Error: no active network interface")
raise
globalIP = globalIPs[0]
globalMAC = globalMACs[0]
self.localHost = localIP
self.globalHost = globalIP
self.macAddress = globalMAC
def setup(self):
'''Perform a setup operation on the actor, after the initial construction
but before the activation of parts
'''
self.logger.info("setup")
self.suffix = self.macAddress
self.disco = DiscoClient(self, self.suffix)
self.disco.start() # Start the discovery service client
self.disco.registerApp(
) # Register this actor with the discovery service
self.logger.info("actor registered with disco")
self.deplc = DeplClient(self, self.suffix)
self.deplc.start()
ok = self.deplc.registerApp()
self.logger.info("actor %s registered with depl" %
("is" if ok else "is not"))
self.controls = {}
self.controlMap = {}
for inst in self.components:
comp = self.components[inst]
control = self.context.socket(zmq.PAIR)
control.bind('inproc://part_' + inst + '_control')
self.controls[inst] = control
self.controlMap[id(control)] = comp
if isinstance(comp, Part):
self.components[inst].setup(control)
else:
self.components[inst].setup()
def registerEndpoint(self, bundle):
'''
Relay the endpoint registration message to the discovery service client
'''
self.logger.info("registerEndpoint")
result = self.disco.registerEndpoint(bundle)
for res in result:
(partName, portName, host, port) = res
self.updatePart(partName, portName, host, port)
def registerDevice(self, bundle):
'''Relay the device registration message to the device interface service client
'''
typeName, args = bundle
msg = (self.appName, self.modelName, typeName, args)
result = self.deplc.registerDevice(msg)
return result
def unregisterDevice(self, bundle):
'''Relay the device unregistration message to the device interface service client
'''
typeName, = bundle
msg = (self.appName, self.modelName, typeName)
result = self.deplc.unregisterDevice(msg)
return result
def activate(self):
'''Activate the parts
'''
self.logger.info("activate")
for inst in self.components:
self.components[inst].activate()
def deactivate(self):
'''Deactivate the parts
'''
self.logger.info("deactivate")
for inst in self.components:
self.components[inst].deactivate()
def recvChannelMessages(self, channel):
'''Collect all messages from the channel queue and return them in a list
'''
msgs = []
while True:
try:
msg = channel.recv(flags=zmq.NOBLOCK)
msgs.append(msg)
except zmq.Again:
break
return msgs
def start(self):
'''
Start and operate the actor (infinite polling loop)
'''
self.logger.info("starting")
self.discoChannel = self.disco.channel # Private channel to the discovery service
self.deplChannel = self.deplc.channel
self.poller = zmq.Poller() # Set up the poller
self.poller.register(self.deplChannel, zmq.POLLIN)
self.poller.register(self.discoChannel, zmq.POLLIN)
for control in self.controls:
self.poller.register(self.controls[control], zmq.POLLIN)
while 1:
sockets = dict(self.poller.poll())
if self.discoChannel in sockets: # If there is a message from a service, handle it
msgs = self.recvChannelMessages(self.discoChannel)
for msg in msgs:
self.handleServiceUpdate(
msg) # Handle message from disco service
del sockets[self.discoChannel]
elif self.deplChannel in sockets:
msgs = self.recvChannelMessages(self.deplChannel)
for msg in msgs:
self.handleDeplMessage(
msg) # Handle message from depl service
del sockets[self.deplChannel]
else: # Handle messages from the components.
toDelete = []
for s in sockets:
if s in self.controls.values():
part = self.controlMap[id(s)]
msg = s.recv_pyobj(
) # receive python object from component
self.handleEventReport(part, msg) # Report event
toDelete += [s]
for s in toDelete:
del sockets[s]
def handleServiceUpdate(self, msgBytes):
'''
Handle a service update message from the discovery service
'''
msgUpd = disco_capnp.DiscoUpd.from_bytes(
msgBytes) # Parse the incoming message
which = msgUpd.which()
if which == 'portUpdate':
msg = msgUpd.portUpdate
client = msg.client
actorHost = client.actorHost
assert actorHost == self.globalHost # It has to be addressed to this actor
actorName = client.actorName
assert actorName == self.name
instanceName = client.instanceName
assert instanceName in self.components # It has to be for a part of this actor
portName = client.portName
scope = msg.scope
socket = msg.socket
host = socket.host
port = socket.port
if scope == "local":
assert host == self.localHost
self.updatePart(instanceName, portName, host,
port) # Update the selected part
def updatePart(self, instanceName, portName, host, port):
'''
Ask a part to update itself
'''
self.logger.info("updatePart %s" % str(
(instanceName, portName, host, port)))
part = self.components[instanceName]
part.handlePortUpdate(portName, host, port)
def handleDeplMessage(self, msgBytes):
'''
Handle a message from the deployment service
'''
msgUpd = deplo_capnp.DeplCmd.from_bytes(
msgBytes) # Parse the incoming message
which = msgUpd.which()
if which == 'resourceMsg':
what = msgUpd.resourceMsg.which()
if what == 'resCPUX':
self.handleCPULimit()
elif what == 'resMemX':
self.handleMemLimit()
elif what == 'resSpcX':
self.handleSpcLimit()
elif what == 'resNetX':
self.handleNetLimit()
else:
self.logger.error("unknown resource msg from deplo: '%s'" %
what)
pass
elif which == 'reinstateCmd':
self.handleReinstate()
elif which == 'nicStateMsg':
stateMsg = msgUpd.nicStateMsg
state = str(stateMsg.nicState)
self.handleNICStateChange(state)
elif which == 'peerInfoMsg':
peerMsg = msgUpd.peerInfoMsg
state = str(peerMsg.peerState)
uuid = peerMsg.uuid
self.handlePeerStateChange(state, uuid)
else:
self.logger.error("unknown msg from deplo: '%s'" % which)
pass
def handleReinstate(self):
self.logger.info('handleReinstate')
self.poller.unregister(self.discoChannel)
self.disco.reconnect()
self.discoChannel = self.disco.channel
self.poller.register(self.discoChannel, zmq.POLLIN)
for inst in self.components:
self.components[inst].handleReinstate()
def handleNICStateChange(self, state):
'''
Handle the NIC state change message: notify components
'''
self.logger.info("handleNICStateChange")
for component in self.components.values():
component.handleNICStateChange(state)
def handlePeerStateChange(self, state, uuid):
'''
Handle the peer state change message: notify components
'''
self.logger.info("handlePeerStateChange")
for component in self.components.values():
component.handlePeerStateChange(state, uuid)
def handleCPULimit(self):
'''
Handle the case when the CPU limit is exceeded: notify each component.
If the component has defined a handler, it will be called.
'''
self.logger.info("handleCPULimit")
for component in self.components.values():
component.handleCPULimit()
def handleMemLimit(self):
'''
Handle the case when the memory limit is exceeded: notify each component.
If the component has defined a handler, it will be called.
'''
self.logger.info("handleMemLimit")
for component in self.components.values():
component.handleMemLimit()
def handleSpcLimit(self):
'''
Handle the case when the file space limit is exceeded: notify each component.
If the component has defined a handler, it will be called.
'''
self.logger.info("handleSpcLimit")
for component in self.components.values():
component.handleSpcLimit()
def handleNetLimit(self):
'''
Handle the case when the net usage limit is exceeded: notify each component.
If the component has defined a handler, it will be called.
'''
self.logger.info("handleNetLimit")
for component in self.components.values():
component.handleNetLimit()
def handleEventReport(self, part, msg):
'''Handle event report from a part
The event report is forwarded to the deplo service.
'''
partName = part.getName()
typeName = part.getTypeName()
bundle = (
partName,
typeName,
) + (msg, )
self.deplc.reportEvent(bundle)
def terminate(self):
'''Terminate all functions of the actor.
Terminate all components, and connections to the deplo/disco services.
Finally exit the process.
'''
self.logger.info("terminating")
for component in self.components.values():
component.terminate()
time.sleep(1.0)
self.deplc.terminate()
self.disco.terminate()
if self.auth:
self.auth.stop()
# Clean up everything
# self.context.destroy()
# time.sleep(1.0)
self.logger.info("terminated")
os._exit(0)
class StupidNode:
pubkey = privkey = None
channel = "" # subscription filter or something (I think)
PORTS = 4 # as we add or remove ports, make sure this is the number of ports a StupidNode uses
def __init__(self, endpoint="*", identity=None, keyring=DEFAULT_KEYRING):
self.keyring = keyring
self.endpoint = (endpoint if isinstance(endpoint, Endpoint) else
Endpoint(endpoint))
self.endpoints = list()
self.identity = identity or f"{gethostname()}-{self.endpoint.pub}"
self.log = logging.getLogger(f"{self.identity}")
self.log.debug("begin node setup / creating context")
self.ctx = zmq.Context()
self.cleartext_ctx = zmq.Context()
self.start_auth()
self.log.debug("creating sockets")
self.pub = self.mk_socket(zmq.PUB)
self.router = self.mk_socket(zmq.ROUTER)
self.router.router_mandatory = (
1 # one of the few opts that can be set after bind()
)
self.rep = self.mk_socket(zmq.REP, enable_curve=False)
self.sub = list()
self.dealer = list()
self.log.debug("binding sockets")
self.bind(self.pub)
self.bind(self.router)
self.bind(self.rep, enable_curve=False)
self.log.debug("registering polling")
self.poller = zmq.Poller()
self.poller.register(self.router, zmq.POLLIN)
self.log.debug("configuring interrupt signal")
signal.signal(signal.SIGINT, self.interrupt)
self.log.debug("configuring WAI Reply Thread")
self._who_are_you_thread = Thread(
target=self.who_are_you_reply_machine)
self._who_are_you_continue = True
self._who_are_you_thread.start()
self.route_queue = deque(list(), ROUTE_QUEUE_LEN)
self.routes = dict()
self.log.debug("node setup complete")
def who_are_you_reply_machine(self):
while self._who_are_you_continue:
if self.rep.poll(200):
self.log.debug("wai polled, trying to recv")
msg = self.rep.recv()
ttype = zmq_socket_type_name(self.rep)
self.log.debug('received "%s" over %s socket', msg, ttype)
msg = [self.identity.encode(), self.pubkey]
self.log.debug('sending "%s" as reply over %s socket', msg,
ttype)
self.rep.send_multipart(msg)
self.log.debug("wai thread seems finished, loop broken")
def start_auth(self):
self.log.debug("starting auth thread")
self.auth = ThreadAuthenticator(self.ctx)
self.auth.start()
self.auth.allow("127.0.0.1")
self.auth.configure_curve(domain="*", location=self.keyring)
self.load_or_create_key()
@property
def key_basename(self):
return scrub_identity_name_for_certfile(self.identity)
@property
def key_filename(self):
return os.path.join(self.keyring, self.key_basename + ".key")
@property
def secret_key_filename(self):
return self.key_filename + "_secret"
def load_key(self):
self.log.debug("loading node key-pair")
self.pubkey, self.privkey = zmq.auth.load_certificate(
self.secret_key_filename)
def load_or_create_key(self):
try:
self.load_key()
except IOError as e:
self.log.debug("error loading key: %s", e)
self.log.debug("creating node key-pair")
os.makedirs(self.keyring, mode=0o0700, exist_ok=True)
zmq.auth.create_certificates(self.keyring, self.key_basename)
self.load_key()
def preprocess_message(self, msg, msg_class=TaggedMessage):
if not isinstance(msg, msg_class):
if not isinstance(msg, (list, tuple)):
msg = (msg, )
msg = msg_class(*msg, name=self.identity)
rmsg = repr(msg)
emsg = msg.encode()
return msg, rmsg, emsg
def route_failed(self, msg):
if not isinstance(msg, RoutedMessage):
raise TypeError("msg must already be a RoutedMessage")
msg.failures += 1
if msg.failures <= 5:
self.log.debug("(re)queueing %s for later delivery", repr(msg))
if len(self.route_queue) == self.route_queue.maxlen:
self.log.error("route_queue full, discarding %s",
repr(self.route_queue[0]))
self.route_queue.append(msg)
else:
self.log.error("discarding %s after %d failures", repr(msg),
msg.failures)
def route_message(self, to, msg):
if isinstance(to, StupidNode):
to = to.identity
if isinstance(to, (list, tuple)):
to = to[-1]
R = self.routes.get(to)
if R:
to = (R[0], to)
if isinstance(msg, RoutedMessage):
msg.to = to
else:
# preprocess passes *msg to msg_class() -- ie, RoutedMessage(to, *msg)
if isinstance(msg, list):
msg = tuple(msg)
elif not isinstance(msg, tuple):
msg = (msg, )
msg = (to, ) + msg
tmsg, rmsg, emsg = self.preprocess_message(msg,
msg_class=RoutedMessage)
self.log.debug("routing message %s -- encoding: %s", rmsg, emsg)
try:
self.router.send_multipart(emsg)
except zmq.error.ZMQError as zmq_e:
self.log.debug("route to %s failed: %s", to, zmq_e)
if "Host unreachable" not in str(zmq_e):
raise
self.route_failed(tmsg)
def deal_message(self, msg):
self.log.debug(
"dealing message (actually publishing with no_publish=True)")
self.publish_message(msg, no_publish=True)
def publish_message(self,
msg,
no_deal=False,
no_deal_to=None,
no_publish=False):
tmsg, rmsg, emsg = self.preprocess_message(msg)
self.log.debug(
"publishing message %s no_publish=%s, no_deal=%s, no_deal_to=%s",
rmsg,
no_publish,
no_deal,
no_deal_to,
)
self.local_workflow(tmsg)
if not no_publish:
self.pub.send_multipart(emsg)
if no_deal:
return
if no_deal_to is None:
ok_send = lambda x: True
elif callable(no_deal_to):
ok_send = no_deal_to
elif isinstance(no_deal_to, zmq.Socket):
npt_i = self.dealer.index(no_deal_to)
ok_send = lambda x: x != npt_i
elif isinstance(no_deal_to, int):
ok_send = lambda x: x != no_deal_to
elif isinstance(no_deal_to, (list, tuple)):
ok_send = lambda x: x not in no_deal_to
for i, sock in enumerate(self.dealer):
if ok_send(i):
self.log.debug("dealing message %s to %s", rmsg,
self.endpoints[i])
sock.send_multipart(emsg)
else:
self.log.debug("not sending %s to %s", rmsg, self.endpoints[i])
def mk_socket(self, stype, enable_curve=True):
# defaults:
# socket.setsockopt(zmq.LINGER, -1) # infinite
# socket.setsockopt(zmq.IDENTITY, None)
# socket.setsockopt(zmq.TCP_KEEPALIVE, -1)
# socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, -1)
# socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, -1)
# socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, -1)
# socket.setsockopt(zmq.RECONNECT_IVL, 100)
# socket.setsockopt(zmq.RECONNECT_IVL_MAX, 0) # 0 := always use IVL
# the above can be accessed as attributes instead (they are case
# insensitive, we choose lower case below so it looks like boring
# python)
if enable_curve:
socket = self.ctx.socket(stype)
self.log.debug("create %s socket in crypto context",
zmq_socket_type_name(stype))
else:
socket = self.cleartext_ctx.socket(stype)
self.log.debug("create %s socket in cleartext context",
zmq_socket_type_name(stype))
socket.linger = 1
socket.identity = self.identity.encode()
socket.reconnect_ivl = 1000
socket.reconnect_ivl_max = 10000
if enable_curve:
socket.curve_secretkey = self.privkey
socket.curve_publickey = self.pubkey
return socket
def local_workflow(self, msg):
self.log.debug("start local_workflow %s", repr(msg))
msg = self.local_react(msg)
if msg:
msg = self.all_react(msg)
return msg
def sub_workflow(self, socket):
idx = self.sub.index(socket)
enp = self.endpoints[idx]
msg = self.sub_receive(socket, idx)
self.log.debug("start sub_workflow (idx=%d -> endpoint=%s) %s", idx,
enp, repr(msg))
for react in (self.sub_react, self.nonlocal_react, self.all_react):
if msg:
msg = react(msg, idx=idx)
self.log.debug("end sub_workflow")
return msg
def router_workflow(self):
msg = self.router_receive()
self.log.debug("start router_workflow %s", repr(msg))
for react in (self.router_react, self.nonlocal_react, self.all_react):
if not msg:
break
msg = react(msg)
self.log.debug("end router_workflow")
return msg
def dealer_workflow(self, socket):
idx = self.dealer.index(socket)
enp = self.endpoints[idx]
msg = self.dealer_receive(socket, idx)
self.log.debug("start deal_workflow (idx=%d -> endpoint=%s) %s", idx,
enp, repr(msg))
for react in (self.dealer_react, self.nonlocal_react, self.all_react):
if not msg:
break
msg = react(msg, idx=idx)
self.log.debug("end deal_workflow")
return msg
def sub_receive(self, socket, idx): # pylint: disable=unused-argument
return TaggedMessage(*socket.recv_multipart())
def dealer_receive(self, socket, idx): # pylint: disable=unused-argument
msg = socket.recv_multipart()
rm = RoutedMessage.decode(msg)
if rm:
return rm
# dealer's always receive a routed message if it doesn't appear to be
# routed, then it's simply intended for us. In that case, build a
# tagged message and mark it as non-publish
msg = TaggedMessage(*msg)
msg.publish_mark = False
return msg
def router_receive(self):
# we ignore the source ID (in '_') and just believe the msg.tag.name ... it's
# roughly the same thing anyway
_, *msg = self.router.recv_multipart()
rm = RoutedMessage.decode(msg)
if rm:
return rm
return TaggedMessage(*msg)
def all_react(self, msg, idx=None): # pylint: disable=unused-argument
return msg
def sub_react(self, msg, idx=None): # pylint: disable=unused-argument
return msg
def dealer_react(self, msg, idx=None): # pylint: disable=unused-argument
return msg
def router_react(self, msg):
return msg
def nonlocal_react(self, msg, idx=None):
if isinstance(msg, RoutedMessage):
msg = self.routed_react(msg, idx=idx)
return msg
def local_react(self, msg):
return msg
def routed_react(self, msg, idx=None): # pylint: disable=unused-argument
return False
def poll(self, timeo=500, other_cb=None):
"""Check to see if there's any incoming messages. If anything seems ready to receive,
invoke the related workflow or invoke other_cb (if given) on the socket item.
"""
items = dict(self.poller.poll(timeo))
ret = list()
for item in items:
if items[item] != zmq.POLLIN:
continue
if item in self.sub:
res = self.sub_workflow(item)
elif item in self.dealer:
res = self.dealer_workflow(item)
elif item is self.router:
res = self.router_workflow()
elif callable(other_cb):
res = other_cb(item)
else:
res = None
if False and isinstance(item, zmq.Socket):
self.log.error(
"no workflow defined for socket of type %s -- received: %s",
zmq_socket_type_name(item),
item.recv_multipart(),
)
else:
self.log.error(
"no workflow defined for socket of type %s -- regarding as fatal",
zmq_socket_type_name(item),
)
# note: this normally doesn't trigger an exit... thanks threading
raise Exception("unhandled poll item")
if isinstance(res, TaggedMessage):
ret.append(res)
return ret
def interrupt(self, signo, eframe): # pylint: disable=unused-argument
print(" kaboom")
self.closekill()
sys.exit(0)
def closekill(self):
if hasattr(self, "auth") and self.auth is not None:
if self.auth.is_alive():
self.log.debug("trying to stop auth thread")
self.auth.stop()
self.log.debug("auth thread seems to have stopped")
del self.auth
if hasattr(self, "_who_are_you_thread"):
if self._who_are_you_thread.is_alive():
self.log.debug("WAI Thread seems to be alive, trying to join")
self._who_are_you_continue = False
self._who_are_you_thread.join()
self.log.debug("WAI Thread seems to jave joined us.")
del self._who_are_you_thread
if hasattr(self, "cleartext_ctx"):
self.log.debug("destroying cleartext context")
self.cleartext_ctx.destroy(1)
del self.cleartext_ctx
if hasattr(self, "ctx"):
self.log.debug("destroying crypto context")
self.ctx.destroy(1)
del self.ctx
def __del__(self):
self.log.debug("%s is being deleted", self)
self.closekill()
def bind(self, socket, enable_curve=True):
if enable_curve:
socket.curve_server = True # must come before bind
try:
f = self.endpoint.format(socket.type)
socket.bind(f)
except zmq.ZMQError as e:
raise zmq.ZMQError(f"unable to bind {f}: {e}") from e
def who_are_you_request(self, endpoint):
req = self.mk_socket(zmq.REQ, enable_curve=False)
req.connect(endpoint.format(zmq.REQ))
msg = b"Who are you?"
self.log.debug("sending cleartext request: %s", msg)
req.send(msg)
self.log.debug("waiting for reply")
res = req.recv_multipart()
self.log.debug("received reply: %s", res)
if len(res) == 2:
return res
req.close()
return None, None
def pubkey_pathname(self, node_id):
if isinstance(node_id, Endpoint):
node_id = Endpoint.host
fname = scrub_identity_name_for_certfile(node_id) + ".key"
pname = os.path.join(self.keyring, fname)
return pname
def learn_or_load_endpoint_pubkey(self, endpoint):
epubk_pname = self.pubkey_pathname(endpoint)
if not os.path.isfile(epubk_pname):
self.log.debug(
"%s does not exist yet, trying to learn certificate",
epubk_pname)
node_id, public_key = self.who_are_you_request(endpoint)
if node_id:
endpoint.identity = node_id.decode()
epubk_pname = self.pubkey_pathname(node_id)
if not os.path.isfile(epubk_pname):
with open(epubk_pname, "wb") as fh:
fh.write(
b"# generated via rep/req pubkey transfer\n\n")
fh.write(b"metadata\n")
# NOTE: in zmq/auth/certs.py's _write_key_file,
# metadata should be key-value pairs; roughly like the
# following (although with their particular py2/py3
# nerosis edited out):
#
# f.write('metadata\n')
# for k,v in metadata.items():
# f.write(f" {k} = {v}\n")
fh.write(b"curve\n")
fh.write(b' public-key = "')
fh.write(public_key)
fh.write(b'"')
self.log.debug("loading certificate %s", epubk_pname)
ret, _ = zmq.auth.load_certificate(epubk_pname)
return ret
def connect_to_endpoints(self, *endpoints):
self.log.debug("connecting remote endpoints")
for item in endpoints:
self.connect_to_endpoint(item)
self.log.debug("remote endpoints connected")
return self
def _create_connected_socket(self,
endpoint,
stype,
pubkey,
preconnect=None):
self.log.debug("creating %s socket to endpoint=%s",
zmq_socket_type_name(stype), endpoint)
s = self.mk_socket(stype)
s.curve_serverkey = pubkey
if callable(preconnect):
preconnect(s)
s.connect(endpoint.format(stype))
return s
def connect_to_endpoint(self, endpoint):
if isinstance(endpoint, StupidNode):
endpoint = endpoint.endpoint
elif not isinstance(endpoint, Endpoint):
endpoint = Endpoint(endpoint)
self.log.debug("learning or loading endpoint=%s pubkey", endpoint)
epk = self.learn_or_load_endpoint_pubkey(endpoint)
sos = lambda s: s.setsockopt_string(zmq.SUBSCRIBE, self.channel)
sub = self._create_connected_socket(endpoint, zmq.SUB, epk, sos)
self.poller.register(sub, zmq.POLLIN)
self.sub.append(sub)
deal = self._create_connected_socket(endpoint, zmq.DEALER, epk)
self.poller.register(deal, zmq.POLLIN)
self.dealer.append(deal)
self.endpoints.append(endpoint)
return self
def __repr__(self):
return f"{self.__class__.__name__}({self.identity})"
class dataTransfer():
def __init__(self,
connectionType,
signalHost=None,
useLog=False,
context=None):
if useLog:
self.log = logging.getLogger("dataTransferAPI")
elif useLog == None:
self.log = noLoggingFunction()
else:
self.log = loggingFunction()
# ZMQ applications always start by creating a context,
# and then using that for creating sockets
# (source: ZeroMQ, Messaging for Many Applications by Pieter Hintjens)
if context:
self.context = context
self.extContext = True
else:
self.context = zmq.Context()
self.extContext = False
self.signalHost = signalHost
self.signalPort = "50000"
self.requestPort = "50001"
self.dataHost = None
self.dataPort = None
self.signalSocket = None
self.dataSocket = None
self.requestSocket = None
self.poller = zmq.Poller()
self.auth = None
self.targets = None
self.supportedConnections = [
"stream", "streamMetadata", "queryNext", "queryMetadata"
]
self.signalExchanged = None
self.streamStarted = None
self.queryNextStarted = None
self.socketResponseTimeout = 1000
if connectionType in self.supportedConnections:
self.connectionType = connectionType
else:
raise NotSupported("Chosen type of connection is not supported.")
# targets: [host, port, prio] or [[host, port, prio], ...]
def initiate(self, targets):
if type(targets) != list:
self.stop()
raise FormatError("Argument 'targets' must be list.")
if not self.context:
self.context = zmq.Context()
self.extContext = False
signal = None
# Signal exchange
if self.connectionType == "stream":
signalPort = self.signalPort
signal = "START_STREAM"
elif self.connectionType == "streamMetadata":
signalPort = self.signalPort
signal = "START_STREAM_METADATA"
elif self.connectionType == "queryNext":
signalPort = self.signalPort
signal = "START_QUERY_NEXT"
elif self.connectionType == "queryMetadata":
signalPort = self.signalPort
signal = "START_QUERY_METADATA"
self.log.debug("Create socket for signal exchange...")
if self.signalHost:
self.__createSignalSocket(signalPort)
else:
self.stop()
raise ConnectionFailed("No host to send signal to specified.")
self.__setTargets(targets)
message = self.__sendSignal(signal)
if message and message == "VERSION_CONFLICT":
self.stop()
raise VersionError("Versions are conflicting.")
elif message and message == "NO_VALID_HOST":
self.stop()
raise AuthenticationFailed("Host is not allowed to connect.")
elif message and message == "CONNECTION_ALREADY_OPEN":
self.stop()
raise CommunicationFailed("Connection is already open.")
elif message and message == "NO_VALID_SIGNAL":
self.stop()
raise CommunicationFailed(
"Connection type is not supported for this kind of sender.")
# if there was no response or the response was of the wrong format, the receiver should be shut down
elif message and message.startswith(signal):
self.log.info("Received confirmation ...")
self.signalExchanged = signal
else:
raise CommunicationFailed("Sending start signal ...failed.")
def __createSignalSocket(self, signalPort):
# To send a notification that a Displayer is up and running, a communication socket is needed
# create socket to exchange signals with Sender
self.signalSocket = self.context.socket(zmq.REQ)
# time to wait for the sender to give a confirmation of the signal
# self.signalSocket.RCVTIMEO = self.socketResponseTimeout
connectionStr = "tcp://" + str(self.signalHost) + ":" + str(signalPort)
try:
self.signalSocket.connect(connectionStr)
self.log.info("signalSocket started (connect) for '" +
connectionStr + "'")
except:
self.log.error("Failed to start signalSocket (connect): '" +
connectionStr + "'")
raise
# using a Poller to implement the signalSocket timeout (in older ZMQ version there is no option RCVTIMEO)
self.poller.register(self.signalSocket, zmq.POLLIN)
def __setTargets(self, targets):
self.targets = []
# [host, port, prio]
if len(targets) == 3 and type(targets[0]) != list and type(
targets[1]) != list and type(targets[2]) != list:
host, port, prio = targets
self.targets = [[host + ":" + port, prio, [""]]]
# [host, port, prio, suffixes]
elif len(targets) == 4 and type(targets[0]) != list and type(
targets[1]) != list and type(targets[2]) != list and type(
targets[3]) == list:
host, port, prio, suffixes = targets
self.targets = [[host + ":" + port, prio, suffixes]]
# [[host, port, prio], ...] or [[host, port, prio, suffixes], ...]
else:
for t in targets:
if type(t) == list and len(t) == 3:
host, port, prio = t
self.targets.append([host + ":" + port, prio, [""]])
elif type(t) == list and len(t) == 4 and type(t[3]):
host, port, prio, suffixes = t
self.targets.append([host + ":" + port, prio, suffixes])
else:
self.stop()
self.log.debug("targets=" + str(targets))
raise FormatError("Argument 'targets' is of wrong format.")
def __sendSignal(self, signal):
if not signal:
return
# Send the signal that the communication infrastructure should be established
self.log.info("Sending Signal")
sendMessage = [__version__, signal]
trg = cPickle.dumps(self.targets)
sendMessage.append(trg)
# sendMessage = [__version__, signal, self.dataHost, self.dataPort]
self.log.debug("Signal: " + str(sendMessage))
try:
self.signalSocket.send_multipart(sendMessage)
except:
self.log.error("Could not send signal")
raise
message = None
try:
socks = dict(self.poller.poll(self.socketResponseTimeout))
except:
self.log.error("Could not poll for new message")
raise
# if there was a response
if self.signalSocket in socks and socks[
self.signalSocket] == zmq.POLLIN:
try:
# Get the reply.
message = self.signalSocket.recv()
self.log.info("Received answer to signal: " + str(message))
except:
self.log.error("Could not receive answer to signal")
raise
return message
def start(self, dataSocket=False, whitelist=None):
# Receive data only from whitelisted nodes
if whitelist:
if type(whitelist) == list:
self.auth = ThreadAuthenticator(self.context)
self.auth.start()
for host in whitelist:
try:
if host == "localhost":
ip = [socket.gethostbyname(host)]
else:
hostname, tmp, ip = socket.gethostbyaddr(host)
self.log.debug("Allowing host " + host + " (" +
str(ip[0]) + ")")
self.auth.allow(ip[0])
except:
self.log.error("Error was: ", exc_info=True)
raise AuthenticationFailed(
"Could not get IP of host " + host)
else:
raise FormatError("Whitelist has to be a list of IPs")
socketIdToConnect = self.streamStarted or self.queryNextStarted
if socketIdToConnect:
self.log.info("Reopening already started connection.")
else:
ip = "0.0.0.0" #TODO use IP of hostname?
host = ""
port = ""
if dataSocket:
if type(dataSocket) == list:
socketIdToConnect = dataSocket[0] + ":" + dataSocket[1]
host = dataSocket[0]
ip = socket.gethostbyaddr(host)[2][0]
port = dataSocket[1]
else:
port = str(dataSocket)
host = socket.gethostname()
socketId = host + ":" + port
ipFromHost = socket.gethostbyaddr(host)[2]
if len(ipFromHost) == 1:
ip = ipFromHost[0]
elif len(self.targets) == 1:
host, port = self.targets[0][0].split(":")
ipFromHost = socket.gethostbyaddr(host)[2]
if len(ipFromHost) == 1:
ip = ipFromHost[0]
else:
raise FormatError(
"Multipe possible ports. Please choose which one to use.")
socketId = host + ":" + port
socketIdToConnect = ip + ":" + port
# socketIdToConnect = "[" + ip + "]:" + port
self.dataSocket = self.context.socket(zmq.PULL)
# An additional socket is needed to establish the data retriving mechanism
connectionStr = "tcp://" + socketIdToConnect
if whitelist:
self.dataSocket.zap_domain = b'global'
try:
# self.dataSocket.ipv6 = True
self.dataSocket.bind(connectionStr)
# self.dataSocket.bind("tcp://[2003:ce:5bc0:a600:fa16:54ff:fef4:9fc0]:50102")
self.log.info("Data socket of type " + self.connectionType +
" started (bind) for '" + connectionStr + "'")
except:
self.log.error("Failed to start Socket of type " +
self.connectionType + " (bind): '" + connectionStr +
"'",
exc_info=True)
raise
self.poller.register(self.dataSocket, zmq.POLLIN)
if self.connectionType in ["queryNext", "queryMetadata"]:
self.requestSocket = self.context.socket(zmq.PUSH)
# An additional socket is needed to establish the data retriving mechanism
connectionStr = "tcp://" + self.signalHost + ":" + self.requestPort
try:
self.requestSocket.connect(connectionStr)
self.log.info("Request socket started (connect) for '" +
connectionStr + "'")
except:
self.log.error("Failed to start Socket of type " +
self.connectionType + " (connect): '" +
connectionStr + "'",
exc_info=True)
raise
self.queryNextStarted = socketId
else:
self.streamStarted = socketId
##
#
# Receives or queries for new files depending on the connection initialized
#
# returns either
# the newest file
# (if connection type "queryNext" or "stream" was choosen)
# the path of the newest file
# (if connection type "queryMetadata" or "streamMetadata" was choosen)
#
##
def get(self, timeout=None):
if not self.streamStarted and not self.queryNextStarted:
self.log.info(
"Could not communicate, no connection was initialized.")
return None, None
if self.queryNextStarted:
sendMessage = ["NEXT", self.queryNextStarted]
try:
self.requestSocket.send_multipart(sendMessage)
except Exception as e:
self.log.error("Could not send request to requestSocket",
exc_info=True)
return None, None
while True:
# receive data
if timeout:
try:
socks = dict(self.poller.poll(timeout))
except:
self.log.error("Could not poll for new message")
raise
else:
try:
socks = dict(self.poller.poll())
except:
self.log.error("Could not poll for new message")
raise
# if there was a response
if self.dataSocket in socks and socks[
self.dataSocket] == zmq.POLLIN:
try:
multipartMessage = self.dataSocket.recv_multipart()
except:
self.log.error("Receiving data..failed.", exc_info=True)
return [None, None]
if multipartMessage[0] == b"ALIVE_TEST":
continue
elif len(multipartMessage) < 2:
self.log.error(
"Received mutipart-message is too short. Either config or file content is missing."
)
self.log.debug("multipartMessage=" +
str(mutipartMessage)[:100])
return [None, None]
# extract multipart message
try:
metadata = cPickle.loads(multipartMessage[0])
except:
self.log.error(
"Could not extract metadata from the multipart-message.",
exc_info=True)
metadata = None
#TODO validate multipartMessage (like correct dict-values for metadata)
try:
payload = multipartMessage[1]
except:
self.log.warning(
"An empty file was received within the multipart-message",
exc_info=True)
payload = None
return [metadata, payload]
else:
self.log.warning("Could not receive data in the given time.")
if self.queryNextStarted:
try:
self.requestSocket.send_multipart(
["CANCEL", self.queryNextStarted])
except Exception as e:
self.log.error("Could not cancel the next query",
exc_info=True)
return [None, None]
def store(self, targetBasePath, dataObject):
if type(dataObject) is not list and len(dataObject) != 2:
raise FormatError("Wrong input type for 'store'")
payloadMetadata = dataObject[0]
payload = dataObject[1]
if type(payloadMetadata) is not dict:
raise FormatError("payload: Wrong input format in 'store'")
#save all chunks to file
while True:
#TODO check if payload != cPickle.dumps(None) ?
if payloadMetadata and payload:
#append to file
try:
self.log.debug(
"append to file based on multipart-message...")
#TODO: save message to file using a thread (avoids blocking)
#TODO: instead of open/close file for each chunk recyle the file-descriptor for all chunks opened
self.__appendChunksToFile(targetBasePath, payloadMetadata,
payload)
self.log.debug(
"append to file based on multipart-message...success.")
except KeyboardInterrupt:
self.log.info(
"KeyboardInterrupt detected. Unable to append multipart-content to file."
)
break
except Exception, e:
self.log.error(
"Unable to append multipart-content to file.",
exc_info=True)
self.log.debug(
"Append to file based on multipart-message...failed.")
if len(payload) < payloadMetadata["chunkSize"]:
#indicated end of file. Leave loop
filename = self.generateTargetFilepath(
targetBasePath, payloadMetadata)
fileModTime = payloadMetadata["fileModTime"]
self.log.info("New file with modification time " +
str(fileModTime) + " received and saved: " +
str(filename))
break
try:
[payloadMetadata, payload] = self.get()
except:
self.log.error("Getting data failed.", exc_info=True)
break
class Device(Actor):
'''
The actor class implements all the management and control functions over its components
'''
def __init__(self, gModel, gModelName, dName, qName, sysArgv):
'''
Constructor
:param dName: device type name
:type dName: str
:param qName: qualified name of the device instance: 'actor.inst'
:type qName: str
'''
self.logger = logging.getLogger(__name__)
self.inst_ = self
self.appName = gModel["name"]
self.modelName = gModelName
aName,iName = qName.split('.')
self.name = qName
self.iName = iName
self.dName = dName
self.pid = os.getpid()
self.uuid = None
self.suffix = ""
self.setupIfaces()
# Assumption : pid is a 4 byte int
self.actorID = ipaddress.IPv4Address(self.globalHost).packed + self.pid.to_bytes(4, 'big')
if dName not in gModel["devices"]:
raise BuildError('Device "%s" unknown' % dName)
# In order to make the rest of the code work, we build an actor model for the device
devModel = gModel["devices"][dName]
self.model = {} # The made-up actor model
formals = devModel["formals"] # Formals are the same as those of the device (component)
self.model["formals"] = formals
devInst = { "type": dName } # There is a single instance, containing the device component
actuals = []
for arg in formals:
name = arg["name"]
actual = {}
actual["name"] = name
actual["param"] = name
actuals.append(actual)
devInst["actuals"] = actuals
self.model["instances"] = { iName: devInst} # Single instance (under iName)
aModel = gModel["actors"][aName]
self.model["locals"] = aModel["locals"] # Locals
self.model["internals"] = aModel["internals"] # Internals
self.INT_RE = re.compile(r"^[-]?\d+$")
self.parseParams(sysArgv)
# Use czmq's context
czmq_ctx = Zsys.init()
self.context = zmq.Context.shadow(czmq_ctx.value)
Zsys.handler_reset() # Reset previous signal
# Context for app sockets
self.appContext = zmq.Context()
if Config.SECURITY:
(self.public_key, self.private_key) = zmq.auth.load_certificate(const.appCertFile)
_public = zmq.curve_public(self.private_key)
if(self.public_key != _public):
self.logger.error("bad security key(s)")
raise BuildError("invalid security key(s)")
hosts = ['127.0.0.1']
try:
with open(const.appDescFile, 'r') as f:
content = yaml.load(f, Loader=yaml.Loader)
hosts += content.hosts
except:
self.logger.error("Error loading app descriptor:s", str(sys.exc_info()[1]))
self.auth = ThreadAuthenticator(self.appContext)
self.auth.start()
self.auth.allow(*hosts)
self.auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY)
else:
(self.public_key, self.private_key) = (None, None)
self.auth = None
self.appContext = self.context
try:
if os.path.isfile(const.logConfFile) and os.access(const.logConfFile, os.R_OK):
spdlog_setup.from_file(const.logConfFile)
except Exception as e:
self.logger.error("error while configuring componentLogger: %s" % repr(e))
messages = gModel["messages"] # Global message types (global on the network)
self.messageNames = []
for messageSpec in messages:
self.messageNames.append(messageSpec["name"])
locals_ = self.model["locals"] # Local message types (local to the host)
self.localNames = []
for messageSpec in locals_:
self.localNames.append(messageSpec["type"])
internals = self.model["internals"] # Internal message types (internal to the actor process)
self.internalNames = []
for messageSpec in internals:
self.internalNames.append(messageSpec["type"])
groups = gModel["groups"]
self.groupTypes = {}
for group in groups:
self.groupTypes[group["name"]] = {
"kind": group["kind"],
"message": group["message"],
"timed": group["timed"]
}
self.components = {}
instSpecs = self.model["instances"]
_compSpecs = gModel["components"]
devSpecs = gModel["devices"]
for instName in instSpecs: # Create the component instances: the 'parts'
instSpec = instSpecs[instName]
instType = instSpec['type']
if instType in devSpecs:
typeSpec = devSpecs[instType]
else:
raise BuildError('Device type "%s" for instance "%s" is undefined' % (instType, instName))
instFormals = typeSpec['formals']
instActuals = instSpec['actuals']
instArgs = self.buildInstArgs(instName, instFormals, instActuals)
# Check whether the component is C++ component
ccComponentFile = 'lib' + instType.lower() + '.so'
ccComp = os.path.isfile(ccComponentFile)
try:
if ccComp:
modObj = importlib.import_module('lib' + instType.lower())
self.components[instName] = modObj.create_component_py(self, self.model,
typeSpec, instName,
instType, instArgs,
self.appName, self.name, groups)
else:
self.components[instName] = Part(self, typeSpec, instName, instType, instArgs)
except Exception as e:
traceback.print_exc()
self.logger.error("Error while constructing part '%s.%s': %s" % (instType, instName, str(e)))
def getPortMessageTypes(self, ports, key, kinds, res):
for _name, spec in ports[key].items():
for kind in kinds:
typeName = spec[kind]
res.append({"type": typeName})
def getMessageTypes(self, devModel):
res = []
ports = devModel["ports"]
self.getPortMessageTypes(ports, "pubs", ["type"], res)
self.getPortMessageTypes(ports, "subs", ["type"], res)
self.getPortMessageTypes(ports, "reqs", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "reps", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "clts", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "srvs", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "qrys", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "anss", ["req_type", "rep_type"], res)
return res
def isDevice(self):
return True
def setup(self):
'''
Perform a setup operation on the actor (after the initial construction but before the activation of parts)
'''
self.logger.info("setup")
# self.setupIfaces()
self.suffix = self.macAddress
self.disco = DiscoClient(self, self.suffix)
self.disco.start() # Start the discovery service client
self.disco.registerActor() # Register this actor with the discovery service
self.logger.info("device registered with disco")
self.deplc = DeplClient(self, self.suffix)
self.deplc.start()
ok = self.deplc.registerActor()
self.logger.info("device %s registered with depl" % ("is" if ok else "is not"))
self.controls = { }
self.controlMap = { }
for inst in self.components:
comp = self.components[inst]
control = self.context.socket(zmq.PAIR)
control.bind('inproc://part_' + inst + '_control')
self.controls[inst] = control
self.controlMap[id(control)] = comp
if isinstance(comp, Part):
self.components[inst].setup(control)
else:
self.components[inst].setup()
def terminate(self):
self.logger.info("terminating")
for component in self.components.values():
component.terminate()
# self.devc.terminate()
self.disco.terminate()
# Clean up everything
# self.context.destroy()
time.sleep(1.0)
self.logger.info("terminated")
os._exit(0)
class StratusApp(StratusServerApp):
def __init__(self, core: StratusCore, **kwargs):
StratusServerApp.__init__(self, core, **kwargs)
self.logger = StratusLogger.getLogger()
self.active = True
self.parms = self.getConfigParms('stratus')
self.client_address = self.parms.get("client_address", "*")
self.request_port = self.parms.get("request_port", 4556)
self.response_port = self.parms.get("response_port", 4557)
self.active_handlers = {}
self.getCertDirs()
def getCertDirs(
self
): # These directories are generated by the generate_certificates script
self.cert_dir = self.parms.get("certificate_path",
os.path.expanduser("~/.stratus/zmq"))
self.logger.info(
f"Loading certificates and keys from directory {self.cert_dir}")
self.keys_dir = os.path.join(self.cert_dir, 'certificates')
self.public_keys_dir = os.path.join(self.cert_dir, 'public_keys')
self.secret_keys_dir = os.path.join(self.cert_dir, 'private_keys')
if not (os.path.exists(self.keys_dir)
and os.path.exists(self.public_keys_dir)
and os.path.exists(self.secret_keys_dir)):
from stratus.handlers.zeromq.security.generate_certificates import generate_certificates
generate_certificates(self.cert_dir)
def initSocket(self):
try:
server_secret_file = os.path.join(self.secret_keys_dir,
"server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
# TODO: this is commented to avoid key checking
#self.request_socket.curve_secretkey = server_secret
#self.request_socket.curve_publickey = server_public
#self.request_socket.curve_server = True
self.request_socket.bind("tcp://{}:{}".format(
self.client_address, self.request_port))
self.logger.info(
"@@STRATUS-APP --> Bound authenticated request socket to client at {} on port: {}"
.format(self.client_address, self.request_port))
except Exception as err:
self.logger.error(
"@@STRATUS-APP: Error initializing request socket on {}, port {}: {}"
.format(self.client_address, self.request_port, err))
self.logger.error(traceback.format_exc())
def addHandler(self, clientId, jobId, handler):
self.active_handlers[clientId + "-" + jobId] = handler
return handler
def removeHandler(self, clientId, jobId):
handlerId = clientId + "-" + jobId
try:
del self.active_handlers[handlerId]
except:
self.logger.error("Error removing handler: " + handlerId +
", active handlers = " +
str(list(self.active_handlers.keys())))
def setExeStatus(self, submissionId: str, status: Status):
self.responder.setExeStatus(submissionId, status)
def sendResponseMessage(self, msg: StratusResponse) -> str:
request_args = [msg.id, msg.message]
packaged_msg = "!".join(request_args)
timeStamp = datetime.datetime.now().strftime("MM/dd HH:mm:ss")
self.logger.info(
"@@STRATUS-APP: Sending response {} on request_socket @({}): {}".
format(msg.id, timeStamp, str(msg)))
self.request_socket.send_string(packaged_msg)
return packaged_msg
def initInteractions(self):
try:
self.zmqContext: zmq.Context = zmq.Context()
self.auth = ThreadAuthenticator(self.zmqContext)
self.auth.start()
self.auth.allow("192.168.0.22")
self.auth.allow(self.client_address)
self.auth.configure_curve(
domain='*', location=zmq.auth.CURVE_ALLOW_ANY
) # self.public_keys_dir ) # Use 'location=zmq.auth.CURVE_ALLOW_ANY' for stonehouse security
self.request_socket: zmq.Socket = self.zmqContext.socket(zmq.REP)
self.responder = StratusZMQResponder(
self.zmqContext,
self.response_port,
client_address=self.client_address,
certificate_path=self.cert_dir)
self.initSocket()
self.logger.info(
"@@STRATUS-APP:Listening for requests on port: {}".format(
self.request_port))
except Exception as err:
self.logger.error(
"@@STRATUS-APP: ------------------------------- StratusApp Init error: {} ------------------------------- "
.format(err))
def processResults(self):
completed_workflows = self.responder.processWorkflows(
self.getWorkflows())
for rid in completed_workflows:
self.clearWorkflow(rid)
def processRequests(self):
while self.request_socket.poll(0) != 0:
request_header = self.request_socket.recv_string().strip().strip(
"'")
parts = request_header.split("!")
submissionId = str(parts[0])
rType = str(parts[1])
request: Dict = json.loads(parts[2]) if len(parts) > 2 else ""
try:
self.logger.info(
"@@STRATUS-APP: ### Processing {} request: {}".format(
rType, request))
if rType == "capabilities":
response = self.core.getCapabilities(request["type"])
self.sendResponseMessage(
StratusResponse(submissionId, response))
elif rType == "exe":
if len(parts) <= 2:
raise Exception("Missing parameters to exe request")
request["rid"] = submissionId
self.logger.info(
"Processing zmq Request: '{}' '{}' '{}'".format(
submissionId, rType, str(request)))
self.submitWorkflow(
request) # TODO: Send results when tasks complete.
response = {"status": "Executing"}
self.sendResponseMessage(
StratusResponse(submissionId, response))
elif rType == "quit" or rType == "shutdown":
response = {"status": "Terminating"}
self.sendResponseMessage(
StratusResponse(submissionId, response))
self.logger.info(
"@@STRATUS-APP: Received Shutdown Message")
exit(0)
else:
msg = "@@STRATUS-APP: Unknown request type: " + rType
self.logger.info(msg)
response = {"status": "error", "error": msg}
self.sendResponseMessage(
StratusResponse(submissionId, response))
except Exception as ex:
self.processError(submissionId, ex)
def processError(self, rid: str, ex: Exception):
tb = traceback.format_exc()
self.logger.error("@@STRATUS-APP: Execution error: " + str(ex))
self.logger.error(tb)
response = {"status": "error", "error": str(ex), "traceback": tb}
self.sendResponseMessage(StratusResponse(rid, response))
def updateInteractions(self):
self.processRequests()
self.processResults()
def term(self, msg):
self.logger.info("@@STRATUS-APP: !!EDAS Shutdown: " + msg)
self.active = False
self.auth.stop()
self.logger.info("@@STRATUS-APP: QUIT PythonWorkerPortal")
try:
self.request_socket.close()
except Exception:
pass
self.logger.info("@@STRATUS-APP: CLOSE request_socket")
self.responder.close_connection()
self.logger.info("@@STRATUS-APP: TERM responder")
self.shutdown()
self.logger.info("@@STRATUS-APP: shutdown complete")
class Driver(drivers.BaseDriver):
def __init__(
self,
args,
encrypted_traffic_data=None,
interface=None,
):
"""Initialize the Driver.
:param args: Arguments parsed by argparse.
:type args: Object
:param encrypted_traffic: Enable|Disable encrypted traffic.
:type encrypted_traffic: Boolean
:param interface: The interface instance (client/server)
:type interface: Object
"""
self.thread_processor = multiprocessing.Process
self.event = multiprocessing.Event()
self.semaphore = multiprocessing.Semaphore
self.flushqueue = _FlushQueue
self.args = args
if getattr(self.args, "zmq_generate_keys", False) is True:
self._generate_certificates()
print("New certificates generated")
raise SystemExit(0)
self.encrypted_traffic_data = encrypted_traffic_data
mode = getattr(self.args, "mode", None)
if mode == "client":
self.bind_address = self.args.zmq_server_address
elif mode == "server":
self.bind_address = self.args.zmq_bind_address
else:
self.bind_address = "*"
self.proto = "tcp"
self.connection_string = "{proto}://{addr}".format(
proto=self.proto, addr=self.bind_address)
if self.encrypted_traffic_data:
self.encrypted_traffic = self.encrypted_traffic_data.get("enabled")
self.secret_keys_dir = self.encrypted_traffic_data.get(
"secret_keys_dir")
self.public_keys_dir = self.encrypted_traffic_data.get(
"public_keys_dir")
else:
self.encrypted_traffic = False
self.secret_keys_dir = None
self.public_keys_dir = None
self._context = zmq.Context()
self.ctx = self._context.instance()
self.poller = zmq.Poller()
self.interface = interface
super(Driver, self).__init__(
args=args,
encrypted_traffic_data=self.encrypted_traffic_data,
interface=interface,
)
self.bind_job = None
self.bind_backend = None
self.hwm = getattr(self.args, "zmq_highwater_mark", 1024)
def __copy__(self):
"""Return a new copy of the driver."""
return Driver(
args=self.args,
encrypted_traffic_data=self.encrypted_traffic_data,
interface=self.interface,
)
def _backend_bind(self):
"""Bind an address to a backend socket and return the socket.
:returns: Object
"""
bind = self._socket_bind(
socket_type=zmq.ROUTER,
connection=self.connection_string,
port=self.args.backend_port,
)
bind.set_hwm(self.hwm)
self.log.debug(
"Identity [ %s ] backend connect hwm state [ %s ]",
self.identity,
bind.get_hwm(),
)
return bind
def _backend_connect(self):
"""Connect to a backend socket and return the socket.
:returns: Object
"""
self.log.debug("Establishing backend connection.")
bind = self._socket_connect(
socket_type=zmq.DEALER,
connection=self.connection_string,
port=self.args.backend_port,
)
bind.set_hwm(self.hwm)
self.log.debug(
"Identity [ %s ] backend connect hwm state [ %s ]",
self.identity,
bind.get_hwm(),
)
return bind
def _bind_check(self, bind, interval=1, constant=1000):
"""Return True if a bind type contains work ready.
:param bind: A given Socket bind to identify.
:type bind: Object
:param interval: Exponential Interval used to determine the polling
duration for a given socket.
:type interval: Integer
:param constant: Constant time used to poll for new jobs.
:type constant: Integer
:returns: Object
"""
socks = dict(self.poller.poll(interval * constant))
if socks.get(bind) == zmq.POLLIN:
return True
else:
return False
def _close(self, socket):
if socket is None:
return
try:
socket.close(linger=2)
close_time = time.time()
while not socket.closed:
if time.time() - close_time > 60:
raise TimeoutError(
"Job [ {} ] failed to close transfer socket".format(
self.job_id))
else:
socket.close(linger=2)
time.sleep(1)
except Exception as e:
self.log.error(
"Ran into an exception while closing the socket %s",
str(e),
)
else:
self.log.debug("Backend socket closed")
def _generate_certificates(self, base_dir="/etc/directord"):
"""Generate client and server CURVE certificate files.
:param base_dir: Directord configuration path.
:type base_dir: String
"""
keys_dir = os.path.join(base_dir, "certificates")
public_keys_dir = os.path.join(base_dir, "public_keys")
secret_keys_dir = os.path.join(base_dir, "private_keys")
for item in [keys_dir, public_keys_dir, secret_keys_dir]:
os.makedirs(item, exist_ok=True)
# Run certificate backup
self._move_certificates(directory=public_keys_dir, backup=True)
self._move_certificates(directory=secret_keys_dir,
backup=True,
suffix=".key_secret")
# create new keys in certificates dir
for item in ["server", "client"]:
self._key_generate(keys_dir=keys_dir, key_type=item)
# Move generated certificates in place
self._move_certificates(
directory=keys_dir,
target_directory=public_keys_dir,
suffix=".key",
)
self._move_certificates(
directory=keys_dir,
target_directory=secret_keys_dir,
suffix=".key_secret",
)
def _job_bind(self):
"""Bind an address to a job socket and return the socket.
:returns: Object
"""
return self._socket_bind(
socket_type=zmq.ROUTER,
connection=self.connection_string,
port=self.args.job_port,
)
def _job_connect(self):
"""Connect to a job socket and return the socket.
:returns: Object
"""
self.log.debug("Establishing Job connection.")
return self._socket_connect(
socket_type=zmq.DEALER,
connection=self.connection_string,
port=self.args.job_port,
)
def _key_generate(self, keys_dir, key_type):
"""Generate certificate.
:param keys_dir: Full Directory path where a given key will be stored.
:type keys_dir: String
:param key_type: Key type to be generated.
:type key_type: String
"""
zmq_auth.create_certificates(keys_dir, key_type)
@staticmethod
def _move_certificates(directory,
target_directory=None,
backup=False,
suffix=".key"):
"""Move certificates when required.
:param directory: Set the origin path.
:type directory: String
:param target_directory: Set the target path.
:type target_directory: String
:param backup: Enable file backup before moving.
:type backup: Boolean
:param suffix: Set the search suffix
:type suffix: String
"""
for item in os.listdir(directory):
if backup:
target_file = "{}.bak".format(os.path.basename(item))
else:
target_file = os.path.basename(item)
if item.endswith(suffix):
os.rename(
os.path.join(directory, item),
os.path.join(target_directory or directory, target_file),
)
def _socket_bind(self, socket_type, connection, port, poller_type=None):
"""Return a socket object which has been bound to a given address.
When the socket_type is not PUB or PUSH, the bound socket will also be
registered with self.poller as defined within the Interface
class.
:param socket_type: Set the Socket type, typically defined using a
ZeroMQ constant.
:type socket_type: Integer
:param connection: Set the Address information used for the bound
socket.
:type connection: String
:param port: Define the port which the socket will be bound to.
:type port: Integer
:param poller_type: Set the Socket type, typically defined using a
ZeroMQ constant.
:type poller_type: Integer
:returns: Object
"""
if poller_type is None:
poller_type = zmq.POLLIN
bind = self._socket_context(socket_type=socket_type)
auth_enabled = (self.args.zmq_shared_key
or self.args.zmq_curve_encryption)
if auth_enabled:
self.auth = ThreadAuthenticator(self.ctx, log=self.log)
self.auth.start()
self.auth.allow()
if self.args.zmq_shared_key:
# Enables basic auth
self.auth.configure_plain(
domain="*", passwords={"admin": self.args.zmq_shared_key})
bind.plain_server = True # Enable shared key authentication
self.log.info("Shared key authentication enabled.")
elif self.args.zmq_curve_encryption:
server_secret_file = os.path.join(self.secret_keys_dir,
"server.key_secret")
for item in [
self.public_keys_dir,
self.secret_keys_dir,
server_secret_file,
]:
if not os.path.exists(item):
raise SystemExit(
"The required path [ {} ] does not exist. Have"
" you generated your keys?".format(item))
self.auth.configure_curve(domain="*",
location=self.public_keys_dir)
try:
server_public, server_secret = zmq_auth.load_certificate(
server_secret_file)
except OSError as e:
self.log.error(
"Failed to load certificates: %s, Configuration: %s",
str(e),
vars(self.args),
)
raise SystemExit("Failed to load certificates")
else:
bind.curve_secretkey = server_secret
bind.curve_publickey = server_public
bind.curve_server = True # Enable curve authentication
bind.bind("{connection}:{port}".format(
connection=connection,
port=port,
))
if socket_type not in [zmq.PUB]:
self.poller.register(bind, poller_type)
return bind
def _socket_connect(self, socket_type, connection, port, poller_type=None):
"""Return a socket object which has been bound to a given address.
> A connection back to the server will wait 10 seconds for an ack
before going into a retry loop. This is done to forcefully cycle
the connection object to reset.
:param socket_type: Set the Socket type, typically defined using a
ZeroMQ constant.
:type socket_type: Integer
:param connection: Set the Address information used for the bound
socket.
:type connection: String
:param port: Define the port which the socket will be bound to.
:type port: Integer
:param poller_type: Set the Socket type, typically defined using a
ZeroMQ constant.
:type poller_type: Integer
:returns: Object
"""
if poller_type is None:
poller_type = zmq.POLLIN
bind = self._socket_context(socket_type=socket_type)
if self.args.zmq_shared_key:
bind.plain_username = b"admin" # User is hard coded.
bind.plain_password = self.args.zmq_shared_key.encode()
self.log.info("Shared key authentication enabled.")
elif self.args.zmq_curve_encryption:
client_secret_file = os.path.join(self.secret_keys_dir,
"client.key_secret")
server_public_file = os.path.join(self.public_keys_dir,
"server.key")
for item in [
self.public_keys_dir,
self.secret_keys_dir,
client_secret_file,
server_public_file,
]:
if not os.path.exists(item):
raise SystemExit(
"The required path [ {} ] does not exist. Have"
" you generated your keys?".format(item))
try:
client_public, client_secret = zmq_auth.load_certificate(
client_secret_file)
server_public, _ = zmq_auth.load_certificate(
server_public_file)
except OSError as e:
self.log.error(
"Error while loading certificates: %s. Configuration: %s",
str(e),
vars(self.args),
)
raise SystemExit("Failed to load keys.")
else:
bind.curve_secretkey = client_secret
bind.curve_publickey = client_public
bind.curve_serverkey = server_public
if socket_type == zmq.SUB:
bind.setsockopt_string(zmq.SUBSCRIBE, self.identity)
else:
bind.setsockopt_string(zmq.IDENTITY, self.identity)
self.poller.register(bind, poller_type)
bind.connect("{connection}:{port}".format(
connection=connection,
port=port,
))
self.log.info("Socket connected to [ %s ].", connection)
return bind
def _socket_context(self, socket_type):
"""Create socket context and return a bind object.
:param socket_type: Set the Socket type, typically defined using a
ZeroMQ constant.
:type socket_type: Integer
:returns: Object
"""
bind = self.ctx.socket(socket_type)
bind.linger = getattr(self.args, "heartbeat_interval", 60)
hwm = int(self.hwm * 4)
try:
bind.sndhwm = bind.rcvhwm = hwm
except AttributeError:
bind.hwm = hwm
bind.set_hwm(hwm)
bind.setsockopt(zmq.SNDHWM, hwm)
bind.setsockopt(zmq.RCVHWM, hwm)
if socket_type == zmq.ROUTER:
bind.setsockopt(zmq.ROUTER_MANDATORY, 1)
return bind
@staticmethod
def _socket_recv(socket, nonblocking=False):
"""Receive a message over a ZM0 socket.
The message specification for server is as follows.
[
b"Identity"
b"ID",
b"ASCII Control Characters",
b"command",
b"data",
b"info",
b"stderr",
b"stdout",
]
The message specification for client is as follows.
[
b"ID",
b"ASCII Control Characters",
b"command",
b"data",
b"info",
b"stderr",
b"stdout",
]
All message parts are byte encoded.
All possible control characters are defined within the Interface class.
For more on control characters review the following
URL(https://donsnotes.com/tech/charsets/ascii.html#cntrl).
:param socket: ZeroMQ socket object.
:type socket: Object
:param nonblocking: Enable non-blocking receve.
:type nonblocking: Boolean
"""
if nonblocking:
flags = zmq.NOBLOCK
else:
flags = 0
return socket.recv_multipart(flags=flags)
@tenacity.retry(
retry=tenacity.retry_if_exception_type(Exception),
wait=tenacity.wait_fixed(5),
before_sleep=tenacity.before_sleep_log(
logger.getLogger(name="directord"), logging.WARN),
)
def _socket_send(
self,
socket,
identity=None,
msg_id=None,
control=None,
command=None,
data=None,
info=None,
stderr=None,
stdout=None,
nonblocking=False,
):
"""Send a message over a ZM0 socket.
The message specification for server is as follows.
[
b"Identity"
b"ID",
b"ASCII Control Characters",
b"command",
b"data",
b"info",
b"stderr",
b"stdout",
]
The message specification for client is as follows.
[
b"ID",
b"ASCII Control Characters",
b"command",
b"data",
b"info",
b"stderr",
b"stdout",
]
All message information is assumed to be byte encoded.
All possible control characters are defined within the Interface class.
For more on control characters review the following
URL(https://donsnotes.com/tech/charsets/ascii.html#cntrl).
:param socket: ZeroMQ socket object.
:type socket: Object
:param identity: Target where message will be sent.
:type identity: Bytes
:param msg_id: ID information for a given message. If no ID is
provided a UUID will be generated.
:type msg_id: Bytes
:param control: ASCII control charaters.
:type control: Bytes
:param command: Command definition for a given message.
:type command: Bytes
:param data: Encoded data that will be transmitted.
:type data: Bytes
:param info: Encoded information that will be transmitted.
:type info: Bytes
:param stderr: Encoded error information from a command.
:type stderr: Bytes
:param stdout: Encoded output information from a command.
:type stdout: Bytes
:param nonblocking: Enable non-blocking send.
:type nonblocking: Boolean
:returns: Object
"""
def _encoder(item):
try:
return item.encode()
except AttributeError:
return item
if not msg_id:
msg_id = utils.get_uuid()
if not control:
control = self.nullbyte
if not command:
command = self.nullbyte
if not data:
data = self.nullbyte
if not info:
info = self.nullbyte
if not stderr:
stderr = self.nullbyte
if not stdout:
stdout = self.nullbyte
message_parts = [msg_id, control, command, data, info, stderr, stdout]
if identity:
message_parts.insert(0, identity)
message_parts = [_encoder(i) for i in message_parts]
if nonblocking:
flags = zmq.NOBLOCK
else:
flags = 0
try:
return socket.send_multipart(message_parts, flags=flags)
except Exception as e:
self.log.warn("Failed to send message to [ %s ]", identity)
raise e
def _recv(self, socket, nonblocking=False):
"""Receive message.
:param socket: ZeroMQ socket object.
:type socket: Object
:param nonblocking: Enable non-blocking receve.
:type nonblocking: Boolean
:returns: Tuple
"""
recv_obj = self._socket_recv(socket=socket, nonblocking=nonblocking)
return tuple([i.decode() for i in recv_obj])
def backend_recv(self, nonblocking=False):
"""Receive a transfer message.
:param nonblocking: Enable non-blocking receve.
:type nonblocking: Boolean
:returns: Tuple
"""
return self._recv(socket=self.bind_backend, nonblocking=nonblocking)
def backend_init(self):
"""Initialize the backend socket.
For server mode, this is a bound local socket.
For client mode, it is a connection to the server socket.
:returns: Object
"""
if self.args.mode == "server":
self.bind_backend = self._backend_bind()
else:
self.bind_backend = self._backend_connect()
def backend_close(self):
"""Close the backend socket."""
self._close(socket=self.bind_backend)
def backend_check(self, interval=1, constant=1000):
"""Return True if the backend contains work ready.
:param bind: A given Socket bind to identify.
:type bind: Object
:param interval: Exponential Interval used to determine the polling
duration for a given socket.
:type interval: Integer
:param constant: Constant time used to poll for new jobs.
:type constant: Integer
:returns: Object
"""
return self._bind_check(bind=self.bind_backend,
interval=interval,
constant=constant)
def backend_send(self, *args, **kwargs):
"""Send a job message.
* All args and kwargs are passed through to the socket send.
:returns: Object
"""
kwargs["socket"] = self.bind_backend
return self._socket_send(*args, **kwargs)
@staticmethod
def get_lock():
"""Returns a thread lock."""
return multiprocessing.Lock()
def heartbeat_send(self,
host_uptime=None,
agent_uptime=None,
version=None,
driver=None):
"""Send a heartbeat.
:param host_uptime: Sender uptime
:type host_uptime: String
:param agent_uptime: Sender agent uptime
:type agent_uptime: String
:param version: Sender directord version
:type version: String
:param version: Driver information
:type version: String
"""
job_id = utils.get_uuid()
self.log.info(
"Job [ %s ] sending heartbeat from [ %s ] to server",
job_id,
self.identity,
)
return self.job_send(
control=self.heartbeat_notice,
msg_id=job_id,
data=json.dumps({
"job_id": job_id,
"version": version,
"host_uptime": host_uptime,
"agent_uptime": agent_uptime,
"machine_id": self.machine_id,
"driver": driver,
}),
)
def job_send(self, *args, **kwargs):
"""Send a job message.
* All args and kwargs are passed through to the socket send.
:returns: Object
"""
kwargs["socket"] = self.bind_job
return self._socket_send(*args, **kwargs)
def job_recv(self, nonblocking=False):
"""Receive a transfer message.
:param nonblocking: Enable non-blocking receve.
:type nonblocking: Boolean
:returns: Tuple
"""
return self._recv(socket=self.bind_job, nonblocking=nonblocking)
def job_init(self):
"""Initialize the job socket.
For server mode, this is a bound local socket.
For client mode, it is a connection to the server socket.
:returns: Object
"""
if self.args.mode == "server":
self.bind_job = self._job_bind()
else:
self.bind_job = self._job_connect()
def job_close(self):
"""Close the job socket."""
self._close(socket=self.bind_job)
def job_check(self, interval=1, constant=1000):
"""Return True if a job contains work ready.
:param bind: A given Socket bind to identify.
:type bind: Object
:param interval: Exponential Interval used to determine the polling
duration for a given socket.
:type interval: Integer
:param constant: Constant time used to poll for new jobs.
:type constant: Integer
:returns: Object
"""
return self._bind_check(bind=self.bind_job,
interval=interval,
constant=constant)
def shutdown(self):
"""Shutdown the driver."""
if hasattr(self.ctx, "close"):
self.ctx.close()
if hasattr(self._context, "close"):
self._context.close()
self.job_close()
self.backend_close()
class CombaZMQAdapter(threading.Thread, CombaBase):
def __init__(self, port):
self.port = str(port)
threading.Thread.__init__ (self)
self.shutdown_event = Event()
self.context = zmq.Context().instance()
self.authserver = ThreadAuthenticator(self.context)
self.loadConfig()
self.start()
#------------------------------------------------------------------------------------------#
def run(self):
"""
run runs on function start
"""
self.startAuthserver()
self.data = ''
self.socket = self.context.socket(zmq.REP)
self.socket.plain_server = True
self.socket.bind("tcp://*:"+self.port)
self.shutdown_event.clear()
self.controller = CombaController(self, self.lqs_socket, self.lqs_recorder_socket)
self.controller.messenger.setMailAddresses(self.get('frommail'), self.get('adminmail'))
self.can_send = False
# Process tasks forever
while not self.shutdown_event.is_set():
self.data = self.socket.recv()
self.can_send = True
data = self.data.split(' ')
command = str(data.pop(0))
params = "()" if len(data) < 1 else "('" + "','".join(data) + "')"
try:
exec"a=self.controller." + command + params
except SyntaxError:
self.controller.message('Warning: Syntax Error')
except AttributeError:
print "Warning: Method " + command + " does not exist"
self.controller.message('Warning: Method ' + command + ' does not exist')
except TypeError:
print "Warning: Wrong number of params"
self.controller.message('Warning: Wrong number of params')
except:
print "Warning: Unknown Error"
self.controller.message('Warning: Unknown Error')
return
#------------------------------------------------------------------------------------------#
def halt(self):
"""
Stop the server
"""
if self.shutdown_event.is_set():
return
try:
del self.controller
except:
pass
self.shutdown_event.set()
result = 'failed'
try:
result = self.socket.unbind("tcp://*:"+self.port)
except:
pass
#self.socket.close()
#------------------------------------------------------------------------------------------#
def reload(self):
"""
stop, reload config and startagaing
"""
if self.shutdown_event.is_set():
return
self.loadConfig()
self.halt()
time.sleep(3)
self.run()
#------------------------------------------------------------------------------------------#
def send(self,message):
"""
Send a message to the client
:param message: string
"""
if self.can_send:
self.socket.send(message)
self.can_send = False
#------------------------------------------------------------------------------------------#
def startAuthserver(self):
"""
Start zmq authentification server
"""
# stop auth server if running
if self.authserver.is_alive():
self.authserver.stop()
if self.securitylevel > 0:
# Authentifizierungsserver starten.
self.authserver.start()
# Bei security level 2 auch passwort und usernamen verlangen
if self.securitylevel > 1:
try:
addresses = CombaWhitelist().getList()
for address in addresses:
self.authserver.allow(address)
except:
pass
# Instruct authenticator to handle PLAIN requests
self.authserver.configure_plain(domain='*', passwords=self.getAccounts())
#------------------------------------------------------------------------------------------#
def getAccounts(self):
"""
Get accounts from redis db
:return: llist - a list of accounts
"""
accounts = CombaUser().getLogins()
db = redis.Redis()
internaccount = db.get('internAccess')
if not internaccount:
user = ''.join(random.sample(string.lowercase,10))
password = ''.join(random.sample(string.lowercase+string.uppercase+string.digits,22))
db.set('internAccess', user + ':' + password)
intern = [user, password]
else:
intern = internaccount.split(':')
accounts[intern[0]] = intern[1]
return accounts
class TaskQueue:
"""Outgoing task queue from the executor to the Interchange"""
def __init__(
self,
address: str,
port: int = 55001,
identity: str = str(uuid.uuid4()),
zmq_context=None,
set_hwm=False,
RCVTIMEO=None,
SNDTIMEO=None,
linger=None,
ironhouse: bool = False,
keys_dir: str = os.path.abspath(".curve"),
mode: str = "client",
):
"""
Parameters
----------
address: str
address to connect
port: int
Port to use
identity : str
Applies only to clients, where the identity must match the endpoint uuid.
This will be utf-8 encoded on the wire. A random uuid4 string is set by
default.
mode: string
Either 'client' or 'server'
keys_dir : string
Directory from which keys will be loaded for curve.
ironhouse: Bool
Only valid for server mode. Setting this flag switches the server to require
client keys to be available on the server in the keys_dir.
"""
if zmq_context:
self.context = zmq_context
else:
self.context = zmq.Context()
self.mode = mode
self.port = port
self.ironhouse = ironhouse
self.keys_dir = keys_dir
assert self.mode in [
"client",
"server",
], "Only two modes are supported: client, server"
if self.mode == "server":
print("Configuring server")
self.zmq_socket = self.context.socket(zmq.ROUTER)
self.zmq_socket.set(zmq.ROUTER_MANDATORY, 1)
self.zmq_socket.set(zmq.ROUTER_HANDOVER, 1)
print("Setting up auth-server")
self.setup_server_auth()
elif self.mode == "client":
self.zmq_socket = self.context.socket(zmq.DEALER)
self.setup_client_auth()
self.zmq_socket.setsockopt(zmq.IDENTITY, identity.encode("utf-8"))
else:
raise ValueError(
"TaskQueue must be initialized with mode set to 'server' or 'client'"
)
if set_hwm:
self.zmq_socket.set_hwm(0)
if RCVTIMEO is not None:
self.zmq_socket.setsockopt(zmq.RCVTIMEO, RCVTIMEO)
if SNDTIMEO is not None:
self.zmq_socket.setsockopt(zmq.SNDTIMEO, SNDTIMEO)
if linger is not None:
self.zmq_socket.setsockopt(zmq.LINGER, linger)
# all zmq setsockopt calls must be done before bind/connect is called
if self.mode == "server":
self.zmq_socket.bind(f"tcp://*:{port}")
elif self.mode == "client":
self.zmq_socket.connect(f"tcp://{address}:{port}")
self.poller = zmq.Poller()
self.poller.register(self.zmq_socket)
os.makedirs(self.keys_dir, exist_ok=True)
log.debug(f"Initializing Taskqueue:{self.mode} on port:{self.port}")
def zmq_context(self):
return self.context
def add_client_key(self, endpoint_id, client_key):
log.info("Adding client key")
if self.ironhouse:
# Use the ironhouse ZMQ pattern: http://hintjens.com/blog:49#toc6
with open(os.path.join(self.keys_dir, f"{endpoint_id}.key"),
"w") as f:
f.write(client_key)
try:
self.auth.configure_curve(domain="*", location=self.keys_dir)
except Exception:
log.exception("Failed to load keys from {self.keys_dir}")
return
def setup_server_auth(self):
# Start an authenticator for this context.
self.auth = ThreadAuthenticator(self.context)
self.auth.start()
self.auth.allow("127.0.0.1")
# Tell the authenticator how to handle CURVE requests
if not self.ironhouse:
# Use the stonehouse ZMQ pattern: http://hintjens.com/blog:49#toc5
self.auth.configure_curve(domain="*",
location=zmq.auth.CURVE_ALLOW_ANY)
server_secret_file = os.path.join(self.keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
self.zmq_socket.curve_secretkey = server_secret
self.zmq_socket.curve_publickey = server_public
self.zmq_socket.curve_server = True # must come before bind
def setup_client_auth(self):
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(self.keys_dir, "endpoint.key_secret")
client_public, client_secret = zmq.auth.load_certificate(
client_secret_file)
self.zmq_socket.curve_secretkey = client_secret
self.zmq_socket.curve_publickey = client_public
# The client must know the server's public key to make a CURVE connection.
server_public_file = os.path.join(self.keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
self.zmq_socket.curve_serverkey = server_public
def get(self, block=True, timeout=1000):
"""
Parameters
----------
block : Bool
Blocks until there's a message, Default is True
timeout : int
Milliseconds to wait.
"""
# timeout is in milliseconds
if block is True:
return self.zmq_socket.recv_multipart()
socks = dict(self.poller.poll(timeout=timeout))
if self.zmq_socket in socks and socks[self.zmq_socket] == zmq.POLLIN:
message = self.zmq_socket.recv_multipart()
return message
else:
raise zmq.Again
def register_client(self, message):
return self.zmq_socket.send_multipart([message])
def put(self, dest, message, max_timeout=1000):
"""This function needs to be fast at the same time aware of the possibility of
ZMQ pipes overflowing.
The timeout increases slowly if contention is detected on ZMQ pipes.
We could set copy=False and get slightly better latency but this results
in ZMQ sockets reaching a broken state once there are ~10k tasks in flight.
This issue can be magnified if each the serialized buffer itself is larger.
Parameters
----------
dest : zmq_identity of the destination endpoint, must be a byte string
message : py object
Python object to send
max_timeout : int
Max timeout in milliseconds that we will wait for before raising an
exception
Raises
------
zmq.EAGAIN if the send failed.
zmq.error.ZMQError: Host unreachable (if client disconnects?)
"""
if self.mode == "client":
return self.zmq_socket.send_multipart([message])
else:
return self.zmq_socket.send_multipart([dest, message])
def close(self):
self.zmq_socket.close()
self.context.term()
class MultiNodeAgent(BEMOSSAgent):
def __init__(self, *args, **kwargs):
super(MultiNodeAgent, self).__init__(*args, **kwargs)
self.multinode_status = dict()
self.getMultinodeData()
self.agent_id = 'multinodeagent'
self.is_parent = False
self.last_sync_with_parent = datetime(1991, 1,
1) #equivalent to -ve infinitive
self.parent_node = None
self.recently_online_node_list = [] # initialize to lists to empty
self.recently_offline_node_list = [
] # they will be filled as nodes are discovered to be online/offline
self.setup()
self.runPeriodically(self.send_heartbeat, 20)
self.runPeriodically(self.check_health, 60, start_immediately=False)
self.runPeriodically(self.sync_all_with_parent, 600)
self.subscribe('relay_message', self.relayDirectMessage)
self.subscribe('update_multinode_data', self.updateMultinodeData)
self.runContinuously(self.pollClients)
self.run()
def getMultinodeData(self):
self.multinode_data = db_helper.get_multinode_data()
self.nodelist_dict = {
node['name']: node
for node in self.multinode_data['known_nodes']
}
self.node_name_list = [
node['name'] for node in self.multinode_data['known_nodes']
]
self.address_list = [
node['address'] for node in self.multinode_data['known_nodes']
]
self.server_key_list = [
node['server_key'] for node in self.multinode_data['known_nodes']
]
self.node_name = self.multinode_data['this_node']
for index, node in enumerate(self.multinode_data['known_nodes']):
if node['name'] == self.node_name:
self.node_index = index
break
else:
raise ValueError(
'"this_node:" entry on the multinode_data json file is invalid'
)
for node_name in self.node_name_list: #initialize all nodes data
if node_name not in self.multinode_status: #initialize new nodes. There could be already the node if this getMultiNode
# data is being called later
self.multinode_status[node_name] = dict()
self.multinode_status[node_name][
'health'] = -10 #initialized; never online/offline
self.multinode_status[node_name]['last_sync_time'] = datetime(
1991, 1, 1)
self.multinode_status[node_name]['last_online_time'] = None
self.multinode_status[node_name]['last_offline_time'] = None
self.multinode_status[node_name]['last_scanned_time'] = None
def setup(self):
print "Setup"
base_dir = settings.PROJECT_DIR + "/"
public_keys_dir = os.path.abspath(os.path.join(base_dir,
'public_keys'))
secret_keys_dir = os.path.abspath(
os.path.join(base_dir, 'private_keys'))
self.secret_keys_dir = secret_keys_dir
self.public_keys_dir = public_keys_dir
if not (os.path.exists(public_keys_dir)
and os.path.exists(secret_keys_dir)):
logging.critical(
"Certificates are missing - run generate_certificates.py script first"
)
sys.exit(1)
ctx = zmq.Context.instance()
self.ctx = ctx
# Start an authenticator for this context.
self.auth = ThreadAuthenticator(ctx)
self.auth.start()
self.configure_authenticator()
server = ctx.socket(zmq.PUB)
server_secret_key_filename = self.multinode_data['known_nodes'][
self.node_index]['server_secret_key']
server_secret_file = os.path.join(secret_keys_dir,
server_secret_key_filename)
server_public, server_secret = zmq.auth.load_certificate(
server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
server.bind(
self.multinode_data['known_nodes'][self.node_index]['address'])
self.server = server
self.configureClient()
def configure_authenticator(self):
self.auth.allow()
# Tell authenticator to use the certificate in a directory
self.auth.configure_curve(domain='*', location=self.public_keys_dir)
def disperseMessage(self, sender, topic, message):
for node_name in self.node_name_list:
if node_name == self.node_name:
continue
self.server.send(
jsonify(sender, node_name + '/republish/' + topic, message))
def republishToParent(self, sender, topic, message):
if self.is_parent:
return #if I am parent, the message is already published
for node_name in self.node_name_list:
if self.multinode_status[node_name][
'health'] == 2: #health = 2 is the parent node
self.server.send(
jsonify(sender, node_name + '/republish/' + topic,
message))
def sync_node_with_parent(self, node_name):
if self.is_parent:
print "Syncing " + node_name
self.last_sync_with_parent = datetime.now()
sync_date_string = self.last_sync_with_parent.strftime(
'%B %d, %Y, %H:%M:%S')
# os.system('pg_dump bemossdb -f ' + self.self_database_dump_path)
# with open(self.self_database_dump_path, 'r') as f:
# file_content = f.read()
# msg = {'database_dump': base64.b64encode(file_content)}
self.server.send(
jsonify(
self.node_name, node_name + '/sync-with-parent/' +
sync_date_string + '/' + self.node_name, ""))
def sync_all_with_parent(self, dbcon):
if self.is_parent:
self.last_sync_with_parent = datetime.now()
sync_date_string = self.last_sync_with_parent.strftime(
'%B %d, %Y, %H:%M:%S')
print "Syncing all nodes"
for node_name in self.node_name_list:
if node_name == self.node_name:
continue
# os.system('pg_dump bemossdb -f ' + self.self_database_dump_path)
# with open(self.self_database_dump_path, 'r') as f:
# file_content = f.read()
# msg = {'database_dump': base64.b64encode(file_content)}
self.server.send(
jsonify(
self.node_name, node_name + '/sync-with-parent/' +
sync_date_string + '/' + self.node_name, ""))
def send_heartbeat(self, dbcon):
#self.vip.pubsub.publish('pubsub', 'listener', None, {'message': 'Hello Listener'})
#print 'publishing'
print "Sending heartbeat"
last_sync_string = self.last_sync_with_parent.strftime(
'%B %d, %Y, %H:%M:%S')
self.server.send(
jsonify(
self.node_name, 'heartbeat/' + self.node_name + '/' +
str(self.is_parent) + '/' + last_sync_string, ""))
def extract_ip(self, addr):
return re.search(r'([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})',
addr).groups()[0]
def getNodeId(self, node_name):
for index, node in enumerate(self.multinode_data['known_nodes']):
if node['name'] == node_name:
node_index = index
break
else:
raise ValueError('the node name: ' + node_name +
' is not found in multinode data')
return node_index
def getNodeName(self, node_id):
return self.multinode_data['known_nodes'][node_id]['name']
def handle_offline_nodes(self, dbcon, node_name_list):
if self.is_parent:
# start all the agents belonging to that node on this node
command_group = []
for node_name in node_name_list:
node_id = self.getNodeId(node_name)
#put the offline event into cassandra events log table, and also create notification
self.EventRegister(dbcon,
'node-offline',
reason='communication-error',
source=node_name)
# get a list of agents that were supposedly running in that offline node
dbcon.execute(
"SELECT agent_id FROM " + node_devices_table +
" WHERE assigned_node_id=%s", (node_id, ))
if dbcon.rowcount:
agent_ids = dbcon.fetchall()
for agent_id in agent_ids:
message = dict()
message[STATUS_CHANGE.AGENT_ID] = agent_id[0]
message[STATUS_CHANGE.NODE] = str(self.node_index)
message[STATUS_CHANGE.AGENT_STATUS] = 'start'
message[
STATUS_CHANGE.
NODE_ASSIGNMENT_TYPE] = ZONE_ASSIGNMENT_TYPES.TEMPORARY
command_group += [message]
dbcon.execute(
"UPDATE " + node_devices_table +
" SET current_node_id=(%s), date_move=(%s)"
" WHERE agent_id=(%s)",
(self.node_index, datetime.now(
pytz.UTC), agent_id[0]))
dbcon.commit()
print "moving agents from offline node to parent: " + str(
node_name_list)
print command_group
if command_group:
self.bemoss_publish(target='networkagent',
topic='status_change',
message=command_group)
def handle_online_nodes(self, dbcon, node_name_list):
if self.is_parent:
# start all the agents belonging to that nodes back on them
command_group = []
for node_name in node_name_list:
node_id = self.getNodeId(node_name)
if self.node_index == node_id:
continue #don't handle self-online
self.EventRegister(dbcon,
'node-online',
reason='communication-restored',
source=node_name)
#get a list of agents that were supposed to be running in that online node
dbcon.execute(
"SELECT agent_id FROM " + node_devices_table +
" WHERE assigned_node_id=%s", (node_id, ))
if dbcon.rowcount:
agent_ids = dbcon.fetchall()
for agent_id in agent_ids:
message = dict()
message[STATUS_CHANGE.AGENT_ID] = agent_id[0]
message[
STATUS_CHANGE.
NODE_ASSIGNMENT_TYPE] = ZONE_ASSIGNMENT_TYPES.PERMANENT
message[STATUS_CHANGE.NODE] = str(self.node_index)
message[STATUS_CHANGE.
AGENT_STATUS] = 'stop' #stop in this node
command_group += [message]
message = dict(message) #create another copy
message[STATUS_CHANGE.NODE] = str(node_id)
message[
STATUS_CHANGE.
AGENT_STATUS] = 'start' #start in the target node
command_group += [message]
#immediately update the multnode device assignment table
dbcon.execute(
"UPDATE " + node_devices_table +
" SET current_node_id=(%s), date_move=(%s)"
" WHERE agent_id=(%s)",
(node_id, datetime.now(pytz.UTC), agent_id[0]))
dbcon.commit()
print "Moving agents back to the online node: " + str(
node_name_list)
print command_group
if command_group:
self.bemoss_publish(target='networkagent',
topic='status_change',
message=command_group)
def updateParent(self, dbcon, parent_node_name):
parent_ip = self.extract_ip(
self.nodelist_dict[parent_node_name]['address'])
write_new = False
if not os.path.isfile(settings.MULTINODE_PARENT_IP_FILE
): # but parent file doesn't exists
write_new = True
else:
with open(settings.MULTINODE_PARENT_IP_FILE, 'r') as f:
read_ip = f.read()
if read_ip != parent_ip:
write_new = True
if write_new:
with open(settings.MULTINODE_PARENT_IP_FILE, 'w') as f:
f.write(parent_ip)
if dbcon:
dbcon.close() #close old connection
dbcon = db_helper.db_connection(
) #start new connection using new parent_ip
self.updateMultinodeData(sender=self.name,
topic='update_parent',
message="")
def check_health(self, dbcon):
for node_name, node in self.multinode_status.items():
if node['health'] > 0: #initialize all online nodes to 0. If they are really online, they should change it
# back to 1 or 2 (parent) within 30 seconds throught the heartbeat.
node['health'] = 0
time.sleep(30)
parent_node_name = None #initialize parent node
online_node_exists = False
for node_name, node in self.multinode_status.items():
node['last_scanned_time'] = datetime.now()
if node['health'] == 0:
node['health'] = -1
node['last_offline_time'] = datetime.now()
self.recently_offline_node_list += [node_name]
elif node['health'] == -1: #offline since long
pass
elif node[
'health'] == -10: #The node was initialized to -10, and never came online. Treat it as recently going
# offline for this iteration so that the agents that were supposed to be running there can be migrated
node['health'] = -1
self.recently_offline_node_list += [node_name]
elif node['health'] == 2: #there is some parent node present
parent_node_name = node_name
if node['health'] > 0:
online_node_exists = True #At-least one node (itself) should be online, if not some problem
assert online_node_exists, "At least one node (current node) must be online"
if not parent_node_name: #parent node doesn't exist
#find a suitable node to elect a parent. The node with latest update from previous parent wins. If there is
#tie, then the node coming earlier in the node-list on multinode data wins
online_node_last_sync = dict(
) #only the online nodes, and their last-sync-times
for node_name, node in self.multinode_status.items(
): #copy only the online nodes
if node['health'] > 0:
online_node_last_sync[node_name] = node['last_sync_time']
latest_node = max(online_node_last_sync,
key=online_node_last_sync.get)
latest_sync_date = online_node_last_sync[latest_node]
for node_name in self.node_name_list:
if self.multinode_status[node_name][
'health'] <= 0: #dead nodes can't be parents
continue
if self.multinode_status[node_name][
'last_sync_time'] == latest_sync_date: # this is the first node with the latest update from parent
#elligible parent found
self.updateParent(dbcon, node_name)
if node_name == self.node_name: # I am the node, so I get to become the parent
self.is_parent = True
print "I am the boss now, " + self.node_name
break
else: #I-am-not-the-first-node with latest update; somebody else is
self.is_parent = False
break
else: #parent node exist
self.updateParent(dbcon, parent_node_name)
for node in self.multinode_data['known_nodes']:
print node['name'] + ': ' + str(
self.multinode_status[node['name']]['health'])
if self.is_parent:
#if this is a parent node, update the node_info table
if dbcon is None: #if no database connection exists make connection
dbcon = db_helper.db_connection()
tbl_node_info = settings.DATABASES['default']['TABLE_node_info']
dbcon.execute('select node_id from ' + tbl_node_info)
to_be_deleted_node_ids = dbcon.fetchall()
for index, node in enumerate(self.multinode_data['known_nodes']):
if (index, ) in to_be_deleted_node_ids:
to_be_deleted_node_ids.remove(
(index, )) #don't remove this current node
result = dbcon.execute(
'select * from ' + tbl_node_info + ' where node_id=%s',
(index, ))
node_type = 'parent' if self.multinode_status[
node['name']]['health'] == 2 else "child"
node_status = "ONLINE" if self.multinode_status[
node['name']]['health'] > 0 else "OFFLINE"
ip_address = self.extract_ip(node['address'])
last_scanned_time = self.multinode_status[
node['name']]['last_online_time']
last_offline_time = self.multinode_status[
node['name']]['last_offline_time']
last_sync_time = self.multinode_status[
node['name']]['last_sync_time']
var_list = "(node_id,node_name,node_type,node_status,ip_address,last_scanned_time,last_offline_time,last_sync_time)"
value_placeholder_list = "(%s,%s,%s,%s,%s,%s,%s,%s)"
actual_values_list = (index, node['name'], node_type,
node_status, ip_address,
last_scanned_time, last_offline_time,
last_sync_time)
if dbcon.rowcount == 0:
dbcon.execute(
"insert into " + tbl_node_info + " " + var_list +
" VALUES" + value_placeholder_list, actual_values_list)
else:
dbcon.execute(
"update " + tbl_node_info + " SET " + var_list +
" = " + value_placeholder_list + " where node_id = %s",
actual_values_list + (index, ))
dbcon.commit()
for id in to_be_deleted_node_ids:
dbcon.execute(
'delete from accounts_userprofile_nodes where nodeinfo_id=%s',
id) #delete entries in user-profile for the old node
dbcon.commit()
dbcon.execute('delete from ' + tbl_node_info +
' where node_id=%s', id) #delete the old nodes
dbcon.commit()
if self.recently_online_node_list: #Online nodes should be handled first because, the same node can first be
#on both recently_online_node_list and recently_offline_node_list, when it goes offline shortly after
#coming online
self.handle_online_nodes(dbcon, self.recently_online_node_list)
self.recently_online_node_list = [] # reset after handling
if self.recently_offline_node_list:
self.handle_offline_nodes(dbcon,
self.recently_offline_node_list)
self.recently_offline_node_list = [] # reset after handling
def connect_client(self, node):
server_public_file = os.path.join(self.public_keys_dir,
node['server_key'])
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
self.client.curve_serverkey = server_public
self.client.setsockopt(zmq.SUBSCRIBE, 'heartbeat/')
self.client.setsockopt(zmq.SUBSCRIBE, self.node_name)
self.client.connect(node['address'])
def disconnect_client(self, node):
self.client.disconnect(node['address'])
def configureClient(self):
print "Starting to receive Heart-beat"
client = self.ctx.socket(zmq.SUB)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_key_filename = self.multinode_data['known_nodes'][
self.node_index]['client_secret_key']
client_secret_file = os.path.join(self.secret_keys_dir,
client_secret_key_filename)
client_public, client_secret = zmq.auth.load_certificate(
client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
self.client = client
for node in self.multinode_data['known_nodes']:
self.connect_client(node)
def pollClients(self, dbcon):
if self.client.poll(1000):
sender, topic, msg = dejsonify(self.client.recv())
topic_list = topic.split('/')
if topic_list[0] == 'heartbeat':
node_name = sender
is_parent = topic_list[2]
last_sync_with_parent = topic_list[3]
if self.multinode_status[node_name][
'health'] < 0: #the node health was <0 , means offline
print node_name + " is back online"
self.recently_online_node_list += [node_name]
self.sync_node_with_parent(node_name)
if is_parent.lower() in ['false', '0']:
self.multinode_status[node_name]['health'] = 1
elif is_parent.lower() in ['true', '1']:
self.multinode_status[node_name]['health'] = 2
self.parent_node = node_name
else:
raise ValueError(
'Invalid is_parent string in heart-beat message')
self.multinode_status[node_name][
'last_online_time'] = datetime.now()
self.multinode_status[node_name][
'last_sync_time'] = datetime.strptime(
last_sync_with_parent, '%B %d, %Y, %H:%M:%S')
if topic_list[0] == self.node_name:
if topic_list[1] == 'sync-with-parent':
pass
# print topic
# self.last_sync_with_parent = datetime.strptime(topic_list[2], '%B %d, %Y, %H:%M:%S')
# content = base64.b64decode(msg['database_dump'])
# newpath = 'bemossdb.sql'
# with open(newpath, 'w') as f:
# f.write(content)
# try:
# os.system(
# 'psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();"')
# os.system(
# 'dropdb bemossdb') # This step requires all connections to be closed
# os.system('createdb bemossdb -O admin')
# dump_result = subprocess.check_output('psql bemossdb < ' + newpath, shell=True)
# except Exception as er:
# print "Couldn't sync database with parent because of error: "
# print er
#
# parent_node_name = topic_list[3]
# self.updateParent(parent_node_name)
if topic_list[1] == 'republish':
target = msg['target']
actual_message = msg['actual_message']
actual_topic = msg['actual_topic']
self.bemoss_publish(target=target,
topic=actual_topic + '/republished',
message=actual_message,
sender=sender)
print self.node_name + ": " + topic, str(msg)
else:
time.sleep(2)
def cleanup(self):
# stop auth thread
self.auth.stop()
def updateMultinodeData(self, dbcon, sender, topic, message):
print "Updating Multinode data"
topic_list = topic.split('/')
self.configure_authenticator()
#to/multinodeagent/from/<doesn't matter>/update_multinode_data
if topic_list[4] == 'update_multinode_data':
old_multinode_data = self.multinode_data
self.getMultinodeData()
for node in self.multinode_data['known_nodes']:
if node not in old_multinode_data['known_nodes']:
print "New node has been added to the cluster: " + node[
'name']
print "We will connect to it"
self.connect_client(node)
for node in old_multinode_data['known_nodes']:
if node not in self.multinode_data['known_nodes']:
print "Node has been removed from the cluster: " + node[
'name']
print "We will disconnect from it"
self.disconnect_client(node)
# TODO: remove it from the node_info table
print "yay! got it"
def relayDirectMessage(self, dbcon, sender, topic, message):
print topic
#to/<some_agent_or_ui>/topic/from/<some_agent_or_ui>
from_entity = sender
target = message['target']
actual_message = message['actual_message']
actual_topic = message['actual_topic']
for to_entity in target:
if to_entity in settings.NO_FORWARD_AGENTS:
return #no forwarding should be done for these agents
elif to_entity in settings.PARENT_NODE_SYSTEM_AGENTS:
if not self.is_parent:
self.republishToParent(sender, topic, message)
elif to_entity == "ALL":
self.disperseMessage(sender, topic=topic, message=message)
else:
dbcon.execute(
"SELECT current_node_id FROM " + node_devices_table +
" WHERE agent_id=%s", (to_entity, ))
if dbcon.rowcount:
node_id = dbcon.fetchone()[0]
if node_id != self.node_index:
self.server.send(
jsonify(
sender,
self.getNodeName(node_id) + '/republish/' +
topic, message))
else:
self.disperseMessage(
sender, topic, message
) #republish to all nodes if we don't know where to send
class dataTransfer():
def __init__ (self, connectionType, signalHost = None, useLog = False, context = None):
if useLog:
self.log = logging.getLogger("dataTransferAPI")
elif useLog == None:
self.log = noLoggingFunction()
else:
self.log = loggingFunction()
# ZMQ applications always start by creating a context,
# and then using that for creating sockets
# (source: ZeroMQ, Messaging for Many Applications by Pieter Hintjens)
if context:
self.context = context
self.extContext = True
else:
self.context = zmq.Context()
self.extContext = False
self.signalHost = signalHost
self.signalPort = "50000"
self.requestPort = "50001"
self.dataHost = None
self.dataPort = None
self.signalSocket = None
self.dataSocket = None
self.requestSocket = None
self.poller = zmq.Poller()
self.auth = None
self.targets = None
self.supportedConnections = ["stream", "streamMetadata", "queryNext", "queryMetadata"]
self.signalExchanged = None
self.streamStarted = None
self.queryNextStarted = None
self.socketResponseTimeout = 1000
if connectionType in self.supportedConnections:
self.connectionType = connectionType
else:
raise NotSupported("Chosen type of connection is not supported.")
# targets: [host, port, prio] or [[host, port, prio], ...]
def initiate (self, targets):
if type(targets) != list:
self.stop()
raise FormatError("Argument 'targets' must be list.")
if not self.context:
self.context = zmq.Context()
self.extContext = False
signal = None
# Signal exchange
if self.connectionType == "stream":
signalPort = self.signalPort
signal = "START_STREAM"
elif self.connectionType == "streamMetadata":
signalPort = self.signalPort
signal = "START_STREAM_METADATA"
elif self.connectionType == "queryNext":
signalPort = self.signalPort
signal = "START_QUERY_NEXT"
elif self.connectionType == "queryMetadata":
signalPort = self.signalPort
signal = "START_QUERY_METADATA"
self.log.debug("Create socket for signal exchange...")
if self.signalHost:
self.__createSignalSocket(signalPort)
else:
self.stop()
raise ConnectionFailed("No host to send signal to specified." )
self.__setTargets (targets)
message = self.__sendSignal(signal)
if message and message == "VERSION_CONFLICT":
self.stop()
raise VersionError("Versions are conflicting.")
elif message and message == "NO_VALID_HOST":
self.stop()
raise AuthenticationFailed("Host is not allowed to connect.")
elif message and message == "CONNECTION_ALREADY_OPEN":
self.stop()
raise CommunicationFailed("Connection is already open.")
elif message and message == "NO_VALID_SIGNAL":
self.stop()
raise CommunicationFailed("Connection type is not supported for this kind of sender.")
# if there was no response or the response was of the wrong format, the receiver should be shut down
elif message and message.startswith(signal):
self.log.info("Received confirmation ...")
self.signalExchanged = signal
else:
raise CommunicationFailed("Sending start signal ...failed.")
def __createSignalSocket (self, signalPort):
# To send a notification that a Displayer is up and running, a communication socket is needed
# create socket to exchange signals with Sender
self.signalSocket = self.context.socket(zmq.REQ)
# time to wait for the sender to give a confirmation of the signal
# self.signalSocket.RCVTIMEO = self.socketResponseTimeout
connectionStr = "tcp://" + str(self.signalHost) + ":" + str(signalPort)
try:
self.signalSocket.connect(connectionStr)
self.log.info("signalSocket started (connect) for '" + connectionStr + "'")
except:
self.log.error("Failed to start signalSocket (connect): '" + connectionStr + "'")
raise
# using a Poller to implement the signalSocket timeout (in older ZMQ version there is no option RCVTIMEO)
self.poller.register(self.signalSocket, zmq.POLLIN)
def __setTargets (self, targets):
self.targets = []
# [host, port, prio]
if len(targets) == 3 and type(targets[0]) != list and type(targets[1]) != list and type(targets[2]) != list:
host, port, prio = targets
self.targets = [[host + ":" + port, prio, [""]]]
# [host, port, prio, suffixes]
elif len(targets) == 4 and type(targets[0]) != list and type(targets[1]) != list and type(targets[2]) != list and type(targets[3]) == list:
host, port, prio, suffixes = targets
self.targets = [[host + ":" + port, prio, suffixes]]
# [[host, port, prio], ...] or [[host, port, prio, suffixes], ...]
else:
for t in targets:
if type(t) == list and len(t) == 3:
host, port, prio = t
self.targets.append([host + ":" + port, prio, [""]])
elif type(t) == list and len(t) == 4 and type(t[3]):
host, port, prio, suffixes = t
self.targets.append([host + ":" + port, prio, suffixes])
else:
self.stop()
self.log.debug("targets=" + str(targets))
raise FormatError("Argument 'targets' is of wrong format.")
def __sendSignal (self, signal):
if not signal:
return
# Send the signal that the communication infrastructure should be established
self.log.info("Sending Signal")
sendMessage = [__version__, signal]
trg = cPickle.dumps(self.targets)
sendMessage.append(trg)
# sendMessage = [__version__, signal, self.dataHost, self.dataPort]
self.log.debug("Signal: " + str(sendMessage))
try:
self.signalSocket.send_multipart(sendMessage)
except:
self.log.error("Could not send signal")
raise
message = None
try:
socks = dict(self.poller.poll(self.socketResponseTimeout))
except:
self.log.error("Could not poll for new message")
raise
# if there was a response
if self.signalSocket in socks and socks[self.signalSocket] == zmq.POLLIN:
try:
# Get the reply.
message = self.signalSocket.recv()
self.log.info("Received answer to signal: " + str(message) )
except:
self.log.error("Could not receive answer to signal")
raise
return message
def start (self, dataSocket = False, whitelist = None):
# Receive data only from whitelisted nodes
if whitelist:
if type(whitelist) == list:
self.auth = ThreadAuthenticator(self.context)
self.auth.start()
for host in whitelist:
try:
if host == "localhost":
ip = [socket.gethostbyname(host)]
else:
hostname, tmp, ip = socket.gethostbyaddr(host)
self.log.debug("Allowing host " + host + " (" + str(ip[0]) + ")")
self.auth.allow(ip[0])
except:
self.log.error("Error was: ", exc_info=True)
raise AuthenticationFailed("Could not get IP of host " + host)
else:
raise FormatError("Whitelist has to be a list of IPs")
socketIdToConnect = self.streamStarted or self.queryNextStarted
if socketIdToConnect:
self.log.info("Reopening already started connection.")
else:
ip = "0.0.0.0" #TODO use IP of hostname?
host = ""
port = ""
if dataSocket:
if type(dataSocket) == list:
socketIdToConnect = dataSocket[0] + ":" + dataSocket[1]
host = dataSocket[0]
ip = socket.gethostbyaddr(host)[2][0]
port = dataSocket[1]
else:
port = str(dataSocket)
host = socket.gethostname()
socketId = host + ":" + port
ipFromHost = socket.gethostbyaddr(host)[2]
if len(ipFromHost) == 1:
ip = ipFromHost[0]
elif len(self.targets) == 1:
host, port = self.targets[0][0].split(":")
ipFromHost = socket.gethostbyaddr(host)[2]
if len(ipFromHost) == 1:
ip = ipFromHost[0]
else:
raise FormatError("Multipe possible ports. Please choose which one to use.")
socketId = host + ":" + port
socketIdToConnect = ip + ":" + port
# socketIdToConnect = "[" + ip + "]:" + port
self.dataSocket = self.context.socket(zmq.PULL)
# An additional socket is needed to establish the data retriving mechanism
connectionStr = "tcp://" + socketIdToConnect
if whitelist:
self.dataSocket.zap_domain = b'global'
try:
# self.dataSocket.ipv6 = True
self.dataSocket.bind(connectionStr)
# self.dataSocket.bind("tcp://[2003:ce:5bc0:a600:fa16:54ff:fef4:9fc0]:50102")
self.log.info("Data socket of type " + self.connectionType + " started (bind) for '" + connectionStr + "'")
except:
self.log.error("Failed to start Socket of type " + self.connectionType + " (bind): '" + connectionStr + "'", exc_info=True)
raise
self.poller.register(self.dataSocket, zmq.POLLIN)
if self.connectionType in ["queryNext", "queryMetadata"]:
self.requestSocket = self.context.socket(zmq.PUSH)
# An additional socket is needed to establish the data retriving mechanism
connectionStr = "tcp://" + self.signalHost + ":" + self.requestPort
try:
self.requestSocket.connect(connectionStr)
self.log.info("Request socket started (connect) for '" + connectionStr + "'")
except:
self.log.error("Failed to start Socket of type " + self.connectionType + " (connect): '" + connectionStr + "'", exc_info=True)
raise
self.queryNextStarted = socketId
else:
self.streamStarted = socketId
##
#
# Receives or queries for new files depending on the connection initialized
#
# returns either
# the newest file
# (if connection type "queryNext" or "stream" was choosen)
# the path of the newest file
# (if connection type "queryMetadata" or "streamMetadata" was choosen)
#
##
def get (self, timeout=None):
if not self.streamStarted and not self.queryNextStarted:
self.log.info("Could not communicate, no connection was initialized.")
return None, None
if self.queryNextStarted :
sendMessage = ["NEXT", self.queryNextStarted]
try:
self.requestSocket.send_multipart(sendMessage)
except Exception as e:
self.log.error("Could not send request to requestSocket", exc_info=True)
return None, None
while True:
# receive data
if timeout:
try:
socks = dict(self.poller.poll(timeout))
except:
self.log.error("Could not poll for new message")
raise
else:
try:
socks = dict(self.poller.poll())
except:
self.log.error("Could not poll for new message")
raise
# if there was a response
if self.dataSocket in socks and socks[self.dataSocket] == zmq.POLLIN:
try:
multipartMessage = self.dataSocket.recv_multipart()
except:
self.log.error("Receiving data..failed.", exc_info=True)
return [None, None]
if multipartMessage[0] == b"ALIVE_TEST":
continue
elif len(multipartMessage) < 2:
self.log.error("Received mutipart-message is too short. Either config or file content is missing.")
self.log.debug("multipartMessage=" + str(mutipartMessage)[:100])
return [None, None]
# extract multipart message
try:
metadata = cPickle.loads(multipartMessage[0])
except:
self.log.error("Could not extract metadata from the multipart-message.", exc_info=True)
metadata = None
#TODO validate multipartMessage (like correct dict-values for metadata)
try:
payload = multipartMessage[1]
except:
self.log.warning("An empty file was received within the multipart-message", exc_info=True)
payload = None
return [metadata, payload]
else:
self.log.warning("Could not receive data in the given time.")
if self.queryNextStarted :
try:
self.requestSocket.send_multipart(["CANCEL", self.queryNextStarted])
except Exception as e:
self.log.error("Could not cancel the next query", exc_info=True)
return [None, None]
def store (self, targetBasePath, dataObject):
if type(dataObject) is not list and len(dataObject) != 2:
raise FormatError("Wrong input type for 'store'")
payloadMetadata = dataObject[0]
payload = dataObject[1]
if type(payloadMetadata) is not dict:
raise FormatError("payload: Wrong input format in 'store'")
#save all chunks to file
while True:
#TODO check if payload != cPickle.dumps(None) ?
if payloadMetadata and payload:
#append to file
try:
self.log.debug("append to file based on multipart-message...")
#TODO: save message to file using a thread (avoids blocking)
#TODO: instead of open/close file for each chunk recyle the file-descriptor for all chunks opened
self.__appendChunksToFile(targetBasePath, payloadMetadata, payload)
self.log.debug("append to file based on multipart-message...success.")
except KeyboardInterrupt:
self.log.info("KeyboardInterrupt detected. Unable to append multipart-content to file.")
break
except Exception, e:
self.log.error("Unable to append multipart-content to file.", exc_info=True)
self.log.debug("Append to file based on multipart-message...failed.")
if len(payload) < payloadMetadata["chunkSize"] :
#indicated end of file. Leave loop
filename = self.generateTargetFilepath(targetBasePath, payloadMetadata)
fileModTime = payloadMetadata["fileModTime"]
self.log.info("New file with modification time " + str(fileModTime) + " received and saved: " + str(filename))
break
try:
[payloadMetadata, payload] = self.get()
except:
self.log.error("Getting data failed.", exc_info=True)
break
class ZmqConnector:
context = None
auth = None
public_keys_dir = None
secret_keys_dir = None
puller = None
publisher = None
HOST = ''
opponent_id = None
available_player = 'XXX'
# Client message protocol:
#
# 1. ID: Player ID (random string created by online broker)
# 2. ACTION: status, command, recipient.
# 3. MATCH: relevant player match data
# Server message protocol:
#
# 1. Recipient: Player ID of recipient, used for filtering
# 2. ACTION: sender (SERVER or opponent player's ID), command (welcome, wait, ready, play)
# 3. Data: forwarded payload
def __init__(self, host='127.0.0.1'):
print("[zmq] Initializing ZMQ client object...")
self.HOST = host
self.context = zmq.Context()
def setup(self):
if not self.check_folder_structure():
return None
else:
self.server_auth()
self.bind_pull()
self.bind_pub()
def check_folder_structure(self):
keys_dir = os.path.join(os.getcwd(), '../certs')
print(f"[#] checking folder structure: {keys_dir}")
self.public_keys_dir = os.path.join(
keys_dir, 'public') # has the public keys of registered clients
self.secret_keys_dir = os.path.join(
keys_dir, 'private') # has the server's private cert
if not os.path.exists(keys_dir) \
and not os.path.exists(self.public_keys_dir) \
and not os.path.exists(self.secret_keys_dir):
print("[!!] Certificates folders are missing")
return False
else:
return True
def server_auth(self):
# Start an authenticator for this context
print("[#] Starting authenticator...")
self.auth = ThreadAuthenticator(self.context)
self.auth.start()
self.auth.allow(self.HOST)
# give authenticator access to approved clients' certificate directory
self.auth.configure_curve(domain='*', location=self.public_keys_dir)
def bind_pull(self, port=5555):
print("[zmq] Binding PULL socket : {}".format(port))
self.puller = self.context.socket(zmq.PULL)
# feed certificates to socket
server_secret_file = os.path.join(self.secret_keys_dir,
"server.key_secret")
self.puller.curve_publickey, self.puller.curve_secretkey = zmq.auth.load_certificate(
server_secret_file)
self.puller.curve_server = True # must come before bind
self.puller.bind("tcp://*:{}".format(port))
def pull_receive_multi(self):
try:
# message = self.puller.recv_multipart(flags=zmq.DONTWAIT)
message = self.puller.recv_multipart()
print(f"[zmq] Received :\n\t{datetime.datetime.now()}- {message}")
return message
except zmq.Again as a:
# print("[!zmq!] Error while getting messages: {}".format(a))
# print(traceback.format_exc())
return None
except zmq.ZMQError as e:
print("[!zmq!] Error while getting messages: {}".format(e))
print(traceback.format_exc())
return None
def bind_pub(self, port=5556):
print("[zmq] Binding PUB socket: {}".format(port))
self.publisher = self.context.socket(zmq.PUB)
# feed own and approved certificates to socket
server_secret_file = os.path.join(self.secret_keys_dir,
"server.key_secret")
self.publisher.curve_publickey, self.publisher.curve_secretkey = zmq.auth.load_certificate(
server_secret_file)
self.publisher.curve_server = True # must come before bind
self.publisher.bind("tcp://*:{}".format(port))
def send(self, recipient, info, payload):
message = list()
message.append(recipient.encode())
message.append(json.dumps(info).encode())
message.append(json.dumps(payload).encode())
self.pub_send_multi(message)
def pub_send_multi(self, message):
try:
self.publisher.send_multipart(message)
print(f"[zmq] Sent :\n\t{datetime.datetime.now()}- {message}")
except TypeError as e:
print("[!zmq!] TypeError while sending message: {}".format(e))
print(traceback.format_exc())
except ValueError as e:
print("[!zmq!] ValueError while sending message: {}".format(e))
print(traceback.format_exc())
except zmq.ZMQError as e:
print("[!zmq!] ZMQError while sending message: {}".format(e))
print(traceback.format_exc())
# GENERIC FUNCTIONS
def disconnect(self):
print("[zmq] Disconnecting client...")
for socket in (self.publisher, self.puller):
if socket is not None:
socket.close()
self.context.term()
def main():
"""
Runs SEND either in transmitter or receiver mode
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"-t",
"--transmit",
action="store_true",
help="Flag indicating that user will be transmitting files"
)
parser.add_argument(
"-r",
"--receive",
action="store_true",
help="Flag indicating that user will be receiving files"
)
parser.add_argument(
"--location",
help="Location of files to send/receive. Can be a specific file if tx."
)
parser.add_argument(
"--ip",
help="IP Address to form connection with"
)
parser.add_argument(
"--port",
nargs='?',
const=6000,
default=6000,
type=int,
help="Port to form connection with (only needed if using non-default)"
)
parser.add_argument(
"--public_key",
nargs='?',
help="Public Key of transmitter in plain-text (only needed if receiver)"
)
args=parser.parse_args()
# Security Authentication Thread
_generate_security_keys()
authenticator = ThreadAuthenticator(manager.ctx)
authenticator.start()
whitelist = [
"127.0.0.1",
args.ip
]
authenticator.allow(*whitelist)
authenticator.configure_curve(domain="*", location=PUBKEYS)
try:
if args.transmit:
thread = manager.publish_folder(
args.port,
args.location
)
elif args.receive:
thread = manager.subscribe_folder(
args.ip,
args.port,
args.location,
args.public_key
)
else:
raise ValueError(f"User did not specify transmit/receive")
except (OSError, ValueError):
raise
except KeyboardInterrupt:
pass
finally:
# Keep things rolling until the transfer is done or the thread dies from
# timing out
while thread.isAlive():
pass
thread.join()
# Clean up and close everything out
authenticator.stop()
# Use destroy versus term: https://github.com/zeromq/pyzmq/issues/991
manager.ctx.destroy()
class Device(Actor):
'''
The actor class implements all the management and control functions over its components
'''
def __init__(self, gModel, gModelName, dName, sysArgv):
'''
Constructor
'''
self.logger = logging.getLogger(__name__)
self.inst_ = self
self.appName = gModel["name"]
self.modelName = gModelName
self.name = dName
self.pid = os.getpid()
self.suffix = ""
if dName not in gModel["devices"]:
raise BuildError('Device "%s" unknown' % dName)
# In order to make the rest of the code work, we build an actor model for the device
devModel = gModel["devices"][dName]
self.model = {} # The made-up actor model
formals = devModel[
"formals"] # Formals are the same as those of the device (component)
self.model["formals"] = formals
devInst = {
"type": dName
} # There is a single instance, containing the device component
actuals = []
for arg in formals:
name = arg["name"]
actual = {}
actual["name"] = name
actual["param"] = name
actuals.append(actual)
devInst["actuals"] = actuals
self.model["instances"] = {dName: devInst}
self.model["locals"] = self.getMessageTypes(
devModel) # All messages are local
self.model["internals"] = {} # No internals
self.INT_RE = re.compile(r"^[-]?\d+$")
self.parseParams(sysArgv)
# Use czmq's context
czmq_ctx = Zsys.init()
self.context = zmq.Context.shadow(czmq_ctx.value)
Zsys.handler_reset() # Reset previous signal
# Context for app sockets
self.appContext = zmq.Context()
if Config.SECURITY:
(self.public_key,
self.private_key) = zmq.auth.load_certificate(const.appCertFile)
hosts = ['127.0.0.1']
try:
with open(const.appDescFile, 'r') as f:
content = yaml.load(f)
hosts += content.hosts
except:
pass
self.auth = ThreadAuthenticator(self.appContext)
self.auth.start()
self.auth.allow(*hosts)
self.auth.configure_curve(domain='*',
location=zmq.auth.CURVE_ALLOW_ANY)
else:
(self.public_key, self.private_key) = (None, None)
self.auth = None
self.appContext = self.context
try:
if os.path.isfile(const.logConfFile) and os.access(
const.logConfFile, os.R_OK):
spdlog_setup.from_file(const.logConfFile)
except Exception as e:
self.logger.error("error while configuring componentLogger: %s" %
repr(e))
messages = gModel[
"messages"] # Global message types (global on the network)
self.messageNames = []
for messageSpec in messages:
self.messageNames.append(messageSpec["name"])
locals_ = self.model[
"locals"] # Local message types (local to the host)
self.localNames = []
for messageSpec in locals_:
self.localNames.append(messageSpec["type"])
internals = self.model[
"internals"] # Internal message types (internal to the actor process)
self.internalNames = []
for messageSpec in internals:
self.internalNames.append(messageSpec["type"])
self.components = {}
instSpecs = self.model["instances"]
compSpecs = gModel["components"]
devSpecs = gModel["devices"]
for instName in instSpecs: # Create the component instances: the 'parts'
instSpec = instSpecs[instName]
instType = instSpec['type']
if instType in devSpecs:
typeSpec = devSpecs[instType]
else:
raise BuildError(
'Device type "%s" for instance "%s" is undefined' %
(instType, instName))
instFormals = typeSpec['formals']
instActuals = instSpec['actuals']
instArgs = self.buildInstArgs(instName, instFormals, instActuals)
# Check whether the component is C++ component
ccComponentFile = 'lib' + instType.lower() + '.so'
ccComp = os.path.isfile(ccComponentFile)
try:
if ccComp:
modObj = importlib.import_module('lib' + instType.lower())
self.components[instName] = modObj.create_component_py(
self, self.model, typeSpec, instName, instType,
instArgs, self.appName, self.name)
else:
self.components[instName] = Part(self, typeSpec, instName,
instType, instArgs)
except Exception as e:
traceback.print_exc()
self.logger.error("Error while constructing part '%s.%s': %s" %
(instType, instName, str(e)))
def getPortMessageTypes(self, ports, key, kinds, res):
for _name, spec in ports[key].items():
for kind in kinds:
typeName = spec[kind]
res.append({"type": typeName})
def getMessageTypes(self, devModel):
res = []
ports = devModel["ports"]
self.getPortMessageTypes(ports, "pubs", ["type"], res)
self.getPortMessageTypes(ports, "subs", ["type"], res)
self.getPortMessageTypes(ports, "reqs", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "reps", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "clts", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "srvs", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "qrys", ["req_type", "rep_type"], res)
self.getPortMessageTypes(ports, "anss", ["req_type", "rep_type"], res)
return res
# def getParameterValueType(self,param,defaultType):
# paramValue, paramType = None, None
# if defaultType != None:
# if defaultType == str:
# paramValue, paramType = param, str
# elif defaultType == int:
# paramValue, paramType = int(param),int
# elif defaultType == float:
# paramValue, paramType = float(param),float
# elif defaultType == bool:
# paramType = bool
# paramValue = False if param == "False" else True if param == "True" else None
# paramValue, paramType = bool(param),float
# else:
# if param == 'True':
# paramValue, paramType = True, bool
# elif param == 'False':
# paramValue, paramType = True, bool
# elif self.INT_RE.match(param) is not None:
# paramValue, paramType = int(param),int
# else:
# try:
# paramValue, paramType = float(param),float
# except:
# paramValue,paramType = str(param), str
# return (paramValue,paramType)
# def parseParams(self,sysArgv):
# self.params = { }
# formals = self.model["formals"]
# optList = []
# for formal in formals:
# key = formal["name"]
# default = None if "default" not in formal else formal["default"]
# self.params[key] = default
# optList.append("%s=" % key)
# try:
# opts,args = getopt.getopt(sysArgv, '', optList)
# except:
# self.logger.info("Error parsing actor options %s" % str(sysArgv))
# return
# # try:
# for opt in opts:
# optName2,optValue = opt
# optName = optName2[2:] # Drop two leading dashes
# if optName in self.params:
# defaultType = None if self.params[optName] == None else type(self.params[optName])
# paramValue,paramType = self.getParameterValueType(optValue,defaultType)
# if self.params[optName] != None:
# if paramType != type(self.params[optName]):
# raise BuildError("Type of default value does not match type of argument %s"
# % str((optName,optValue)))
# self.params[optName] = paramValue
# else:
# self.logger.info("Unknown argument %s - ignored" % optName)
# for param in self.params:
# if self.params[param] == None:
# raise BuildError("Required parameter %s missing" % param)
# def buildInstArgs(self,instName,formals,actuals):
# args = {}
# for formal in formals:
# argName = formal['name']
# argValue = None
# actual = next((actual for actual in actuals if actual['name'] == argName), None)
# defaultValue = None
# if 'default' in formal:
# defaultValue = formal['default']
# if actual != None:
# assert(actual['name'] == argName)
# if 'param'in actual:
# paramName = actual['param']
# if paramName in self.params:
# argValue = self.params[paramName]
# else:
# raise BuildError("Unspecified parameter %s referenced in %s"
# %(paramName,instName))
# elif 'value' in actual:
# argValue = actual['value']
# else:
# raise BuildError("Actual parameter %s has no value" % argName)
# elif defaultValue != None:
# argValue = defaultValue
# else:
# raise BuildError("Argument %s in %s has no defined value" % (argName,instName))
# args[argName] = argValue
# return args
# def isLocalMessage(self,msgTypeName):
# '''
# Return True if the message type is local
# '''
# return msgTypeName in self.localNames
#
# def getLocalIface(self):
# '''
# Return the IP address of the host-local network interface (usually 127.0.0.1)
# '''
# return self.localHost
#
# def getGlobalIface(self):
# '''
# Return the IP address of the global network interface
# '''
# return self.globalHost
#
# def setupIfaces(self):
# '''
# Find the IP addresses of the (host-)local and network(-global) interfaces
# '''
# (globalIPs,globalMACs,globalNames,localIP) = getNetworkInterfaces()
# assert len(globalIPs) > 0 and len(globalMACs) > 0
# globalIP = globalIPs[0]
# globalMAC = globalMACs[0]
# self.localHost = localIP
# self.globalHost = globalIP
# self.macAddress = globalMAC
#
def setup(self):
'''
Perform a setup operation on the actor (after the initial construction but before the activation of parts)
'''
self.logger.info("setup")
self.setupIfaces()
self.suffix = self.macAddress
self.disco = DiscoClient(self, self.suffix)
self.disco.start() # Start the discovery service client
self.disco.registerApp(
) # Register this actor with the discovery service
self.logger.info("device registered with disco")
self.deplc = DeplClient(self, self.suffix)
self.deplc.start()
ok = self.deplc.registerApp(isDevice=True)
self.logger.info("device %s registered with depl" %
("is" if ok else "is not"))
self.controls = {}
self.controlMap = {}
for inst in self.components:
comp = self.components[inst]
control = self.context.socket(zmq.PAIR)
control.bind('inproc://part_' + inst + '_control')
self.controls[inst] = control
self.controlMap[id(control)] = comp
if isinstance(comp, Part):
self.components[inst].setup(control)
else:
self.components[inst].setup()
# def start(self):
# '''
# Start and operate the actor (infinite polling loop)
# '''
# self.logger.info("starting")
# self.discoChannel = self.disco.channel # Private channel to the discovery service
# self.deplChannel = self.deplc.channel
#
# self.poller = zmq.Poller() # Set up the poller
# self.poller.register(self.deplChannel,zmq.POLLIN)
# self.poller.register(self.discoChannel,zmq.POLLIN)
#
# while 1:
# sockets = dict(self.poller.poll())
# if self.discoChannel in sockets: # If there is a message from a service, handle it
# msgs = self.recvChannelMessages(self.discoChannel)
# for msg in msgs:
# self.handleServiceUpdate(msg) # Handle message from disco service
# del sockets[self.discoChannel]
# elif self.deplChannel in sockets:
# msgs = self.recvChannelMessages(self.deplChannel)
# for msg in msgs:
# self.handleDeplMessage(msg) # Handle message from depl service
# del sockets[self.deplChannel]
# else:
# pass
def terminate(self):
self.logger.info("terminating")
for component in self.components.values():
component.terminate()
# self.devc.terminate()
self.disco.terminate()
# Clean up everything
# self.context.destroy()
time.sleep(1.0)
self.logger.info("terminated")
os._exit(0)
