def __init__(self, name, db_address,
pull_address, pub_address, pipelined=False,
log_level=logging.INFO, messages=sys.maxsize):
self.name = name
self.cache = DictDB()
self.db_address = db_address
self.pull_address = pull_address
self.pub_address = pub_address
self.pipelined = pipelined
self.message = None
self.cache.set('name', name.encode('utf-8'))
self.cache.set('pull_address', pull_address.encode('utf-8'))
self.cache.set('pub_address', pub_address.encode('utf-8'))
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
self.messages = messages
self.pull_socket = zmq_context.socket(zmq.PULL)
self.pull_socket.bind(self.pull_address)
self.pub_socket = zmq_context.socket(zmq.PUB)
self.pub_socket.bind(self.pub_address)
def __init__(self, name, db_address,
sub_address, pub_address, previous, to_client=True,
log_level=logging.INFO, messages=sys.maxsize):
self.name = name
self.cache = DictDB()
self.db_address = db_address
self.sub_address = sub_address
self.pub_address = pub_address
self.pipelined = not to_client
self.message = None
self.cache.set('name', name.encode('utf-8'))
self.cache.set('sub_address', sub_address.encode('utf-8'))
self.cache.set('pub_address', pub_address.encode('utf-8'))
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
self.messages = messages
self.sub_socket = zmq_context.socket(zmq.SUB)
self.sub_socket.setsockopt_string(zmq.SUBSCRIBE, previous)
self.sub_socket.connect(self.sub_address)
self.pub_socket = zmq_context.socket(zmq.PUB)
self.pub_socket.bind(self.pub_address)
def eval(self,
function,
payload: bytes,
messages: int = 1,
cache: str = ''):
"""
Execute single job.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: Binary message to be sent
:param messages: Number of messages expected to be sent back to the
client
:param cache: Cache data included in the message
:return: If messages=1, the result data. If messages > 1, a list with the results
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
push_socket.send(message.SerializeToString())
result = []
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
message.ParseFromString(message_data)
result.append(message.payload)
if messages == 1:
return result[0]
else:
return result
def __init__(self, name, db_address,
sub_addresses, pub_address, previous, to_client=True,
log_level=logging.INFO, messages=sys.maxsize):
self.name = name
self.cache = DictDB()
self.db_address = db_address
self.sub_addresses = sub_addresses
self.pub_address = pub_address
self.pipelined = not to_client
self.message = None
self.cache.set('name', name.encode('utf-8'))
for i, address in enumerate(sub_addresses):
self.cache.set('sub_address_{}'.format(i), address.encode('utf-8'))
self.cache.set('pub_address', pub_address.encode('utf-8'))
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
self.messages = messages
self.sub_sockets = list()
# Simple type checks
assert type(previous) == list
assert type(sub_addresses) == list
for address, prev in zip(self.sub_addresses, previous):
self.sub_sockets.append(zmq_context.socket(zmq.SUB))
self.sub_sockets[-1].setsockopt_string(zmq.SUBSCRIBE, prev)
self.sub_sockets[-1].connect(address)
self.pub_socket = zmq_context.socket(zmq.PUB)
self.pub_socket.bind(self.pub_address)
self.poller = zmq.Poller()
for sock in self.sub_sockets:
self.poller.register(sock, zmq.POLLIN)
def dummy_response():
dummy_router = zmq_context.socket(zmq.REP)
dummy_router.bind('inproc://broker')
msg = dummy_router.recv()
message = PalmMessage()
message.ParseFromString(msg)
dummy_router.send(msg)
return message.payload
def fake_client():
socket = zmq_context.socket(zmq.SUB)
socket.setsockopt_string(zmq.SUBSCRIBE, 'client')
socket.connect(listen_address)
# Pub sockets take some time.
time.sleep(1.0)
return socket.recv_multipart()
def job(self,
function,
generator,
messages: int = sys.maxsize,
cache: str = ''):
"""
Submit a job with multiple messages to a server.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: A generator that yields a series of binary messages.
:param messages: Number of messages expected to be sent back to the
client. Defaults to infinity (sys.maxsize)
:param cache: Cache data included in the message
:return: an iterator with the messages that are sent back to the client.
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
# Remember that sockets are not thread safe
sender_thread = Thread(target=self._sender,
args=(push_socket, function, generator, cache))
# Sender runs in background.
sender_thread.start()
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
if not client.decode('utf-8') == self.uuid:
raise ValueError(
'The client got a message that does not belong')
message = PalmMessage()
message.ParseFromString(message_data)
yield message.payload
def fake_terminator():
socket = zmq_context.socket(zmq.REP)
socket.bind('inproc://terminator')
message = PalmMessage()
message.ParseFromString(socket.recv())
print("Got the message at the terminator: ")
socket.send(b'0')
return message
def fake_router():
socket = zmq_context.socket(zmq.REQ)
socket.bind('inproc://broker')
# Give some time for everything to start
time.sleep(1.0)
message = PalmMessage()
message.payload = b'test message'
socket.send(message.SerializeToString())
socket.recv()
def broker():
socket = zmq_context.socket(zmq.ROUTER)
socket.bind(broker_address)
message = socket.recv_multipart()
# Unblock. Here you see why the actual router is complicated.
socket.send_multipart(message)
socket.send_multipart([b'publisher', b'', message[2]])
socket.close()
return b'router'
def __init__(self, name='', db_address='', push_address=None,
pull_address=None, log_level=logging.INFO,
messages=sys.maxsize):
self.uuid = str(uuid4())
# Give a random name if not given
if not name:
self.name = self.uuid
else:
self.name = name
# Configure the log handler
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
# Configure the connections.
self.push_address = push_address
self.pull_address = pull_address
if not db_address:
raise ValueError('db_address argument is mandatory')
self.db_address = db_address
self.db = zmq_context.socket(zmq.REQ)
self.db.connect(db_address)
self._get_config_from_master()
self.pull = zmq_context.socket(zmq.PULL)
self.pull.connect(self.push_address)
self.push = zmq_context.socket(zmq.PUSH)
self.push.connect(self.pull_address)
self.messages = messages
self.message = PalmMessage()
def dummy_initiator():
dummy_client = zmq_context.socket(zmq.REQ)
dummy_client.identity = b'0'
dummy_client.connect('inproc://gateway_router')
message = PalmMessage()
message.client = dummy_client.identity
message.pipeline = '0'
message.function = 'f.servername'
message.stage = 1
message.payload = b'This is a message'
dummy_client.send(message.SerializeToString())
def dummy_response():
dummy_router = zmq_context.socket(zmq.ROUTER)
dummy_router.bind('inproc://broker')
[target, empty, message] = dummy_router.recv_multipart()
dummy_router.send_multipart([target, empty, b'0'])
broker_message = PalmMessage()
broker_message.ParseFromString(message)
dummy_router.send_multipart([b'gateway_dealer', empty, message])
[target, message] = dummy_router.recv_multipart()
def fake_server(messages=1):
db_socket = zmq_context.socket(zmq.REP)
db_socket.bind('inproc://db')
pull_socket = zmq_context.socket(zmq.PULL)
pull_socket.bind('inproc://pull')
pub_socket = zmq_context.socket(zmq.PUB)
pub_socket.bind('inproc://pub')
# PUB-SUB takes a while
time.sleep(1.0)
for i in range(messages):
message_data = pull_socket.recv()
print(i)
message = PalmMessage()
message.ParseFromString(message_data)
topic = message.client
pub_socket.send_multipart([topic.encode('utf-8'), message_data])
def __init__(self,
name,
push_address,
pull_address,
broker_address="inproc://broker",
logger=None,
cache=None,
messages=sys.maxsize):
"""
:param name: Name of the component
:param listen_address: ZMQ socket address to listen to
:param socket_type: ZMQ inbound socket type
:param reply: True if the listening socket blocks waiting a reply
:param broker_address: ZMQ socket address for the broker
:param bind: True if socket has to bind, instead of connect.
:param logger: Logger instance
:param cache: Cache for shared data in the server
:param messages: Maximum number of inbound messages. Defaults to infinity.
:return:
"""
self.name = name.encode('utf-8')
self.push = zmq_context.socket(zmq.PUSH)
self.pull = zmq_context.socket(zmq.PULL)
self.push_address = push_address
self.pull_address = pull_address
self.push.bind(push_address)
self.pull.bind(pull_address)
self.broker = zmq_context.socket(zmq.REQ)
self.broker.identity = self.name
self.broker.connect(broker_address)
self.logger = logger
self.cache = cache
self.messages = messages
self.last_message = b''
def fake_router():
original_message = PalmMessage()
original_message.pipeline = 'pipeline'
original_message.client = 'client'
original_message.stage = 1
original_message.cache = '0'
original_message.function = 'function'
original_message.payload = b'0'
socket = zmq_context.socket(zmq.REQ)
socket.bind(broker_address)
serialized = original_message.SerializeToString()
socket.send(serialized)
socket.recv()
def __init__(self,
name,
hostname,
port,
broker_address="inproc://broker",
logger=None,
cache=None):
self.name = name.encode('utf-8')
self.hostname = hostname
self.port = port
self.logger = logger
self.broker = zmq_context.socket(zmq.REQ)
self.broker.identity = self.name
self.broker.connect(broker_address)
self.cache = cache
def __init__(self,
name='',
listen_address='inproc://gateway_router',
broker_address="inproc://broker",
cache=None,
logger=None,
messages=sys.maxsize):
self.name = 'gateway_dealer'.encode('utf-8')
self.listen_to = zmq_context.socket(zmq.DEALER)
self.listen_to.identity = b'dealer'
self.bind = False
self.listen_address = listen_address
self.broker = zmq_context.socket(zmq.DEALER)
self.broker.identity = self.name
self.broker.connect(broker_address)
self.logger = logger
self.cache = cache
self.messages = messages
self.reply = False
self.last_message = b''
if name:
self.logger.warning('Gateway dealer part is called "gateway_dealer",')
self.logger.warning('check that you have called this way')
def __init__(self,
server_name: str,
db_address: str,
push_address: str = None,
sub_address: str = None,
session: str = None,
logging_level: int = logging.INFO,
this_config=False):
self.server_name = server_name
self.db_address = db_address
if session:
self.pipeline = session
self.session_set = True
else:
self.pipeline = str(uuid4())
self.session_set = False
self.uuid = str(uuid4())
self.db = zmq_context.socket(zmq.REQ)
self.db.identity = self.uuid.encode('utf-8')
self.db.connect(db_address)
self.sub_address = sub_address
self.push_address = push_address
# Basic console logging
self.logger = logging.getLogger(name=self.uuid)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
self.logger.addHandler(handler)
self.logger.setLevel(logging_level)
if this_config:
self.logger.warning('Not fetching config from the server')
else:
self.logger.info('Fetching configuration from the server')
self._get_config_from_master()
# PUB-SUB takes a while
time.sleep(0.5)
def __init__(self,
name,
listen_address,
reply=True,
broker_address="inproc://broker",
logger=None,
cache=None,
max_workers=4,
messages=sys.maxsize):
self.name = name.encode('utf-8')
self.broker = zmq_context.socket(zmq.REP)
self.broker.identity = self.name
self.broker.connect(broker_address)
self.logger = logger
self.cache = cache
self.messages = messages
self.reply = reply
self.last_message = b''
self.url = listen_address
self.max_workers = max_workers
def do_GET(self):
socket = zmq_context.socket(zmq.REQ)
# This is the identity of the socket and the client.
identity = str(uuid4()).encode('utf-8')
socket.identity = identity
socket.connect(self.gateway_router_address)
function = self.path_parser(self.path)
if function:
message = PalmMessage()
message.pipeline = str(uuid4())
message.function = function
content_length = self.headers.get('content-length')
if content_length:
message.payload = self.rfile.read(int(content_length))
else:
message.payload = b'No Payload'
message.stage = 0
# Uses the same identity as the socket to tell the gateway
# router where it has to route to
message.client = identity
socket.send(message.SerializeToString())
message.ParseFromString(socket.recv())
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
else:
self.send_response(404)
self.send_header('Content-type', 'text/plain')
self.end_headers()
message = b'Not found'
self.wfile.write(message.payload)
socket.close()
return
def __init__(self, name, key, function='update',
broker_address='inproc://broker',
logger=None, messages=sys.maxsize):
"""
:param name: Name of the connection
:param key: Key of the dict to poll to
:param broker_address: ZMQ address of the broker
:param logger: Logger instance
:param messages: Maximum number of messages. Intended for debugging.
:return:
"""
self.name = name.encode('utf-8')
self.broker = zmq_context.socket(zmq.REQ)
self.broker.identity = self.name
self.broker.connect(broker_address)
self.logger = logger
self.messages = messages
self.key = key
self.function = function
self.etcd = Client()
self.wait_index = 0
def job_list(self,
function,
list_generator,
messages: int=sys.maxsize,
workers: int=sys.maxsize,
cache: str=''):
"""
Submit a job with multiple messages to a server.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: A generator that yields a series of binary messages.
:param messages: Number of messages expected to be sent back to the
client. Defaults to infinity (sys.maxsize)
:param list_generator: List generator.
:param workers: Number of workers, default 1:
:param cache: Cache data included in the message
:return: an iterator with the messages that are sent back to the client.
"""
def _generator_from_list(a_list):
"""
Stupid function to create a
generator from list.
:param some_list:
:return: a generator.
"""
for element in a_list:
yield element
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
if messages != sys.maxsize:
self.logger.debug('There are {} messages'.format(messages))
if workers != sys.maxsize:
self.logger.debug('There are {} workers'.format(workers))
self.logger.debug('Sending jobs to workers')
for i in range(0, messages, workers):
self.logger.debug('JSONs from {} to {}'.format(i, i + workers))
sub_list_generator = list_generator[i:i + workers]
# Remember that sockets are not thread safe
sub_generator = _generator_from_list(sub_list_generator)
sender_thread = Thread(target=self._sender,
args=(push_socket, function, sub_generator, cache))
# Sender runs in background.
sender_thread.start()
for i in range(sys.maxsize):
[client, message_data] = sub_socket.recv_multipart()
if not client.decode('utf-8') == self.uuid:
raise ValueError('The client got a message that does not belong')
message = PalmMessage()
message.ParseFromString(message_data)
if i == messages - 1:
return message.payload
else:
yield message.payload
def broker():
socket = zmq_context.socket(zmq.ROUTER)
socket.bind(broker_address)
