def set(self, value: bytes, key=None):
"""
Sets a key value pare in the remote database. If the key is not set,
the function returns a new key. Note that the order of the arguments
is reversed from the usual.
.. warning::
If the session attribute is specified, all the keys will be
prepended with the session id.
:param value: Value to be stored
:param key: Key for the k-v storage
:return: New key or the same key
"""
if not type(value) == bytes:
raise TypeError(
'First argument {} must be of type <bytes>'.format(value))
message = PalmMessage()
message.pipeline = str(
uuid4()) # For a set job, the pipeline is not important
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'set'])
message.payload = value
if key and self.session_set:
message.cache = ''.join([self.pipeline, key])
elif key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def start(self):
message = PalmMessage
self.logger.info('{} Successfully started'.format(self.name))
initial_message = PalmMessage()
initial_message.pipeline = '0'
initial_message.client = '0'
initial_message.stage = 0
initial_message.function = ''
initial_message.payload = b'0'
self.broker.send(initial_message.SerializeToString())
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('Got message {} from broker'.format(i))
message.ParseFromString(message_data)
for scattered in self.scatter(message):
self.push.send(scattered)
self.handle_feedback(self.pull.recv())
self.broker.send(self.reply_feedback())
return self.name
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
self.listen_to.bind(self.listen_address)
self.logger.info('{} successfully started'.format(self.name))
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('{} Got message from broker'.format(self.name))
message_data = self._translate_from_broker(message_data)
message.ParseFromString(message_data)
for scattered in self.scatter(message):
topic, scattered = self.handle_stream(scattered)
self.listen_to.send_multipart(
[topic.encode('utf-8'),
message.SerializeToString()])
self.logger.debug('Component {} Sent message. Topic {}'.format(
self.name, topic))
if self.reply:
feedback = self.listen_to.recv()
feedback = self._translate_to_broker(feedback)
self.handle_feedback(feedback)
self.broker.send(self.reply_feedback())
return self.name
def fake_router():
socket = zmq_context.socket(zmq.REQ)
socket.bind('inproc://broker')
# Give some time for everything to start
time.sleep(1.0)
message = PalmMessage()
message.payload = b'test message'
socket.send(message.SerializeToString())
socket.recv()
def dummy_initiator():
dummy_client = zmq_context.socket(zmq.REQ)
dummy_client.identity = b'0'
dummy_client.connect('inproc://gateway_router')
message = PalmMessage()
message.client = dummy_client.identity
message.pipeline = '0'
message.function = 'f.servername'
message.stage = 1
message.payload = b'This is a message'
dummy_client.send(message.SerializeToString())
def _sender(self, socket, function, generator, cache):
for payload in generator:
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
socket.send(message.SerializeToString())
def eval(self,
function,
payload: bytes,
messages: int = 1,
cache: str = ''):
"""
Execute single job.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: Binary message to be sent
:param messages: Number of messages expected to be sent back to the
client
:param cache: Cache data included in the message
:return: If messages=1, the result data. If messages > 1, a list with the results
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
push_socket.send(message.SerializeToString())
result = []
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
message.ParseFromString(message_data)
result.append(message.payload)
if messages == 1:
return result[0]
else:
return result
def fake_router():
original_message = PalmMessage()
original_message.pipeline = 'pipeline'
original_message.client = 'client'
original_message.stage = 1
original_message.cache = '0'
original_message.function = 'function'
original_message.payload = b'0'
socket = zmq_context.socket(zmq.REQ)
socket.bind(broker_address)
serialized = original_message.SerializeToString()
socket.send(serialized)
socket.recv()
def delete(self, key):
"""
Deletes data in the server's internal cache.
:param key: Key of the data to be deleted
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'delete'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def get(self, key):
"""
Gets a value from server's internal cache
:param key: Key for the data to be selected.
:return: Value
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'get'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv()
def set(self, value, key=None):
"""
Sets a key value pare in the remote database.
:param key:
:param value:
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'set'])
message.payload = value
if key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def do_GET(self):
socket = zmq_context.socket(zmq.REQ)
# This is the identity of the socket and the client.
identity = str(uuid4()).encode('utf-8')
socket.identity = identity
socket.connect(self.gateway_router_address)
function = self.path_parser(self.path)
if function:
message = PalmMessage()
message.pipeline = str(uuid4())
message.function = function
content_length = self.headers.get('content-length')
if content_length:
message.payload = self.rfile.read(int(content_length))
else:
message.payload = b'No Payload'
message.stage = 0
# Uses the same identity as the socket to tell the gateway
# router where it has to route to
message.client = identity
socket.send(message.SerializeToString())
message.ParseFromString(socket.recv())
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
else:
self.send_response(404)
self.send_header('Content-type', 'text/plain')
self.end_headers()
message = b'Not found'
self.wfile.write(message.payload)
socket.close()
return
def handle(self):
if self.request.method == 'POST':
try:
message = PalmMessage()
message.ParseFromString(self.request.data)
# This exports the message information
self.message = message
instruction = message.function.split('.')[1]
result = getattr(self, instruction)(message.payload)
message.payload = result
response_body = message.SerializeToString()
status = '200 OK'
except Exception as exc:
status = '500 Internal Server Error'
response_body = b''
else:
status = '405 Method not allowed'
response_body = b''
return status, response_body
def start(self):
self.logger.info('Launch Component {}'.format(self.name))
for i in range(self.messages):
self.logger.debug('Waiting for etcd')
if self.wait_index > 0:
response = self.etcd.wait(self.key, wait_index=self.wait_index)
else:
response = self.etcd.wait(self.key)
self.wait_index = response['node']['modifiedIndex']+1
self.logger.debug('New wait index: {}'.format(self.wait_index))
message = PalmMessage()
message.function = self.function
message.pipeline = ''
message.stage = 0
message.client = 'EtcdPoller'
message.payload = json.dumps(response).encode('utf-8')
# Build the PALM message that tells what to do with the data
self.broker.send(message.SerializeToString())
# Just unblock
self.logger.debug('blocked waiting broker')
self.broker.recv()
self.logger.debug('Got response from broker')
class Sink(Server):
"""
Minimal server that acts as a sink of multiple streams.
:param str name: Name of the server
:param str db_address: ZeroMQ address of the cache service.
:param str sub_addresses: List of addresses of the pub socket of the previous servers
:param str pub_address: Address of the pub socket
:param previous: List of names of the previous servers.
:param to_client: True if the message is sent back to the client. Defaults to True
:param log_level: Minimum output log level. Defaults to INFO
:param int messages: Total number of messages that the server processes. Defaults to Infty
Useful for debugging.
"""
def __init__(self, name, db_address,
sub_addresses, pub_address, previous, to_client=True,
log_level=logging.INFO, messages=sys.maxsize):
self.name = name
self.cache = DictDB()
self.db_address = db_address
self.sub_addresses = sub_addresses
self.pub_address = pub_address
self.pipelined = not to_client
self.message = None
self.cache.set('name', name.encode('utf-8'))
for i, address in enumerate(sub_addresses):
self.cache.set('sub_address_{}'.format(i), address.encode('utf-8'))
self.cache.set('pub_address', pub_address.encode('utf-8'))
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
self.messages = messages
self.sub_sockets = list()
# Simple type checks
assert type(previous) == list
assert type(sub_addresses) == list
for address, prev in zip(self.sub_addresses, previous):
self.sub_sockets.append(zmq_context.socket(zmq.SUB))
self.sub_sockets[-1].setsockopt_string(zmq.SUBSCRIBE, prev)
self.sub_sockets[-1].connect(address)
self.pub_socket = zmq_context.socket(zmq.PUB)
self.pub_socket.bind(self.pub_address)
self.poller = zmq.Poller()
for sock in self.sub_sockets:
self.poller.register(sock, zmq.POLLIN)
def _execution_handler(self):
for i in range(self.messages):
self.logger.debug('Server waiting for messages')
locked_socks = dict(self.poller.poll())
for sock in self.sub_sockets:
if sock in locked_socks:
message_data = sock.recv_multipart()[1]
self.logger.debug('Got message {}'.format(i + 1))
result = b'0'
self.message = PalmMessage()
try:
self.message.ParseFromString(message_data)
# Handle the fact that the message may be a complete pipeline
try:
if ' ' in self.message.function:
[server, function] = self.message.function.split()[
self.message.stage].split('.')
else:
[server, function] = self.message.function.split('.')
except IndexError:
raise ValueError('Pipeline call not correct. Review the '
'config in your client')
if not self.name == server:
self.logger.error('You called {}, instead of {}'.format(
server, self.name))
else:
try:
user_function = getattr(self, function)
self.logger.debug('Looking for {}'.format(function))
try:
result = user_function(self.message.payload)
except:
self.logger.error('User function gave an error')
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(
exc_type, exc_value, exc_traceback)
for l in lines:
self.logger.exception(l)
except KeyError:
self.logger.error(
'Function {} was not found'.format(function)
)
except DecodeError:
self.logger.error('Message could not be decoded')
# Do nothing if the function returns no value
if result is None:
continue
self.message.payload = result
topic, self.message = self.handle_stream(self.message)
self.pub_socket.send_multipart(
[topic.encode('utf-8'), self.message.SerializeToString()]
)
class Server(object):
"""
Standalone and minimal server that replies single requests.
:param str name: Name of the server
:param str db_address: ZeroMQ address of the cache service.
:param str pull_address: Address of the pull socket
:param str pub_address: Address of the pub socket
:param pipelined: True if the server is chained to another server.
:param log_level: Minimum output log level.
:param int messages: Total number of messages that the server processes.
Useful for debugging.
"""
def __init__(self, name, db_address,
pull_address, pub_address, pipelined=False,
log_level=logging.INFO, messages=sys.maxsize):
self.name = name
self.cache = DictDB()
self.db_address = db_address
self.pull_address = pull_address
self.pub_address = pub_address
self.pipelined = pipelined
self.message = None
self.cache.set('name', name.encode('utf-8'))
self.cache.set('pull_address', pull_address.encode('utf-8'))
self.cache.set('pub_address', pub_address.encode('utf-8'))
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
self.messages = messages
self.pull_socket = zmq_context.socket(zmq.PULL)
self.pull_socket.bind(self.pull_address)
self.pub_socket = zmq_context.socket(zmq.PUB)
self.pub_socket.bind(self.pub_address)
def handle_stream(self, message):
"""
Handle the stream of messages.
:param message: The message about to be sent to the next step in the
cluster
:return: topic (str) and message (PalmMessage)
The default behaviour is the following. If you leave this function
unchanged and pipeline is set to False, the topic is the ID of the
client, which makes the message return to the client. If the pipeline
parameter is set to True, the topic is set as the name of the server and
the step of the message is incremented by one.
You can alter this default behaviour by overriding this function.
Take into account that the message is also available in this function,
and you can change other parameters like the stage or the function.
"""
if self.pipelined:
topic = self.name
message.stage += 1
else:
topic = message.client
return topic, message
def echo(self, payload):
"""
Echo utility function that returns the unchanged payload. This
function is useful when the server is there as just to modify the
stream of messages.
:return: payload (bytes)
"""
return payload
def _execution_handler(self):
for i in range(self.messages):
self.logger.debug('Server waiting for messages')
message_data = self.pull_socket.recv()
self.logger.debug('Got message {}'.format(i + 1))
result = b'0'
self.message = PalmMessage()
try:
self.message.ParseFromString(message_data)
# Handle the fact that the message may be a complete pipeline
try:
if ' ' in self.message.function:
[server, function] = self.message.function.split()[
self.message.stage].split('.')
else:
[server, function] = self.message.function.split('.')
except IndexError:
raise ValueError('Pipeline call not correct. Review the '
'config in your client')
if not self.name == server:
self.logger.error('You called {}, instead of {}'.format(
server, self.name))
else:
try:
user_function = getattr(self, function)
self.logger.debug('Looking for {}'.format(function))
try:
result = user_function(self.message.payload)
except:
self.logger.error('User function gave an error')
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(
exc_type, exc_value, exc_traceback)
for l in lines:
self.logger.exception(l)
except KeyError:
self.logger.error(
'Function {} was not found'.format(function)
)
except DecodeError:
self.logger.error('Message could not be decoded')
self.message.payload = result
topic, self.message = self.handle_stream(self.message)
self.pub_socket.send_multipart(
[topic.encode('utf-8'), self.message.SerializeToString()]
)
def start(self, cache_messages=sys.maxsize):
"""
Start the server
:param cache_messages: Number of messages the cache service handles
before it shuts down. Useful for debugging
"""
threads = []
cache = CacheService('cache', self.db_address, logger=self.logger,
cache=self.cache, messages=cache_messages)
threads.append(cache.start)
threads.append(self._execution_handler)
with concurrent.futures.ThreadPoolExecutor(max_workers=len(threads)) as executor:
results = [executor.submit(thread) for thread in threads]
for future in concurrent.futures.as_completed(results):
try:
future.result()
except Exception as exc:
self.logger.error(
'This is critical, one of the components of the '
'server died')
lines = traceback.format_exception(*sys.exc_info())
for line in lines:
self.logger.error(line.strip('\n'))
return self.name.encode('utf-8')
class Worker(object):
"""
Standalone worker for the standalone master.
:param name: Name assigned to this worker server
:param db_address: Address of the db service of the master
:param push_address: Address the workers push to. If left blank, fetches
it from the master
:param pull_address: Address the workers pull from. If left blank,
fetches it from the master
:param log_level: Log level for this server.
:param messages: Number of messages before it is shut down.
"""
def __init__(self, name='', db_address='', push_address=None,
pull_address=None, log_level=logging.INFO,
messages=sys.maxsize):
self.uuid = str(uuid4())
# Give a random name if not given
if not name:
self.name = self.uuid
else:
self.name = name
# Configure the log handler
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
# Configure the connections.
self.push_address = push_address
self.pull_address = pull_address
if not db_address:
raise ValueError('db_address argument is mandatory')
self.db_address = db_address
self.db = zmq_context.socket(zmq.REQ)
self.db.connect(db_address)
self._get_config_from_master()
self.pull = zmq_context.socket(zmq.PULL)
self.pull.connect(self.push_address)
self.push = zmq_context.socket(zmq.PUSH)
self.push.connect(self.pull_address)
self.messages = messages
self.message = PalmMessage()
def _get_config_from_master(self):
if not self.push_address:
self.push_address = self.get('worker_push_address').decode('utf-8')
self.logger.info(
'Got worker push address: {}'.format(self.push_address))
if not self.pull_address:
self.pull_address = self.get('worker_pull_address').decode('utf-8')
self.logger.info(
'Got worker pull address: {}'.format(self.pull_address))
return {'push_address': self.push_address,
'pull_address': self.pull_address}
def _exec_function(self):
"""
Waits for a message and return the result
"""
message_data = self.pull.recv()
self.logger.debug('{} Got a message'.format(self.name))
result = b'0'
try:
self.message.ParseFromString(message_data)
try:
if ' ' in self.message.function:
instruction = self.message.function.split()[
self.message.stage].split('.')[1]
else:
instruction = self.message.function.split('.')[1]
except IndexError:
raise ValueError('Pipeline call not correct. Review the '
'config in your client')
try:
user_function = getattr(self, instruction)
self.logger.debug('Looking for {}'.format(instruction))
try:
result = user_function(self.message.payload)
self.logger.debug('{} Ok'.format(instruction))
except:
self.logger.error(
'{} User function {} gave an error'.format(
self.name, instruction)
)
lines = traceback.format_exception(*sys.exc_info())
self.logger.exception(lines[0])
except AttributeError:
self.logger.error(
'Function {} was not found'.format(instruction)
)
except DecodeError:
self.logger.error('Message could not be decoded')
return result
def start(self):
"""
Starts the server
"""
for i in range(self.messages):
self.message.payload = self._exec_function()
self.push.send(self.message.SerializeToString())
def set(self, value, key=None):
"""
Sets a key value pare in the remote database.
:param key:
:param value:
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'set'])
message.payload = value
if key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def get(self, key):
"""
Gets a value from server's internal cache
:param key: Key for the data to be selected.
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'get'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv()
def delete(self, key):
"""
Deletes data in the server's internal cache.
:param key: Key of the data to be deleted
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'delete'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
