def __init__(self, name, db_address, pull_address, pub_address, pipelined=False, log_level=logging.INFO, messages=sys.maxsize): self.name = name self.cache = DictDB() self.db_address = db_address self.pull_address = pull_address self.pub_address = pub_address self.pipelined = pipelined self.message = None self.cache.set('name', name.encode('utf-8')) self.cache.set('pull_address', pull_address.encode('utf-8')) self.cache.set('pub_address', pub_address.encode('utf-8')) self.logger = logging.getLogger(name=name) handler = logging.StreamHandler(sys.stdout) handler.setFormatter( logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) ) self.logger.addHandler(handler) self.logger.setLevel(log_level) self.messages = messages self.pull_socket = zmq_context.socket(zmq.PULL) self.pull_socket.bind(self.pull_address) self.pub_socket = zmq_context.socket(zmq.PUB) self.pub_socket.bind(self.pub_address)
def __init__(self, name, db_address, sub_address, pub_address, previous, to_client=True, log_level=logging.INFO, messages=sys.maxsize): self.name = name self.cache = DictDB() self.db_address = db_address self.sub_address = sub_address self.pub_address = pub_address self.pipelined = not to_client self.message = None self.cache.set('name', name.encode('utf-8')) self.cache.set('sub_address', sub_address.encode('utf-8')) self.cache.set('pub_address', pub_address.encode('utf-8')) self.logger = logging.getLogger(name=name) handler = logging.StreamHandler(sys.stdout) handler.setFormatter( logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) ) self.logger.addHandler(handler) self.logger.setLevel(log_level) self.messages = messages self.sub_socket = zmq_context.socket(zmq.SUB) self.sub_socket.setsockopt_string(zmq.SUBSCRIBE, previous) self.sub_socket.connect(self.sub_address) self.pub_socket = zmq_context.socket(zmq.PUB) self.pub_socket.bind(self.pub_address)
def eval(self, function, payload: bytes, messages: int = 1, cache: str = ''): """ Execute single job. :param function: Sting or list of strings following the format ``server.function``. :param payload: Binary message to be sent :param messages: Number of messages expected to be sent back to the client :param cache: Cache data included in the message :return: If messages=1, the result data. If messages > 1, a list with the results """ push_socket = zmq_context.socket(zmq.PUSH) push_socket.connect(self.push_address) sub_socket = zmq_context.socket(zmq.SUB) sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid) sub_socket.connect(self.sub_address) if type(function) == str: # Single-stage job pass elif type(function) == list: # Pipelined job. function = ' '.join(function) message = PalmMessage() message.function = function message.stage = 0 message.pipeline = self.pipeline message.client = self.uuid message.payload = payload if cache: message.cache = cache push_socket.send(message.SerializeToString()) result = [] for i in range(messages): [client, message_data] = sub_socket.recv_multipart() message.ParseFromString(message_data) result.append(message.payload) if messages == 1: return result[0] else: return result
def __init__(self, name, db_address, sub_addresses, pub_address, previous, to_client=True, log_level=logging.INFO, messages=sys.maxsize): self.name = name self.cache = DictDB() self.db_address = db_address self.sub_addresses = sub_addresses self.pub_address = pub_address self.pipelined = not to_client self.message = None self.cache.set('name', name.encode('utf-8')) for i, address in enumerate(sub_addresses): self.cache.set('sub_address_{}'.format(i), address.encode('utf-8')) self.cache.set('pub_address', pub_address.encode('utf-8')) self.logger = logging.getLogger(name=name) handler = logging.StreamHandler(sys.stdout) handler.setFormatter( logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) ) self.logger.addHandler(handler) self.logger.setLevel(log_level) self.messages = messages self.sub_sockets = list() # Simple type checks assert type(previous) == list assert type(sub_addresses) == list for address, prev in zip(self.sub_addresses, previous): self.sub_sockets.append(zmq_context.socket(zmq.SUB)) self.sub_sockets[-1].setsockopt_string(zmq.SUBSCRIBE, prev) self.sub_sockets[-1].connect(address) self.pub_socket = zmq_context.socket(zmq.PUB) self.pub_socket.bind(self.pub_address) self.poller = zmq.Poller() for sock in self.sub_sockets: self.poller.register(sock, zmq.POLLIN)
def dummy_response(): dummy_router = zmq_context.socket(zmq.REP) dummy_router.bind('inproc://broker') msg = dummy_router.recv() message = PalmMessage() message.ParseFromString(msg) dummy_router.send(msg) return message.payload
def fake_client(): socket = zmq_context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'client') socket.connect(listen_address) # Pub sockets take some time. time.sleep(1.0) return socket.recv_multipart()
def job(self, function, generator, messages: int = sys.maxsize, cache: str = ''): """ Submit a job with multiple messages to a server. :param function: Sting or list of strings following the format ``server.function``. :param payload: A generator that yields a series of binary messages. :param messages: Number of messages expected to be sent back to the client. Defaults to infinity (sys.maxsize) :param cache: Cache data included in the message :return: an iterator with the messages that are sent back to the client. """ push_socket = zmq_context.socket(zmq.PUSH) push_socket.connect(self.push_address) sub_socket = zmq_context.socket(zmq.SUB) sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid) sub_socket.connect(self.sub_address) if type(function) == str: # Single-stage job pass elif type(function) == list: # Pipelined job. function = ' '.join(function) # Remember that sockets are not thread safe sender_thread = Thread(target=self._sender, args=(push_socket, function, generator, cache)) # Sender runs in background. sender_thread.start() for i in range(messages): [client, message_data] = sub_socket.recv_multipart() if not client.decode('utf-8') == self.uuid: raise ValueError( 'The client got a message that does not belong') message = PalmMessage() message.ParseFromString(message_data) yield message.payload
def fake_terminator(): socket = zmq_context.socket(zmq.REP) socket.bind('inproc://terminator') message = PalmMessage() message.ParseFromString(socket.recv()) print("Got the message at the terminator: ") socket.send(b'0') return message
def fake_router(): socket = zmq_context.socket(zmq.REQ) socket.bind('inproc://broker') # Give some time for everything to start time.sleep(1.0) message = PalmMessage() message.payload = b'test message' socket.send(message.SerializeToString()) socket.recv()
def broker(): socket = zmq_context.socket(zmq.ROUTER) socket.bind(broker_address) message = socket.recv_multipart() # Unblock. Here you see why the actual router is complicated. socket.send_multipart(message) socket.send_multipart([b'publisher', b'', message[2]]) socket.close() return b'router'
def __init__(self, name='', db_address='', push_address=None, pull_address=None, log_level=logging.INFO, messages=sys.maxsize): self.uuid = str(uuid4()) # Give a random name if not given if not name: self.name = self.uuid else: self.name = name # Configure the log handler self.logger = logging.getLogger(name=name) handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ) self.logger.addHandler(handler) self.logger.setLevel(log_level) # Configure the connections. self.push_address = push_address self.pull_address = pull_address if not db_address: raise ValueError('db_address argument is mandatory') self.db_address = db_address self.db = zmq_context.socket(zmq.REQ) self.db.connect(db_address) self._get_config_from_master() self.pull = zmq_context.socket(zmq.PULL) self.pull.connect(self.push_address) self.push = zmq_context.socket(zmq.PUSH) self.push.connect(self.pull_address) self.messages = messages self.message = PalmMessage()
def dummy_initiator(): dummy_client = zmq_context.socket(zmq.REQ) dummy_client.identity = b'0' dummy_client.connect('inproc://gateway_router') message = PalmMessage() message.client = dummy_client.identity message.pipeline = '0' message.function = 'f.servername' message.stage = 1 message.payload = b'This is a message' dummy_client.send(message.SerializeToString())
def dummy_response(): dummy_router = zmq_context.socket(zmq.ROUTER) dummy_router.bind('inproc://broker') [target, empty, message] = dummy_router.recv_multipart() dummy_router.send_multipart([target, empty, b'0']) broker_message = PalmMessage() broker_message.ParseFromString(message) dummy_router.send_multipart([b'gateway_dealer', empty, message]) [target, message] = dummy_router.recv_multipart()
def fake_server(messages=1): db_socket = zmq_context.socket(zmq.REP) db_socket.bind('inproc://db') pull_socket = zmq_context.socket(zmq.PULL) pull_socket.bind('inproc://pull') pub_socket = zmq_context.socket(zmq.PUB) pub_socket.bind('inproc://pub') # PUB-SUB takes a while time.sleep(1.0) for i in range(messages): message_data = pull_socket.recv() print(i) message = PalmMessage() message.ParseFromString(message_data) topic = message.client pub_socket.send_multipart([topic.encode('utf-8'), message_data])
def __init__(self, name, push_address, pull_address, broker_address="inproc://broker", logger=None, cache=None, messages=sys.maxsize): """ :param name: Name of the component :param listen_address: ZMQ socket address to listen to :param socket_type: ZMQ inbound socket type :param reply: True if the listening socket blocks waiting a reply :param broker_address: ZMQ socket address for the broker :param bind: True if socket has to bind, instead of connect. :param logger: Logger instance :param cache: Cache for shared data in the server :param messages: Maximum number of inbound messages. Defaults to infinity. :return: """ self.name = name.encode('utf-8') self.push = zmq_context.socket(zmq.PUSH) self.pull = zmq_context.socket(zmq.PULL) self.push_address = push_address self.pull_address = pull_address self.push.bind(push_address) self.pull.bind(pull_address) self.broker = zmq_context.socket(zmq.REQ) self.broker.identity = self.name self.broker.connect(broker_address) self.logger = logger self.cache = cache self.messages = messages self.last_message = b''
def fake_router(): original_message = PalmMessage() original_message.pipeline = 'pipeline' original_message.client = 'client' original_message.stage = 1 original_message.cache = '0' original_message.function = 'function' original_message.payload = b'0' socket = zmq_context.socket(zmq.REQ) socket.bind(broker_address) serialized = original_message.SerializeToString() socket.send(serialized) socket.recv()
def __init__(self, name, hostname, port, broker_address="inproc://broker", logger=None, cache=None): self.name = name.encode('utf-8') self.hostname = hostname self.port = port self.logger = logger self.broker = zmq_context.socket(zmq.REQ) self.broker.identity = self.name self.broker.connect(broker_address) self.cache = cache
def __init__(self, name='', listen_address='inproc://gateway_router', broker_address="inproc://broker", cache=None, logger=None, messages=sys.maxsize): self.name = 'gateway_dealer'.encode('utf-8') self.listen_to = zmq_context.socket(zmq.DEALER) self.listen_to.identity = b'dealer' self.bind = False self.listen_address = listen_address self.broker = zmq_context.socket(zmq.DEALER) self.broker.identity = self.name self.broker.connect(broker_address) self.logger = logger self.cache = cache self.messages = messages self.reply = False self.last_message = b'' if name: self.logger.warning('Gateway dealer part is called "gateway_dealer",') self.logger.warning('check that you have called this way')
def __init__(self, server_name: str, db_address: str, push_address: str = None, sub_address: str = None, session: str = None, logging_level: int = logging.INFO, this_config=False): self.server_name = server_name self.db_address = db_address if session: self.pipeline = session self.session_set = True else: self.pipeline = str(uuid4()) self.session_set = False self.uuid = str(uuid4()) self.db = zmq_context.socket(zmq.REQ) self.db.identity = self.uuid.encode('utf-8') self.db.connect(db_address) self.sub_address = sub_address self.push_address = push_address # Basic console logging self.logger = logging.getLogger(name=self.uuid) handler = logging.StreamHandler(sys.stdout) handler.setFormatter( logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) self.logger.addHandler(handler) self.logger.setLevel(logging_level) if this_config: self.logger.warning('Not fetching config from the server') else: self.logger.info('Fetching configuration from the server') self._get_config_from_master() # PUB-SUB takes a while time.sleep(0.5)
def __init__(self, name, listen_address, reply=True, broker_address="inproc://broker", logger=None, cache=None, max_workers=4, messages=sys.maxsize): self.name = name.encode('utf-8') self.broker = zmq_context.socket(zmq.REP) self.broker.identity = self.name self.broker.connect(broker_address) self.logger = logger self.cache = cache self.messages = messages self.reply = reply self.last_message = b'' self.url = listen_address self.max_workers = max_workers
def do_GET(self): socket = zmq_context.socket(zmq.REQ) # This is the identity of the socket and the client. identity = str(uuid4()).encode('utf-8') socket.identity = identity socket.connect(self.gateway_router_address) function = self.path_parser(self.path) if function: message = PalmMessage() message.pipeline = str(uuid4()) message.function = function content_length = self.headers.get('content-length') if content_length: message.payload = self.rfile.read(int(content_length)) else: message.payload = b'No Payload' message.stage = 0 # Uses the same identity as the socket to tell the gateway # router where it has to route to message.client = identity socket.send(message.SerializeToString()) message.ParseFromString(socket.recv()) self.send_response(200) self.send_header('Content-type', 'text/plain') self.end_headers() else: self.send_response(404) self.send_header('Content-type', 'text/plain') self.end_headers() message = b'Not found' self.wfile.write(message.payload) socket.close() return
def __init__(self, name, key, function='update', broker_address='inproc://broker', logger=None, messages=sys.maxsize): """ :param name: Name of the connection :param key: Key of the dict to poll to :param broker_address: ZMQ address of the broker :param logger: Logger instance :param messages: Maximum number of messages. Intended for debugging. :return: """ self.name = name.encode('utf-8') self.broker = zmq_context.socket(zmq.REQ) self.broker.identity = self.name self.broker.connect(broker_address) self.logger = logger self.messages = messages self.key = key self.function = function self.etcd = Client() self.wait_index = 0
def job_list(self, function, list_generator, messages: int=sys.maxsize, workers: int=sys.maxsize, cache: str=''): """ Submit a job with multiple messages to a server. :param function: Sting or list of strings following the format ``server.function``. :param payload: A generator that yields a series of binary messages. :param messages: Number of messages expected to be sent back to the client. Defaults to infinity (sys.maxsize) :param list_generator: List generator. :param workers: Number of workers, default 1: :param cache: Cache data included in the message :return: an iterator with the messages that are sent back to the client. """ def _generator_from_list(a_list): """ Stupid function to create a generator from list. :param some_list: :return: a generator. """ for element in a_list: yield element push_socket = zmq_context.socket(zmq.PUSH) push_socket.connect(self.push_address) sub_socket = zmq_context.socket(zmq.SUB) sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid) sub_socket.connect(self.sub_address) if type(function) == str: # Single-stage job pass elif type(function) == list: # Pipelined job. function = ' '.join(function) if messages != sys.maxsize: self.logger.debug('There are {} messages'.format(messages)) if workers != sys.maxsize: self.logger.debug('There are {} workers'.format(workers)) self.logger.debug('Sending jobs to workers') for i in range(0, messages, workers): self.logger.debug('JSONs from {} to {}'.format(i, i + workers)) sub_list_generator = list_generator[i:i + workers] # Remember that sockets are not thread safe sub_generator = _generator_from_list(sub_list_generator) sender_thread = Thread(target=self._sender, args=(push_socket, function, sub_generator, cache)) # Sender runs in background. sender_thread.start() for i in range(sys.maxsize): [client, message_data] = sub_socket.recv_multipart() if not client.decode('utf-8') == self.uuid: raise ValueError('The client got a message that does not belong') message = PalmMessage() message.ParseFromString(message_data) if i == messages - 1: return message.payload else: yield message.payload
def broker(): socket = zmq_context.socket(zmq.ROUTER) socket.bind(broker_address)