Example #1
0
class HomeBase(Thread):

    def __init__(self):
        super(HomeBase, self).__init__(name="HomeBase")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.bind("tcp://*:7001")
        self._shutdown = False
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)

    def cleanup(self):
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv()
                msg += ", WORK RECEIVED "
                print msg
            if self._shutdown:
                break
        self.cleanup()
Example #2
0
class Listener(Thread):
    def __init__(self):
        super(Listener, self).__init__(name="Listener")

        self._shutdown = False
        self.context = Context()
        self.sub = self.context.socket(SUB)
        self.sub.bind('tcp://*:7000')
        self.sub.setsockopt(SUBSCRIBE, "")

        self.poller = Poller()
        self.poller.register(self.sub, POLLIN)

    def cleanup(self):
        self.sub.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.sub) == POLLIN:
                msg = self.sub.recv(flags=NOBLOCK)
                print msg
            if self._shutdown:
                break
        self.cleanup()
Example #3
0
class Listener(Thread):
    def __init__(self):
        super(Listener, self).__init__(name="Listener")

        self._shutdown = False
        self.context = Context()
        self.sub = self.context.socket(SUB)
        self.sub.bind('tcp://*:7000')
        self.sub.setsockopt(SUBSCRIBE, "")

        self.poller = Poller()
        self.poller.register(self.sub, POLLIN)

    def cleanup(self):
        self.sub.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.sub) == POLLIN:
                msg = self.sub.recv(flags=NOBLOCK)
                print msg
            if self._shutdown:
                break
        self.cleanup()
class Leatherneck(Thread):
    def __init__(self):
        super(Leatherneck, self).__init__(name="Leatherneck")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.connect("tcp://localhost:7000")
        self.push = self.context.socket(PUSH)
        self.push.connect("tcp://localhost:7001")
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        self._shutdown = False

    def cleanup(self):
        self.push.close()
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv()
                msg += " WORK COMPLETE, " + str(time())
                self.push.send(msg)
            if self._shutdown:
                break
        self.cleanup()
Example #5
0
class Stethoscope(Thread):
    def __init__(self, context, *args, **kw):
        self.context = context
        self.recv = self.context.socket(PAIR)
        self.recv.connect("inproc://#1")

        self.pub = self.context.socket(PUB)
        self.pub.connect('tcp://localhost:7003')
        self.pub.setsockopt(HWM, 1000)

        self.poller = Poller()
        self.poller.register(self.recv, POLLIN)
        super(Stethoscope, self).__init__(*args, **kw)

    def cleanup(self):
        self.recv.close()
        self.pub.close()

    def run(self):
        try:
            while not shutdown.is_set():
                socks = dict(self.poller.poll())
                if socks.get(self.recv) == POLLIN:
                    msg = self.recv.recv()
                    self.pub.send(msg, flags=NOBLOCK)
                    if msg == "DIE":
                        raise KillThread
        except KillThread:
            print "%s exiting..." % self.name
        finally:
            self.cleanup()
Example #6
0
    def run(self, *args):
        """Run the listener and answer to requests.
        """
        del args

        arec = AddressReceiver(max_age=self._max_age)
        arec.start()
        port = PORT

        try:
            self.listener = context.socket(REP)
            self.listener.bind("tcp://*:" + str(port))
            poller = Poller()
            poller.register(self.listener, POLLIN)
            while self.loop:
                socks = dict(poller.poll(1000))
                if socks:
                    if socks.get(self.listener) == POLLIN:
                        msg = self.listener.recv()
                else:
                    continue
                logger.debug("Replying to request: " + str(msg))
                msg = Message.decode(msg)
                self.listener.send_unicode(str(get_active_address(
                    msg.data["service"], arec)))
        except KeyboardInterrupt:
            # Needed to stop the nameserver.
            pass
        finally:
            arec.stop()
            self.listener.close()
Example #7
0
File: ns.py Project: ch-k/posttroll
def get_pub_address(name, timeout=10, nameserver="localhost"):
    """Get the address of the publisher for a given publisher *name* from the
    nameserver on *nameserver* (localhost by default).
    """

    # Socket to talk to server
    socket = context.socket(REQ)
    try:
        socket.setsockopt(LINGER, timeout * 1000)
        socket.connect("tcp://" + nameserver + ":" + str(PORT))

        poller = Poller()
        poller.register(socket, POLLIN)

        message = Message("/oper/ns", "request", {"service": name})
        socket.send(str(message))

        # Get the reply.
        sock = poller.poll(timeout=timeout * 1000)
        if sock:
            if sock[0][0] == socket:
                message = Message.decode(socket.recv(NOBLOCK))
                return message.data
        else:
            raise TimeoutError("Didn't get an address after %d seconds."
                               % timeout)
    finally:
        socket.close()
Example #8
0
class ControllerServer:
    def __init__(self):
        self.pairSockets = []
        self.poller = Poller()

    def connect(self, address):
        socket = zmq.Context().socket(zmq.PAIR)
        socket.connect(address)
        self.poller.register(socket, zmq.POLLIN)
        self.pairSockets.append(socket)
        return socket

    def send(self, message, socket=None):
        message = IO.wrap(message)
        if socket:
            socket.send(message)
        else:
            for socket in self.pairSockets:
                socket.send(message)

    def sendAction(self, action, socket=None):
        message = {'action': action}
        self.send(message, socket)

    def receive(self, receiveCb, timeout=-1):
        socks = dict(self.poller.poll(timeout))
        if socks:
            for si, socket in enumerate(self.pairSockets):
                if self.pairSockets[si] in socks and socks[
                        self.pairSockets[si]] == zmq.POLLIN:
                    msg = socket.recv()
                    msg, _ = IO.unwrap(msg)
                    receiveCb(si, msg)
Example #9
0
def get_pub_address(name, timeout=10, nameserver="localhost"):
    """Get the address of the publisher for a given publisher *name* from the
    nameserver on *nameserver* (localhost by default).
    """

    # Socket to talk to server
    socket = get_context().socket(REQ)
    try:
        socket.setsockopt(LINGER, timeout * 1000)
        socket.connect("tcp://" + nameserver + ":" + str(PORT))
        logger.debug('Connecting to %s',
                     "tcp://" + nameserver + ":" + str(PORT))
        poller = Poller()
        poller.register(socket, POLLIN)

        message = Message("/oper/ns", "request", {"service": name})
        socket.send_string(six.text_type(message))

        # Get the reply.
        sock = poller.poll(timeout=timeout * 1000)
        if sock:
            if sock[0][0] == socket:
                message = Message.decode(socket.recv_string(NOBLOCK))
                return message.data
        else:
            raise TimeoutError("Didn't get an address after %d seconds." %
                               timeout)
    finally:
        socket.close()
class Leatherneck(Thread):

    def __init__(self):
        super(Leatherneck, self).__init__(name="Leatherneck")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.connect("tcp://localhost:7000")
        self.push = self.context.socket(PUSH)
        self.push.connect("tcp://localhost:7001")
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        self._shutdown = False
        for th in t_enum():
            if th.name == "MainThread":
                self.mainthread = th

    def cleanup(self):
        print "Workers exiting..."
        self.push.close()
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            if not self.mainthread.is_alive():
                self._shutdown = True
                break
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv(flags=NOBLOCK)
                msg += " WORK COMPLETE, " + str(time())
                self.push.send(msg, flags=NOBLOCK)
            if self._shutdown:
                break
        self.cleanup()
class HomeBase(Thread):

    def __init__(self):
        super(HomeBase, self).__init__(name="HomeBase")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.bind("tcp://*:7001")
        self._shutdown = False
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        for th in t_enum():
            if th.name == "MainThread":
                self.mainthread = th

    def cleanup(self):
        print "Home exiting..."
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            if not self.mainthread.is_alive():
                self._shutdown = True
                break
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv(flags=NOBLOCK)
                msg += ", WORK RECEIVED "
                print msg
            if self._shutdown:
                break
        self.cleanup()
class HomeBase(Thread):
    def __init__(self):
        super(HomeBase, self).__init__(name="HomeBase")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.bind("tcp://*:7001")
        self._shutdown = False
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        for th in t_enum():
            if th.name == "MainThread":
                self.mainthread = th

    def cleanup(self):
        print "Home exiting..."
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            if not self.mainthread.is_alive():
                self._shutdown = True
                break
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv(flags=NOBLOCK)
                msg += ", WORK RECEIVED "
                print msg
            if self._shutdown:
                break
        self.cleanup()
class Leatherneck(Thread):
    def __init__(self):
        super(Leatherneck, self).__init__(name="Leatherneck")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.connect("tcp://localhost:7000")
        self.push = self.context.socket(PUSH)
        self.push.connect("tcp://localhost:7001")
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        self._shutdown = False
        for th in t_enum():
            if th.name == "MainThread":
                self.mainthread = th

    def cleanup(self):
        print "Workers exiting..."
        self.push.close()
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            if not self.mainthread.is_alive():
                self._shutdown = True
                break
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv(flags=NOBLOCK)
                msg += " WORK COMPLETE, " + str(time())
                self.push.send(msg, flags=NOBLOCK)
            if self._shutdown:
                break
        self.cleanup()
Example #14
0
class Leatherneck(Thread):

    def __init__(self):
        super(Leatherneck, self).__init__(name="Leatherneck")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.connect("tcp://localhost:7000")
        self.push = self.context.socket(PUSH)
        self.push.connect("tcp://localhost:7001")
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)
        self._shutdown = False

    def cleanup(self):
        self.push.close()
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv()
                msg += " WORK COMPLETE, " + str(time())
                self.push.send(msg)
            if self._shutdown:
                break
        self.cleanup()
Example #15
0
class Stethoscope(Thread):

    def __init__(self, context, *args, **kw):
        self.context = context
        self.recv = self.context.socket(PAIR)
        self.recv.connect("inproc://#1")

        self.pub = self.context.socket(PUB)
        self.pub.connect('tcp://localhost:7003')
        self.pub.setsockopt(HWM, 1000)

        self.poller = Poller()
        self.poller.register(self.recv, POLLIN)
        super(Stethoscope, self).__init__(*args, **kw)

    def cleanup(self):
        self.recv.close()
        self.pub.close()

    def run(self):
        try:
            while not shutdown.is_set():
                socks = dict(self.poller.poll())
                if socks.get(self.recv) == POLLIN:
                    msg = self.recv.recv()
                    self.pub.send(msg, flags=NOBLOCK)
                    if msg == "DIE":
                        raise KillThread
        except KillThread:
            print "%s exiting..." % self.name
        finally:
            self.cleanup()
Example #16
0
def _send_request(req_id, action, key, value=None):
    """ Generate a request ID, push the request to the node and wait for the
    result, filtering by the ID on the subscription.

    This should be a request/response call to the node, but since many must be
    possible at the same time, a PUSH/PULL AND PUB/SUB with an unique ID for
    each request is used instead.

    :return: the node response
    :rtype: []
    """
    # Create and connect the sockets.
    context = current_app.config["context"]
    subscriber = context.socket(SUB)
    subscriber.setsockopt_string(SUBSCRIBE, req_id)
    subscriber.connect(SUB_ENDPOINT)
    request = context.socket(PUSH)
    request.setsockopt(SNDTIMEO, _TIMEOUT)
    request.connect(PUSH_ENDPOINT)
    # Push the request.
    request.send_json([req_id, action, key, value])
    # Wait for the response from the publisher.
    poller = Poller()
    poller.register(subscriber, POLLIN)
    sockets = dict(poller.poll(_TIMEOUT))
    if subscriber not in sockets:  # no response, time out
        raise InternalServerError()
    # Return the response, without the unique request ID
    return subscriber.recv_multipart()[1:]
Example #17
0
class MirrorWatcher(Thread):

    """Watches a other server.
    """

    def __init__(self, holder, host, pubport, reqport, sched):
        Thread.__init__(self)
        self._holder = holder
        self._pubaddress = "tcp://" + host + ":" + str(pubport)
        self._reqaddress = "tcp://" + host + ":" + str(reqport)

        self._req = SimpleRequester(host, reqport)

        self._subsocket = context.socket(SUB)
        self._subsocket.connect(self._pubaddress)
        self._subsocket.setsockopt(SUBSCRIBE, "pytroll")
        self._poller = Poller()
        self._poller.register(self._subsocket, POLLIN)
        self._lock = Lock()
        self._loop = True
        self._sched = sched

    def run(self):
        last_hb = datetime.now()
        minutes = 2
        while self._loop:
            if datetime.now() - last_hb > timedelta(minutes=minutes):
                logger.error("No heartbeat from " + str(self._pubaddress))
                last_hb = datetime.now()
                minutes = 1440
            socks = dict(self._poller.poll(2000))
            if (socks and
                    self._subsocket in socks and
                    socks[self._subsocket] == POLLIN):
                message = Message.decode(self._subsocket.recv())
            else:
                continue
            if message.type == "have":
                sat = message.data["satellite"]
                key = strp_isoformat(message.data["timecode"])
                elevation = message.data["elevation"]
                quality = message.data.get("quality", 100)
                data = _MirrorGetter(self._req, sat, key)
                self._holder.add(sat, key, elevation, quality, data)
            if message.type == "heartbeat":
                logger.debug("Got heartbeat from " + str(self._pubaddress)
                             + ": " + str(message))
                self._sched.mirror_next_pass = message.data["next_pass"]
                last_hb = datetime.now()
                minutes = 2

    def stop(self):
        """Stop the watcher
        """
        self._loop = False
        self._req.stop()
        self._subsocket.setsockopt(LINGER, 0)
        self._subsocket.close()
Example #18
0
def start_poll(pull, push, sub, hash_checker):
    global INPUT_DATA
    seq_no = 0
    poller = Poller()
    poller.register(pull, zmq.POLLIN)
    poller.register(sub, zmq.POLLIN)
    while True:
        try:
            socks = dict(poller.poll())
        except KeyboardInterrupt:
            return

        if sub in socks:
            log.debug('recv -> {}'.format(sub.recv_multipart()))
            try:
                log.info('Concatting...')
                concat_string = process()
                concat_string_hash = HASHER(concat_string).digest()

                # Send final hash
                push.send_multipart([
                    MessageType.INPUT.value,
                    seq_no.to_bytes(2, byteorder='big'),
                    concat_string_hash,
                ])
                log.debug('send -> {} | {} | {}', MessageType.INPUT.name,
                          seq_no, concat_string_hash.hex())

                # Send original concatted string
                push.send_multipart([
                    MessageType.COMMIT.value,
                    seq_no.to_bytes(2, byteorder='big'),
                    concat_string,
                ])
                log.debug('send -> {} | {} | {}...', MessageType.COMMIT.name,
                          seq_no, concat_string[:20])

                INPUT_DATA.clear()
                log.info("Reset concat")

                if seq_no == 0xFFFF:
                    seq_no = 0
                else:
                    seq_no += 1
            except Exception as e:
                log.error("Could not concat inputs -> {}".format(e))

        if pull in socks:
            inp = pull.recv()
            log.debug('recv -> {}'.format(inp.hex()))
            try:
                hash_checker.check(inp)
            except:
                log.warn('Bad hash received! -> {}', inp)
                continue
            INPUT_DATA.append(inp)
            log.info('Input added | Elements: {}'.format(len(INPUT_DATA)))
Example #19
0
class Requester:
    def __init__(self, timeout=-1, poolSize=4):
        self.timeout = timeout
        self.socket = None
        self.numConnections = 0
        self.poller = Poller()

    def addServer(self, address, bind=False, linger=-1):
        if self.socket is None:
            self.socket = zmq.Context().socket(zmq.REQ)
            # self.socket.setsockopt(zmq.LINGER, linger)
            self.poller.register(self.socket, zmq.POLLIN)

        if bind:
            self.socket.bind(address)
        else:
            self.socket.connect(address)

        self.numConnections += 1

    def numConnections(self):
        return self.numConnections

    def request(self, message):
        if message is None or self.socket is None: return None
        message = IO.wrap(message)

        self.socket.send(message)
        socks = dict(self.poller.poll(self.timeout))
        if socks:
            replyMessage = self.socket.recv()

            reply, _ = IO.unwrap(replyMessage)
            return reply

        return None

    def requestAllServers(self, message):
        replies = []
        for n in range(self.numConnections):
            reply = self.request(message)
            replies.append(reply)

        return replies

    def requestAllServersCb(self, message, replyCb):
        for n in range(self.numConnections):
            reply = self.request(message)
            replyCb(reply)

    def requestAllServersPool(self, message, poolSize=0):
        if poolSize < 1: poolSize = self.numConnections
        pool = Pool(processes=poolSize)
        replies = pool.map(self.request,
                           [message for n in range(self.numConnections)])
        return replies
Example #20
0
class MirrorWatcher(Thread):
    """Watches a other server.
    """
    def __init__(self, holder, host, pubport, reqport, sched):
        Thread.__init__(self)
        self._holder = holder
        self._pubaddress = "tcp://" + host + ":" + str(pubport)
        self._reqaddress = "tcp://" + host + ":" + str(reqport)

        self._req = SimpleRequester(host, reqport)

        self._subsocket = get_context().socket(SUB)
        self._subsocket.connect(self._pubaddress)
        self._subsocket.setsockopt_string(SUBSCRIBE, "pytroll")
        self._poller = Poller()
        self._poller.register(self._subsocket, POLLIN)
        self._lock = Lock()
        self._loop = True
        self._sched = sched

    def run(self):
        last_hb = datetime.now()
        minutes = 2
        while self._loop:
            if datetime.now() - last_hb > timedelta(minutes=minutes):
                logger.error("No heartbeat from " + str(self._pubaddress))
                last_hb = datetime.now()
                minutes = 1440
            socks = dict(self._poller.poll(2000))
            if (socks and self._subsocket in socks
                    and socks[self._subsocket] == POLLIN):
                message = Message.decode(self._subsocket.recv())
            else:
                continue
            if message.type == "have":
                sat = message.data["satellite"]
                key = strp_isoformat(message.data["timecode"])
                elevation = message.data["elevation"]
                quality = message.data.get("quality", 100)
                data = _MirrorGetter(self._req, sat, key)
                self._holder.add(sat, key, elevation, quality, data)
            if message.type == "heartbeat":
                logger.debug("Got heartbeat from " + str(self._pubaddress) +
                             ": " + str(message))
                self._sched.mirror_next_pass = message.data["next_pass"]
                last_hb = datetime.now()
                minutes = 2

    def stop(self):
        """Stop the watcher
        """
        self._loop = False
        self._req.stop()
        self._subsocket.setsockopt(LINGER, 0)
        self._subsocket.close()
Example #21
0
	def refresh_serum_index(self):
		try:
			rawData = []
			for i in range(3):
				socket = self.context.socket(REQ)
				socket.connect("tcp://serum-server:6900")
				socket.setsockopt(LINGER, 0)
				poller = Poller()
				poller.register(socket, POLLIN)

				socket.send(dumps({"endpoint": "list"}))
				responses = poller.poll(10000)

				if len(responses) != 0:
					response = socket.recv()
					socket.close()
					rawData = loads(response)
					break
				else:
					socket.close()

			for market in rawData["markets"]:
				base, quote = market["name"].split("/", 1)
				if base not in self.serumIndex:
					self.serumIndex[base] = []
				self.serumIndex[base].append({"id": market["address"], "name": base, "base": base, "quote": quote, "image": None, "program": market["programId"]})

			for token in rawData["tokenList"]:
				symbol = token["symbol"].upper()
				if symbol not in self.serumIndex:
					self.serumIndex[symbol] = []
				processed = []
				for market in self.serumIndex[symbol]:
					processed.append(market["quote"])
					market["name"] = token["name"]
					market["image"] = token.get("logoURI")
				for extension, address in token.get("extensions", {}).items():
					if extension.startswith("serumV3"):
						quote = extension.removeprefix("serumV3").upper()
						if quote not in processed:
							processed.append(quote)
							self.serumIndex[symbol].append({"id": address, "name": token["name"], "base": symbol, "quote": quote, "image": token.get("logoURI"), "program": "9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin"})
				if len(self.serumIndex[symbol]) == 0:
					self.serumIndex.pop(symbol)
				elif self.serumIndex[symbol][0]["quote"] != "USDC":
					usdcMarket = None
					for index, market in enumerate(self.serumIndex[symbol]):
						if market["quote"] == "USDC":
							usdcMarket = self.serumIndex[symbol].pop(index)
							break
					if usdcMarket is not None:
						self.serumIndex[symbol].insert(0, usdcMarket)

		except Exception:
			print(format_exc())
Example #22
0
    def __stop(self):
        """ try to stop all of this Role's services """

        # send commands
        poller = Poller()
        for (pipe, svc) in self.__services.items():
            pipe.send_string('STOP')
            self.logger.debug('sent STOP command to %s service' % svc)
            poller.register(pipe, POLLIN)

        # give services a few seconds to cleanup and exit before checking responses
        sleep(1)

        max_attempts = len(self.__services)
        attempts = 0

        while self.__some_alive() and attempts < max_attempts:
            attempts += 1

            # poll for any replies
            items = dict(poller.poll(60000))  # wait for messages

            # mark responding services as stopped
            alive = dict(self.__services)  # make copy
            for (pipe, svc) in alive.items():
                if pipe in items:
                    reply = pipe.recv_string()
                    if 'STOPPED' == reply:
                        self.logger.debug('received STOPPED control reply from %s service' % svc)
                        svc.join(timeout=5)  # STOPPED response should be sent right before svc exit
                        if svc.is_alive():
                            self.logger.error('%s service is still alive; not waiting' % svc)
                        else:
                            self.logger.debug('%s service thread stopped' % svc)
                        poller.unregister(pipe)
                        pipe.close()
                        del (self.__services[pipe])
                    else:
                        self.logger.debug('unknown control reply: %s' % reply)

            # log some useful info
            if len(self.__services) > 0:
                msg = '%s services still alive after %d cycles; ' % (
                    [str(s) for s in self.__services.values()], attempts)
                if attempts < max_attempts:
                    msg += 'waiting'
                else:
                    msg += 'giving up'
                self.logger.debug(msg)
Example #23
0
    def listen(self, topics: Iterable[str] = ("log|", )):
        """
        Listens to incoming log messages on the ZeroMQ socket(s).
        Blocks until `Ctrl + C` is pressed,
        or ``SIGINT`` or ``SIGTERM`` is received by the process.

        Returns immediately if either a bind address nor at least
        one connect address is set.

        :param topics:
            The topic to subscribe with.
            Can be used for selecting log messages from specific source.
            Defaults to ``["log|"]``.
        :type topics: Iterable[str]
        """
        if not self._bind_address and not self._connect_addresses:
            return
        poller = Poller()
        if self._bind_address:
            self._pull_socket = self._ctx.socket(PULL)
            self._pull_socket.bind('tcp://' + self._bind_address)
            poller.register(self._pull_socket, POLLIN)
        if self._connect_addresses:
            self._sub_socket = self._ctx.socket(SUB)
            for topic in topics:
                self._sub_socket.setsockopt(
                    SUBSCRIBE, topic.encode(encoding=_TOPIC_ENCODING))
            for address in self._connect_addresses:
                self._sub_socket.connect('tcp://' + address)
            poller.register(self._sub_socket, POLLIN)
        try:
            while not is_stopped():
                events = dict(poller.poll(self._interval * 1000))
                if self._pull_socket in events \
                        and events[self._pull_socket] == POLLIN:
                    data = self._pull_socket.recv_multipart(flags=NOBLOCK)
                    self._handle_message(data)
                if self._sub_socket in events \
                        and events[self._sub_socket] == POLLIN:
                    data = self._sub_socket.recv_multipart(flags=NOBLOCK)
                    self._handle_message(data)
        finally:
            if self._pull_socket:
                self._pull_socket.close()
                self._pull_socket = None
            if self._sub_socket:
                self._sub_socket.close()
                self._sub_socket = None
Example #24
0
	def _request_quote(cls, request, ticker, context=None):
		try:
			socket = context.socket(REQ)
			socket.connect("tcp://serum-server:6900")
			socket.setsockopt(LINGER, 0)
			poller = Poller()
			poller.register(socket, POLLIN)

			socket.send(bytes(dumps({"endpoint": "quote", "marketAddress": ticker.get("id"), "program": ticker.get("symbol")}), encoding='utf8'))
			responses = poller.poll(5000)

			if len(responses) != 0:
				response = socket.recv()
				socket.close()
				rawData = loads(response)
			else:
				socket.close()
				return [{}, ""]
		except:
			return [{}, ""]

		coinThumbnail = static_storage.icon if ticker.get("image") is None else ticker.get("image")

		price = [float(rawData["price"])]
		priceChange = 0
		volume = 0
		tokenImage = static_storage.icon

		base = "USD" if ticker.get("base") in AbstractProvider.stableCoinTickers else ticker.get("base")
		quote = "USD" if ticker.get("quote") in AbstractProvider.stableCoinTickers else ticker.get("quote")
		payload = {
			"quotePrice": "{:,.10f}".format(price[0]).rstrip('0').rstrip('.') + " " + quote,
			"quoteVolume": "{:,.4f}".format(volume).rstrip('0').rstrip('.') + " " + base,
			"title": ticker.get("name"),
			"thumbnailUrl": coinThumbnail,
			"messageColor": "amber" if priceChange == 0 else ("green" if priceChange > 0 else "red"),
			"sourceText": "Data from Serum DEX",
			"platform": "Serum",
			"raw": {
				"quotePrice": [price[0]],
				"quoteVolume": [volume],
				"timestamp": time()
			}
		}

		return [payload, ""]
    def _request_translation(self, language, country, key, plural):
        """ Start up a worker, sync it and then send it a translation request.

        Returns the result, shuts down the worker at the end as well.

        Fails the current test, if something goes wrong.
        """
        request = [
            language, country if country is not None else "", key,
            str(plural) if plural is not None else ""]
        request = [x.encode(_ENCODING) for x in request]
        context = Context()
        # Create synchronization and backend sockets.
        try:
            sync_socket = context.socket(ROUTER)
            try:
                sync_socket.bind(_SYNC_ENDPOINT)
                backend = context.socket(DEALER)
                try:
                    backend.bind(_REQUEST_ENDPOINT)
                    worker_threads, worker_identities = _start_workers(
                        context, sync_socket, 1, _TIMEOUT)
                    poller = Poller()
                    poller.register(backend, POLLIN)
                    poller.register(sync_socket, POLLIN)
                    # Send request.
                    backend.send_multipart(
                        [worker_identities[0], b""] + request)
                    sockets = dict(poller.poll(_TIMEOUT))
                    # Shutdown worker.
                    _shut_down_workers(
                        sync_socket, worker_threads, worker_identities,
                        _TIMEOUT / 1000.0)
                    if backend in sockets:
                        # Return translation.
                        return backend.recv_multipart()[2].decode("utf-8")
                    self.fail("Worker did not response the request in time.")
                finally:
                    backend.set(LINGER, 0)
                    backend.close()
            finally:
                sync_socket.set(LINGER, 0)
                sync_socket.close()
        finally:
            context.destroy(linger=0)
Example #26
0
class Server(object):
    def __init__(self, port):
        self._ctx = Context()

        self._socket = self._ctx.socket(DEALER)
        self._socket.bind('tcp://*:%s' % port)

        self._poller = Poller()
        self._poller.register(self._socket, POLLIN)

        self._clients = {}

        self._actions = {'register': self._register_client}

    def _register_client(self, message):
        if message['pseudonym'] not in self._clients:
            self._clients[message['pseudonym']] = message['cert']
            message = {'type': 'Accepted'}

            self._send_msg(message)
        else:
            message = {'type': 'Failed'}
            self._send_msg(message)

    def _send_msg(self, message):
        pickled_message = dumps(message)
        self._socket.send(pickled_message)

        print('Sent --> ', message)

    def _recv_msg(self):
        pickled_message = self._socket.recv()
        received_message = loads(pickled_message)

        print("Received --> ", received_message)
        return received_message

    def execute(self):
        while True:
            sockets = dict(self._poller.poll())

            if sockets.get(self._socket) == POLLIN:
                received_message = self._recv_msg()

                self._actions[received_message['type']](received_message)
Example #27
0
    def execute_parser_request(endpoint, parameters, timeout=5):
        socket = TickerParser.zmqContext.socket(REQ)
        payload, responseText = None, None
        socket.connect("tcp://parser:6900")
        socket.setsockopt(LINGER, 0)
        poller = Poller()
        poller.register(socket, POLLIN)

        socket.send_multipart([endpoint] + parameters)
        responses = poller.poll(timeout * 1000)

        if len(responses) != 0:
            response = socket.recv_multipart()
            socket.close()
            return response
        else:
            socket.close()
            raise Exception("time out")
        return None
 def _server(self, response):
     """ Wait for a client request, record it and send the response. """
     context = Context()
     try:
         socket = context.socket(REP)
         try:
             socket.set(LINGER, 0)
             socket.bind("tcp://*:{}".format(_PORT))
             poller = Poller()
             poller.register(socket, POLLIN)
             sockets = dict(poller.poll(_TIMEOUT))
             if socket in sockets:
                 self._client_request = socket.recv_multipart()
                 if response:
                     socket.send_multipart(response)
         finally:
             socket.close()
     finally:
         context.destroy(linger=0)
Example #29
0
 def _server(self, response):
     """ Wait for a client request, record it and send the response. """
     context = Context()
     try:
         socket = context.socket(REP)
         try:
             socket.set(LINGER, 0)
             socket.bind("tcp://*:{}".format(_PORT))
             poller = Poller()
             poller.register(socket, POLLIN)
             sockets = dict(poller.poll(_TIMEOUT))
             if socket in sockets:
                 self._client_request = socket.recv_multipart()
                 if response:
                     socket.send_multipart(response)
         finally:
             socket.close()
     finally:
         context.destroy(linger=0)
Example #30
0
class ZMQPoller(object):
    """A poller that can be used in the tornado IOLoop.
    
    This simply wraps a regular zmq.Poller, scaling the timeout
    by 1000, so that it is in seconds rather than milliseconds.
    """
    
    def __init__(self):
        self._poller = Poller()
    
    @staticmethod
    def _map_events(events):
        """translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR"""
        z_events = 0
        if events & IOLoop.READ:
            z_events |= POLLIN
        if events & IOLoop.WRITE:
            z_events |= POLLOUT
        if events & IOLoop.ERROR:
            z_events |= POLLERR
        return z_events
    
    @staticmethod
    def _remap_events(z_events):
        """translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR"""
        events = 0
        if z_events & POLLIN:
            events |= IOLoop.READ
        if z_events & POLLOUT:
            events |= IOLoop.WRITE
        if z_events & POLLERR:
            events |= IOLoop.ERROR
        return events
    
    def register(self, fd, events):
        return self._poller.register(fd, self._map_events(events))
    
    def modify(self, fd, events):
        return self._poller.modify(fd, self._map_events(events))
    
    def unregister(self, fd):
        return self._poller.unregister(fd)
    
    def poll(self, timeout):
        """poll in seconds rather than milliseconds.
        
        Event masks will be IOLoop.READ/WRITE/ERROR
        """
        z_events = self._poller.poll(1000*timeout)
        return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ]
    
    def close(self):
        pass
Example #31
0
class ZMQPoller(object):
    """A poller that can be used in the tornado IOLoop.
    
    This simply wraps a regular zmq.Poller, scaling the timeout
    by 1000, so that it is in seconds rather than milliseconds.
    """
    
    def __init__(self):
        self._poller = Poller()
    
    @staticmethod
    def _map_events(events):
        """translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR"""
        z_events = 0
        if events & IOLoop.READ:
            z_events |= POLLIN
        if events & IOLoop.WRITE:
            z_events |= POLLOUT
        if events & IOLoop.ERROR:
            z_events |= POLLERR
        return z_events
    
    @staticmethod
    def _remap_events(z_events):
        """translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR"""
        events = 0
        if z_events & POLLIN:
            events |= IOLoop.READ
        if z_events & POLLOUT:
            events |= IOLoop.WRITE
        if z_events & POLLERR:
            events |= IOLoop.ERROR
        return events
    
    def register(self, fd, events):
        return self._poller.register(fd, self._map_events(events))
    
    def modify(self, fd, events):
        return self._poller.modify(fd, self._map_events(events))
    
    def unregister(self, fd):
        return self._poller.unregister(fd)
    
    def poll(self, timeout):
        """poll in seconds rather than milliseconds.
        
        Event masks will be IOLoop.READ/WRITE/ERROR
        """
        z_events = self._poller.poll(1000*timeout)
        return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ]
    
    def close(self):
        pass
Example #32
0
    def execute_database_request(endpoint, parameters, timeout=1):
        socket = DatabaseConnector.zmqContext.socket(REQ)
        payload, responseText = None, None
        socket.connect("tcp://database:6900")
        socket.setsockopt(LINGER, 0)
        poller = Poller()
        poller.register(socket, POLLIN)

        socket.send_multipart([
            endpoint,
            bytes(str(int((time() + timeout) * 1000)), encoding='utf8'),
            parameters
        ])
        responses = poller.poll(timeout * 1000)

        if len(responses) != 0:
            [response] = socket.recv_multipart()
            socket.close()
            return loads(response)
        else:
            socket.close()
        return None
Example #33
0
def directory_server(store: DirectoryServerStore, zmq_context: Context):
    # pylint: disable=no-member # These zmq.ROUTER and zmq.PUB must be actually exists
    print("Starting on libzmq {} with PyZMQ {}".format(zmq.zmq_version(),
                                                       zmq.pyzmq_version()))
    entrypoint: Socket = zmq_context.socket(zmq.ROUTER)
    entrypoint.bind("tcp://127.0.0.1:5350")  # This is just a PROTOTYPE!
    pub_file_changes: Socket = zmq_context.socket(zmq.PUB)
    pub_file_changes.bind("tcp://127.0.0.1:5351")
    poller = Poller()
    poller.register(entrypoint, flags=zmq.POLLIN)
    print(
        "Directory server is started on 127.0.0.1:5350 (commands) and 127.0.0.1:5351 (file_changes_push)"
    )
    while True:
        events: List[Tuple[Socket, int]] = poller.poll()
        for socket, _ in events:
            frames: List[Frame] = socket.recv_multipart(copy=False)
            id_frame: Frame = frames.pop(0)
            empty_frame: Frame = frames.pop(0)
            assert (len(empty_frame.bytes) == 0)
            command_frame: Frame = frames.pop(0)
            command = str(command_frame.bytes, encoding='utf8')
            if command == 'ping':
                ping_handler(store, socket, frames, id_frame)
            elif command == 'device.cast_address':
                casting_address_handler(store, socket, frames, id_frame)
            elif command == 'device.get_addresses':
                get_addresses_handler(store, socket, frames, id_frame)
            elif command == 'fs.list':
                file_list_handler(store, socket, id_frame)
            elif command == 'fs.declare':
                file_declare_handler(store, socket, frames, id_frame,
                                     pub_file_changes)
            elif command == 'fs.disown':
                file_disown_handler(store, socket, frames, id_frame,
                                    pub_file_changes)
            elif command == 'fs.get':
                file_get_handler(store, socket, frames, id_frame)
Example #34
0
    def run(self, *args):
        """Run the listener and answer to requests.
        """
        del args

        arec = AddressReceiver(
            max_age=self._max_age,
            multicast_enabled=self._multicast_enabled,
            restrict_to_localhost=self._restrict_to_localhost)
        arec.start()
        port = PORT

        try:
            with nslock:
                self.listener = get_context().socket(REP)
                self.listener.bind("tcp://*:" + str(port))
                logger.debug('Listening on port %s', str(port))
                poller = Poller()
                poller.register(self.listener, POLLIN)
            while self.loop:
                with nslock:
                    socks = dict(poller.poll(1000))
                    if socks:
                        if socks.get(self.listener) == POLLIN:
                            msg = self.listener.recv_string()
                    else:
                        continue
                    logger.debug("Replying to request: " + str(msg))
                    msg = Message.decode(msg)
                    active_address = get_active_address(
                        msg.data["service"], arec)
                    self.listener.send_unicode(six.text_type(active_address))
        except KeyboardInterrupt:
            # Needed to stop the nameserver.
            pass
        finally:
            arec.stop()
            self.stop()
Example #35
0
    def start_poll(self, start_seq_no=None):
        seq_no = start_seq_no if start_seq_no is not None else 0
        last_process = time.time()
        poller = Poller()
        poller.register(self.pull, zmq.POLLIN)
        poller.register(self.router, zmq.POLLIN)
        while True:
            try:
                socks = dict(poller.poll(timeout=self.gather_time * 100))
            except KeyboardInterrupt:
                return

            if self.pull in socks:
                self.recv_input()

            if self.router in socks:
                self.recv_from_comp()

            if self.worker_deque and self.ready_to_process() and time.time() - last_process >= self.gather_time:
                compute_id = self.worker_deque.popleft()
                last_process = time.time()
                try:
                    self.process()

                    self.send_to_comp(compute_id, seq_no)

                    self.reset_inputs()

                    if seq_no == 0xFFFF:
                        seq_no = 0
                    else:
                        seq_no += 1
                except Exception as e:
                    log.error("Could not process send and reset inputs -> {}".format(e))
            else:
                log.debug('time since last {}, deque size {}',
                          time.time() - last_process, len(self.worker_deque))
Example #36
0
class HomeBase(Thread):
    def __init__(self):
        super(HomeBase, self).__init__(name="HomeBase")
        self.context = Context()
        self.pull = self.context.socket(PULL)
        self.pull.bind("tcp://*:7001")
        self._shutdown = False
        self.poller = Poller()
        self.poller.register(self.pull, POLLIN)

    def cleanup(self):
        self.pull.close()
        self.context.term()

    def run(self):
        while True:
            socks = dict(self.poller.poll(timeout=1))
            if socks.get(self.pull) == POLLIN:
                msg = self.pull.recv()
                msg += ", WORK RECEIVED "
                print msg
            if self._shutdown:
                break
        self.cleanup()
Example #37
0
def _start_workers(context, sync_socket, count, timeout=None):
    """
    :type context: zmq.Context
    :type sync_socker: zmq.Socket
    :type count: int
    :param timeout: Timeout for waiting for worker messages, in milliseconds.
    :type timeout: float
    """
    _LOG.debug("Starting workers...")
    worker_threads = [
        Thread(
            target=_handle_requests, name="worker {}".format(i),
            args=(context, ))
        for i in range(count)]
    for thread in worker_threads:
        thread.start()
    _LOG.debug("Synchronizing workers...")
    poller = Poller()
    poller.register(sync_socket, POLLIN)
    worker_identities = []
    for _ in worker_threads:
        sockets = dict(poller.poll(timeout=timeout))
        if sync_socket in sockets:
            worker_identities.append(sync_socket.recv_multipart()[0])
        else:
            raise RuntimeError("Worker did not respond in time.")
    for worker_identity in worker_identities:
        sync_socket.send_multipart([worker_identity, b"", b""])
    for _ in worker_identities:
        sockets = dict(poller.poll(timeout=timeout))
        if sync_socket in sockets:
            sync_socket.recv_multipart()
        else:
            raise RuntimeError("Worker did not respond in time.")
    _LOG.debug("Workers synchronized.")
    return worker_threads, worker_identities
Example #38
0
def _handle_requests(context):
    """ This is supposed to run as a background thread.

    It listens for translation requests and answers them until a message is
    arrived on the sync socket, whereupon the function (thread) ends.

    :type context: zmq.Context
    """
    sync_socket = context.socket(REQ)
    sync_socket.connect(_SYNC_ENDPOINT)
    requests_socket = context.socket(REP)
    requests_socket.connect(_REQUEST_ENDPOINT)

    _LOG.debug("Synchronizing worker")
    sync_socket.send(b"")
    sync_socket.recv()
    sync_socket.send(b"")

    keep_running = True
    poller = Poller()
    poller.register(requests_socket, POLLIN)
    poller.register(sync_socket, POLLIN)
    _LOG.debug("Running worker")
    while keep_running:
        sockets = dict(poller.poll())
        if requests_socket in sockets:
            try:
                response = _handle_request(requests_socket.recv_multipart())
            except Exception:  # pylint: disable=broad-except
                _LOG.critical("Handler crashed!", exc_info=True)
                response = [b""]
            requests_socket.send_multipart(response)
        if sync_socket in sockets:
            sync_socket.recv()
            keep_running = False
    _LOG.debug("Terminating worker")
Example #39
0
class Listener:
    def __init__(self, timeout=-1):
        self.controllerSocket = None
        self.timecodeSocket = None
        self.requestSocket = None
        self.subscribers = {}
        self.subscriberCallbacks = {}
        self.subscriberTopicFilters = {}

        self.periodicPublisher = None

        self.commandCb = None
        self.requestCb = None

        self.timeout = timeout
        self.poller = Poller()
        self.running = False
        self.pollCounter = 0

    def connectController(self, address, commandCb=None):
        self.controllerSocket = zmq.Context().socket(zmq.PAIR)
        self.controllerSocket.bind(address)
        self.notifyConnected(address)
        self.poller.register(self.controllerSocket, zmq.POLLIN)
        self.commandCb = commandCb

    def sendController(self, message):
        wrappedMsg = IO.wrap(message)
        self.controllerSocket.send(wrappedMsg)

    def notifyConnected(self, address):
        msg = {'status': 'connected', 'address': address}
        self.sendController(msg)

    def notifyReady(self, address):
        msg = {'status': 'ready', 'address': address}
        self.sendController(msg)

    def listenToRequests(self, address, requestCb):
        self.requestCb = requestCb
        self.requestSocket = zmq.Context().socket(zmq.REP)
        self.requestSocket.bind(address)
        self.poller.register(self.requestSocket, zmq.POLLIN)

    def subscribeTimecode(self, address, timecodeCb):
        self.addSubscription(address,
                             timecodeCb,
                             label='_TC_',
                             filters=['TIMECODE'])

    def addSubscription(self,
                        address,
                        subCb,
                        label='default',
                        filters=['default']):
        if label not in self.subscribers:
            self.subscribers[label] = zmq.Context().socket(zmq.SUB)
            self.subscribers[label].set_hwm(1)
            #self.subscribers[label].setsockopt(zmq.RCVHWM, 1)
            self.subscriberCallbacks[label] = subCb
            self.subscriberTopicFilters[label] = filters
            self.poller.register(self.subscribers[label], zmq.POLLIN)

        print "Subscribing", label, "to:", address
        self.subscribers[label].connect(address)

        if filters:
            for filter in filters:
                self.subscribers[label].setsockopt(zmq.SUBSCRIBE, filter)
        else:
            self.subscribers[label].setsockopt(zmq.SUBSCRIBE, '')

    def setPeriodicPublisher(self, publisher, overrideTimeout=True):
        self.periodicPublisher = publisher
        if overrideTimeout: self.timeout = publisher.getTimeout()

    def start(self):
        self.running = True
        self._main()

    def stop(self):
        self.running = False

    def pollRegistered(self, expected=-1, timeout=-1):
        if expected == -1: expected = len(self.poller.sockets)
        self.pollCounter = expected
        while self.pollCounter > 0:
            self.poll(timeout)

    def poll(self, timeout=-1):
        if timeout == -1: timeout = self.timeout
        socks = dict(self.poller.poll(timeout))

        if self.controllerSocket in socks and socks[
                self.controllerSocket] == zmq.POLLIN:
            command = self.controllerSocket.recv()
            command, _ = IO.unwrap(command)
            self.pollCounter -= 1

            # Allow the user of this module to react to this command and to modify its contents
            if self.commandCb:
                modifiedCommand = self.commandCb(command)
                if modifiedCommand is not None: command = modifiedCommand

            if 'action' not in command: return
            action = command['action']
            if action == 'exit':
                self.running = False

        if self.requestSocket in socks and socks[
                self.requestSocket] == zmq.POLLIN:
            self.pollCounter -= 1
            message = self.requestSocket.recv()
            message, _ = IO.unwrap(message)
            replyMessage = self.requestCb(message)
            if replyMessage is None: return
            wrappedReply = IO.wrap(replyMessage)
            self.requestSocket.send(wrappedReply)

        if socks:
            for label in self.subscribers.keys():
                if self.subscribers[label] in socks and socks[
                        self.subscribers[label]] == zmq.POLLIN:
                    self.pollCounter -= 1
                    data = self.subscribers[label].recv()

                    # In case there is not space in the data, use find. Find returns -1 when the string isn't found
                    sepIndex = data.find(' ')
                    if sepIndex > -1 and not data[:4] == 'TIMS' and data[
                            0:sepIndex] in self.subscriberTopicFilters[label]:
                        topic, content = data[0:sepIndex], data[sepIndex + 1:]
                    else:
                        topic, content = '', data

                    content, _ = IO.unwrap(content)
                    if content is not None:
                        self.subscriberCallbacks[label](content)

        if self.periodicPublisher:
            self.periodicPublisher.publish()

    def _main(self):
        while self.running:
            self.poll()
Example #40
0
#
# Weather Server
#
if __name__ == '__main__':
    context = Context()
    socket = context.socket(PUB)
    socket.bind('tcp://*:6667')

    # socket to receive control messages on
    controller = context.socket(SUB)
    controller.connect('tcp://localhost:6670')
    controller.setsockopt(SUBSCRIBE, "")

    # poller for handling receiver and controller messages
    poller = Poller()
    poller.register(socket, POLLIN)
    poller.register(controller, POLLIN)

    print 'Weather Update Server started ...'
    while True:
        socks = dict(poller.poll(timeout=1))

        # exit the program if the kill command is executed
        if socks.get(controller) == POLLIN:
            controller.recv()
            print 'Got kill command'
            break

        # if not killed, then tranmit the next weather update
        zipcode = random.choice(zip_list)
        temperature = random.randrange(1, 215) - 80
Example #41
0
class Server():
    """A node server.

    Each node has a request URL where request are accepted and a service URL
    where the node periodically publishes its list of known nodes and the time
    of the last contact (last publication from that node on the service URL).

    The request commands are:
    -new: register a new node
    -get: get a key
    -set: set a key

    Each node owns a Nodes object which tells to which node a key belongs and a
    Cache object to store key/value pairs.

    When keys need to be moved (node list did change) then the old and new
    owners are compared for each key and the key gets sent or deleted
    accordingly.
    """

    PUB_INTERVALL = timedelta(seconds=5)

    IO_TIMEOUT = 5 * 1000  # milliseconds

    CMD_NEW = "new"

    CMD_GET = "get"

    CMD_SET = "set"

    def __init__(self, context, req_address, pub_address):
        self._context = context
        self._poller = Poller()
        self._nodes_sockets = {}  # requests and service sockets
        # FIFO request queue for nodes: [in_progress, data, callback(response)]
        self._nodes_requests = defaultdict(list)  # requests queues for nodes
        self._cache = Cache()  # the cache
        node_id = uuid1().hex  # unique node id, based on current time
        _LOG.info("node id: %s", node_id)
        self._nodes = Nodes(node_id, req_address, pub_address)  # other nodes

    def _open_req_socket(self, addr):
        req_socket = self._context.socket(REQ)
        req_socket.setsockopt(RCVTIMEO, self.IO_TIMEOUT)
        req_socket.connect(addr)
        self._poller.register(req_socket, POLLIN)
        return req_socket

    def _open_sub_socket(self, addr):
        sub_socket = self._context.socket(SUB)
        sub_socket.setsockopt_string(SUBSCRIBE, "")
        sub_socket.connect(addr)
        self._poller.register(sub_socket, POLLIN)
        return sub_socket

    def _add_request(self, node_id, data, callback):
        """Adds a new request for a particular node.

        Each request in the queue consists of:
        - in progress flag
        - data to be send
        - function (with one argument, the response) to be called when the
            response is ready
        """
        self._nodes_requests[node_id].append([False, data, callback])

    def _hanlde_subscriptions(self, poll_sockets):
        """Read the nodes list from each other node and merge the information
        into the local node list.

        :return: true if nodes where added
        """
        added = []
        for node_id, (_, pub_socket) in self._nodes_sockets.items():
            if pub_socket in poll_sockets:
                node_id, nodes = pub_socket.recv_json()
                # Convert string to datetime.
                nodes = {
                    i: (req_addr, pub_addr, _str_to_dt(last))
                    for i, (req_addr, pub_addr, last) in nodes.items()}
                # Merge nodes list.
                new_nodes = self._nodes.update_nodes(nodes)
                # Create sockets for new nodes.
                for node_id, req_addr, pub_addr in new_nodes:
                    _LOG.debug("adding node: %s", node_id)
                    req_socket = self._open_req_socket(req_addr)
                    sub_socket = self._open_sub_socket(pub_addr)
                    added.append((node_id, (req_socket, sub_socket)))
        self._nodes_sockets.update(dict(added))  # save sockets
        return len(added) > 0  # nodes list got changed?

    def _add_node(self, node_id, req_addr, pub_addr, req_socket=None):
        """Add one node to the list and create the sockets for the request
        and service URLs (subscribe to the node).
        """
        _LOG.debug("adding node: %s", node_id)
        # Add node to the nodes list.
        self._nodes.add_node(node_id, req_addr, pub_addr)
        if req_socket is None:  # does that not yet exist?
            req_socket = self._open_req_socket(req_addr)
        sub_socket = self._open_sub_socket(pub_addr)
        # Remember sockets.
        self._nodes_sockets[node_id] = (req_socket, sub_socket)

    def _remove_nodes(self, removed_nodes):
        """Unregister and remove a node, remove all pending request, too.
        """
        for node_id in removed_nodes:
            _LOG.debug("removing node: %s", node_id)
            # Unregister sockets from poller.
            req_socket, pub_socket = self._nodes_sockets.get(
                node_id, (None, None))
            if req_socket is not None:
                self._poller.unregister(req_socket)
            if pub_socket is not None:
                self._poller.unregister(pub_socket)
            # Forget the sockets and erase request queue.
            self._nodes_sockets.pop(node_id, None)
            self._nodes_requests.pop(node_id, None)
        return len(removed_nodes) > 0  # nodes list got changed?

    def _handle_responses(self, poll_sockets):
        """When a request from the queue gets an answer call the function for
        that request, if possible.
        """
        for node_id, (req_socket, _) in self._nodes_sockets.items():
            # Some node did replay?
            if req_socket in poll_sockets:
                # Get the answer and call the callback function with it, if
                # there was one.
                response = req_socket.recv_json()
                requests = self._nodes_requests[node_id]
                if requests:
                    in_progress, _, callback = requests[0]
                    if in_progress:
                        callback(response)
                        # Request done, remove it from the queue.
                        self._nodes_requests[node_id] = requests[1:]

    def _handle_request(self, node_changes, action, data):
        # Handle the requested action.
        _LOG.debug("Request %s: %s", action, str(data))
        if action == self.CMD_NEW:  # new node, register it
            node_id, req_addr, pub_addr = data
            self._add_node(node_id, req_addr, pub_addr)
            node_changes[0] = True  # a node was added!
            return (self._nodes.id, self._nodes.pub_address)
        if action == self.CMD_SET:  # set a key and value
            key, timestamp, value = data
            timestamp = _str_to_dt(timestamp)
            return self._cache.set(
                key, value, timestamp,
                self._nodes.get_nodes_for_index(key_index(key)))
        if action == self.CMD_GET:  # return value for a key
            in_cache, timestamp, value = self._cache.get(data)
            return (in_cache, _dt_to_str(timestamp), value)

    def _rebalance(self):
        """ Check if key should be on some other nodes now, move/delete as
        necessary.
        """
        send_entries = defaultdict(list)
        remove_entries = []
        # Loop through all keys and compare on which nodes they should be.
        for key, ts, nodes, value, index in self._cache.key_indices:
            new_nodes = self._nodes.get_nodes_for_index(index)
            if new_nodes != nodes:  # node list for that key changed?
                if self._nodes.id not in new_nodes:
                    remove_entries.append(key)  # no longer local
                for new_node in new_nodes - nodes:  # where should they be?
                    send_entries[new_node].append((key, ts, value))
                self._cache.set(key, value, ts, new_nodes)  # save new nodelist
        # Queue requests to set the keys on the new nodes.
        moves = 0
        for new_node, entries in send_entries.items():
            if new_node != self._nodes.id:
                for key, ts, value in entries:
                    self._add_request(
                        new_node, (self.CMD_SET, (key, _dt_to_str(ts), value)),
                        lambda response: None)
                    moves += 1
        # Remove dead nodes from cache.
        for key in remove_entries:
            self._cache.delete_key(key)
        _LOG.debug(
            "adjusted distribution, %d moves, %d deletes", moves,
            len(remove_entries))

    def _handle_api_get(self, api_pub_socket, req_id, key):
        """Handle a API get request from the local API.

        If the key is not local, send a request to one of the responsible
        nodes. Upon arrival of the response publish the response for the API,
        using the request id, which only the right listens for.
        """
        req_id = req_id.encode("utf-8")

        # Publish the result.
        def send_response(resp, req_id=req_id, api_pub_socket=api_pub_socket):
            found, _, value = resp
            found = b"1" if found else b"0"
            value = value.encode("utf-8") if value else b""
            api_pub_socket.send_multipart([req_id, found, value])

        # Where is the key stored?
        nodes = self._nodes.get_nodes_for_index(key_index(key))
        if nodes:
            if self._nodes.id in nodes:  # local answer directly
                in_cache, _, value = self._cache.get(key)
                send_response((in_cache, None, value))
            else:  # remote, make request
                node_id = choice(list(nodes))
                self._add_request(node_id, (self.CMD_GET, key), send_response)
        else:  # can not be stored in cache, for whatever strange reason
            send_response((False, None, None))

    def _handle_api_set(self, api_pub_socket, req_id, key, value):
        """Handle a API set request from the local API.

        Check where the key should be stored and make the appropriate requests.
        The publish all results to the API, alltough the API only cares about
        the fastest response.
        """
        req_id = req_id.encode("utf-8")

        def send_response(resp, req_id=req_id, api_pub_socket=api_pub_socket):
            resp = str(resp).encode("utf-8")  # "0", "-1", ...
            api_pub_socket.send_multipart([req_id, resp])

        # Where should the key go?
        nodes = self._nodes.get_nodes_for_index(key_index(key))
        if nodes:
            timestamp = datetime.now()
            if self._nodes.id in nodes:  # save locally and publish response
                resp = self._cache.set(key, value, timestamp, nodes)
                send_response(resp)
                nodes.remove(self._nodes.id)  # done with local node!
            for node_id in nodes:  # make requests to all remote nodes
                self._add_request(
                    node_id,
                    (self.CMD_SET, (key, _dt_to_str(timestamp), value)),
                    send_response)
        else:
            send_response(-2)  # no nodes

    def loop(self, api_port, req_addr):
        """Event loop, listen to all sockets and handle all messages.

        If a req_address of an existing node is given the this node will
        register itself there, before entering the loop.

        The request on the api_port are handled by a Python WSGI instance
        running the Flask API app.
        """
        # Register to existing node?
        if req_addr is not None:
            _LOG.debug("Contacting %s", req_addr)
            req_socket = self._open_req_socket(req_addr)
            req_socket.send_json(
                (self.CMD_NEW,
                    (self._nodes.id, self._nodes.req_address,
                        self._nodes.pub_address)))
            node_id, pub_addr = req_socket.recv_json()
            _LOG.debug("Received: %s %s", str(node_id), pub_addr)
            self._add_node(node_id, req_addr, pub_addr, req_socket)
            self._rebalance()

        # Create request and service sockets.
        nodes_publisher = self._context.socket(PUB)
        nodes_publisher.setsockopt(RCVTIMEO, self.IO_TIMEOUT)
        nodes_publisher.setsockopt(SNDTIMEO, self.IO_TIMEOUT)
        _LOG.debug("Publishing on %s", self._nodes.pub_address)
        nodes_publisher.bind(self._nodes.pub_address)
        req_socket = self._context.socket(REP)
        req_socket.setsockopt(RCVTIMEO, self.IO_TIMEOUT)
        req_socket.setsockopt(SNDTIMEO, self.IO_TIMEOUT)
        _LOG.debug("waiting for requests on %s", self._nodes.req_address)
        req_socket.bind(self._nodes.req_address)
        self._poller.register(req_socket, POLLIN)

        # Create in-process sockets to the API app.
        api_pull_socket = self._context.socket(PULL)
        api_pull_socket.bind(PUSH_ENDPOINT)
        self._poller.register(api_pull_socket, POLLIN)
        api_pub_socket = self._context.socket(PUB)
        api_pub_socket.bind(SUB_ENDPOINT)

        _LOG.info("Entering server loop")
        # Start web server.
        set_config(self._context)
        httpd = simple_server.make_server(
            '0.0.0.0', int(api_port), app)
        Thread(target=httpd.serve_forever).start()
        # Enter ZMQ loop.
        stop = False
        last_published = None
        try:
            while not stop:
                try:
                    # Wait for messages, but not too long!
                    sockets = dict(
                        self._poller.poll(self.PUB_INTERVALL.seconds * 1000))
                except KeyboardInterrupt:
                    stop = True
                else:
                    changes = [False]
                    # Handle incoming node updates (subscriptions).
                    if self._hanlde_subscriptions(sockets):
                        changes[0] = True
                    # Handle incoming responses.
                    self._handle_responses(sockets)
                    # Incoming requests?
                    if req_socket in sockets:
                        req_socket.send_json(
                            self._handle_request(
                                changes, *req_socket.recv_json()))
                    # Remove dead nodes?
                    if self._remove_nodes(self._nodes.remove_dead_nodes()):
                        changes[0] = True
                    # Did nodes change?
                    if changes[0]:
                        self._rebalance()
                    # Request something?
                    for node_id, requests in self._nodes_requests.items():
                        if requests and not requests[0][0]:
                            request = requests[0]
                            request[0] = True  # in progress
                            self._nodes_sockets[node_id][0].send_json(
                                request[1])
                    # Publish something?
                    now = datetime.now()
                    if (last_published is None or
                            now - last_published > self.PUB_INTERVALL):
                        # Get nodes, convert last datetime to string
                        nodes = {
                            i: (req_addr, pub_addr, _dt_to_str(last))
                            for i, (req_addr, pub_addr, last)
                            in self._nodes.nodes.items()}
                        _LOG.debug(
                            "publishing:\n%s", "\n".join(
                                "{}: {}".format(i, str(n))
                                for i, n in nodes.items()))
                        nodes_publisher.send_json((self._nodes.id, nodes))
                        last_published = now
                    # Handle API get requests
                    if api_pull_socket in sockets:
                        req_id, action, key, value = (
                            api_pull_socket.recv_json())
                        if action == "get":
                            self._handle_api_get(api_pub_socket, req_id, key)
                        if action == "set":
                            self._handle_api_set(
                                api_pub_socket, req_id, key, value)
        finally:
            httpd.shutdown()
Example #42
0
class Distributor(Thread):
    def __init__(self, query):
        Thread.__init__(self)
        self._pool = ThreadPool(processes=4)
        self._workers = HashRing(WORKER_SERVERS)
        self._identity = bytes(uuid.uuid4())
        self._query = query
        self._init_sock()

    def _log(self, text):
        if LOG_DISTRIBUTOR:
            log_debug(self, text)

    def _init_sock(self):
        self._context = Context(1)
        self._poller = Poller()
        self._set_sock()

    def _set_sock(self):
        self._socket = self._context.socket(DEALER)
        self._socket.setsockopt(IDENTITY, self._identity)
        self._poller.register(self._socket, POLLIN)
        self._socket.connect(zmqaddr(self._get_broker(), BROKER_PORT))
        self._socket.send(PPP_READY)

    def _reset_sock(self):
        self._poller.unregister(self._socket)
        self._socket.setsockopt(LINGER, 0)
        self._socket.close()
        self._set_sock()

    def _get_broker(self):
        length = len(BROKER_SERVERS)
        return BROKER_SERVERS[randint(0, length - 1)]

    def _get_worker(self, uid):
        return self._workers.get_node(uid)

    def _get_user(self, buf):
        if len(buf) < USERNAME_SIZE:
            log_err(self, 'failed to get user, invalid length')
            raise Exception(log_get(self, 'failed to get user'))
        name = filter(lambda x:x != '*', buf[:USERNAME_SIZE])
        if not name:
            log_err(self, 'failed to get user')
            raise Exception(log_get(self, 'failed to get user'))
        return name

    def _get_token(self, buf):
        if len(buf) < UID_SIZE:
            log_err(self, 'failed to get token, invalid length')
            raise Exception(log_get(self, 'failed to get token'))
        uid = None
        token = None
        if buf[UID_SIZE - 1] == '*':
            user = self._get_user(buf)
            uid, token = self._query.user.get(user, 'uid', 'password')
        else:
            uid = buf[0:UID_SIZE]
            token = self._query.token.get(uid)
        if uid and token:
            return (uid, token)
        else:
            log_err(self, 'failed to get token')
            raise Exception(log_get(self, 'failed to get token'))

    def _reply(self, identity, seq, buf):
        msg = [identity, '', seq, buf]
        self._socket.send_multipart(msg)

    def _request(self, addr, **args):
        sock = io.connect(addr, WORKER_PORT)
        try:
            buf = bson.dumps(args)
            io.send_pkt(sock, buf)
            res = io.recv_pkt(sock)
            return unicode2str(bson.loads(res)['result'])
        finally:
            io.close(sock)

    def _proc(self, identity, seq, buf):
        uid, token = self._get_token(buf)
        if not uid or not token:
            log_err(self, 'failed to process, cannot get token')
            return
        addr = self._get_worker(uid)
        ret = self._request(addr, uid=uid, token=token, buf=buf)
        self._reply(identity, seq, ret)

    def _proc_safe(self, identity, seq, buf):
        try:
            self._proc(identity, seq, buf)
        except:
            log_err(self, 'failed to process')

    def _handler(self, identity, seq, buf):
        if DEBUG and not SAFE:
            self._proc(identity, seq, buf)
        else:
            self._proc_safe(identity, seq, buf)

    def run(self):
        self._log('start ...')
        liveness = PPP_HEARTBEAT_LIVENESS
        timeout = time.time() + PPP_HEARTBEAT_INTERVAL
        while True:
            sockets = dict(self._poller.poll(PPP_HEARTBEAT_INTERVAL * 1000))
            if sockets.get(self._socket) == POLLIN:
                frames = self._socket.recv_multipart()
                if not frames:
                    log_err(self, 'invalid request')
                    break
                if len(frames) == PPP_NR_FRAMES:
                    self._pool.apply_async(self._proc, args=(frames[PPP_FRAME_ID], frames[PPP_FRAME_SEQ], frames[PPP_FRAME_BUF]))
                elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
                    liveness = PPP_HEARTBEAT_LIVENESS
                else:
                    log_err(self, "invalid request")
            else:
                liveness -= 1
                if liveness == 0:
                    time.sleep(random.randint(SLEEP_TIME / 2, SLEEP_TIME))
                    self._reset_sock()
                    liveness = PPP_HEARTBEAT_LIVENESS

            if time.time() > timeout:
                timeout = time.time() + PPP_HEARTBEAT_INTERVAL
                self._socket.send(PPP_HEARTBEAT)
Example #43
0
class StagerZmq(Component, Process, Connexions):

    """`StagerZmq` class represents a Stager pipeline Step.
    It is derived from Process class.
    It receives new input from its prev stage, thanks to its ZMQ REQ socket,
    and executes its coroutine objet's run method by passing
    input as parameter. Finaly it sends coroutine returned value to its next
    stage, thanks to its ZMQ REQ socket,
    The processus is launched by calling run method.
    init() method is call by run method.
    The processus is stopped by setting share data stop to True
    """
    def __init__(
            self, coroutine, sock_job_for_me_port,
            name=None, connexions=dict(),main_connexion_name=None):
        """
        Parameters
        ----------
        coroutine : Class instance that contains init, run and finish methods
        sock_job_for_me_port: str
            Port number for input socket url
        name: str
            Stage name
        main_connexion_name : str
            Default next step name. Used to send data when destination is not provided
        connexions: dict {'STEP_NANE' : (zmq STEP_NANE port in)}
            Port number for socket for each next steps
        """
        Process.__init__(self)
        Component.__init__(self,parent=None)
        self.name = name
        Connexions.__init__(self,main_connexion_name,connexions)
        self.coroutine = coroutine
        self.sock_job_for_me_url = 'tcp://localhost:' + sock_job_for_me_port
        self.done = False
        self.waiting_since = Value('i',0)
        self._nb_job_done = Value('i',0)
        self._stop = Value('i',0)
        self._running = Value('i',0)

    def init(self):
        """
        Initialise coroutine sockets and poller
        Returns
        -------
        True if coroutine init and init_connexions methods returns True,
         otherwise False
        """
        if self.name is None:
            self.name = "STAGER"
        if self.coroutine is None:
            return False
        if self.coroutine.init() == False:
            return False
        return self.init_connexions()

    def run(self):
        """
        Method representing the processus's activity.
        It polls its socket and when received new input from it,
        it executes coroutine run method by passing new input.
        Then it sends coroutine return value to its next stage,
        thanks to its ZMQ REQ socket.
        The poll method's timeout is 100 ms in case of self.stop flag
        has been set to False.
        Atfer the main while loop, coroutine.finish method is called
        """
        if self.init()  :
            while not self.stop:
                sockets = dict(self.poll.poll(100))  # Poll or time out (100ms)
                if (self.sock_for_me in sockets and
                        sockets[self.sock_for_me] == POLLIN):
                    #  Get the input from prev_stage
                    self.waiting_since.value = 0
                    self.running = 1
                    request = self.sock_for_me.recv_multipart()
                    receiv_input = loads(request[0])
                    # do the job
                    results = self.coroutine.run(receiv_input)
                    if isinstance(results, GeneratorType):
                        for val in results:
                            msg,destination = self.get_destination_msg_from_result(val)
                            self.send_msg(msg,destination)
                    else:
                        msg,destination = self.get_destination_msg_from_result(results)
                        self.send_msg(msg,destination)
                    # send acknoledgement to prev router/queue to inform it that I
                    # am available
                    self.sock_for_me.send_multipart(request)
                    self._nb_job_done.value = self._nb_job_done.value + 1
                    self.running = 0
                else:
                    self.waiting_since.value = self.waiting_since.value+100 # 100 ms
            self.sock_for_me.close()
        self.finish()
        self.done = True

    def finish(self):
        self.coroutine.finish()

    def init_connexions(self):
        """
        Initialise zmq sockets.
        Because this class is s Process, This method must be call in the run
         method to be hold by the correct processus.
        """
        Connexions.init_connexions(self)
        context = Context()
        self.sock_for_me = context.socket(REQ)
        self.sock_for_me.connect(self.sock_job_for_me_url)
        # Use a ZMQ Pool to get multichannel message
        self.poll = Poller()
        # Register sockets
        self.poll.register(self.sock_for_me, POLLIN)
        # Send READY to next_router to inform about my capacity to compute new
        # job
        self.sock_for_me.send_pyobj("READY")
        return True

    @property
    def wait_since(self):
        return self.waiting_since.value

    @property
    def stop(self):
        return self._stop.value

    @stop.setter
    def stop(self, value):
        self._stop.value = value

    @property
    def nb_job_done(self):
        return self._nb_job_done.value

    @nb_job_done.setter
    def nb_job_done(self, value):
        self._nb_job_done.value = value


    @property
    def running(self):
        return self._running.value

    @running.setter
    def running(self, value):
        self._running.value = value
Example #44
0
# socket to receive messages on
receiver = context.socket(PULL)
receiver.connect("tcp://localhost:7779")

# socket to send messages to
sender = context.socket(PUSH)
sender.connect("tcp://localhost:7780")

# socket to receive control messages on
controller = context.socket(SUB)
controller.connect("tcp://localhost:6670")
controller.setsockopt(SUBSCRIBE, "")

# poler for handling receiver and controller messages
poller = Poller()
poller.register(receiver, POLLIN)
poller.register(controller, POLLIN)

print "Waiting for job ..."

# poller handling
while True:
    socks = dict(poller.poll())

    if socks.get(receiver) == POLLIN:
        msg = receiver.recv()

        # process task
        workload_msec = int(msg)
        time.sleep(workload_msec * 0.001)
Example #45
0
    def run(self):
        poller = Poller()
        poller.register(self._socket, POLLIN)
        
        while self._loop:
            socks = dict(poller.poll(timeout=2000))
            if self._socket in socks and socks[self._socket] == POLLIN:
                message = Message(rawstr=self._socket.recv(NOBLOCK))
                logger.debug("Request: " + str(message))
                # send list of scanlines
                if(message.type == "request" and
                   message.data["type"] == "scanlines"):
                    sat = message.data["satellite"]
                    epoch = "1950-01-01T00:00:00"
                    start_time = strp_isoformat(message.data.get("start_time",
                                                                 epoch))
                    end_time = strp_isoformat(message.data.get("end_time",
                                                               epoch))

                    resp = Message('/oper/polar/direct_readout/' + self._station,
                                   "scanlines",
                                   [(utctime.isoformat(),
                                     self._holder[sat][utctime][2])
                                    for utctime in self._holder.get(sat, [])
                                    if utctime >= start_time
                                    and utctime <= end_time])
                    self._socket.send(str(resp))

                # send one scanline
                elif(message.type == "request" and
                     message.data["type"] == "scanline"):
                    sat = message.data["satellite"]
                    utctime = strp_isoformat(message.data["utctime"])
                    try:
                        url = urlparse(self._holder[sat][utctime][1])
                    except KeyError:
                        resp = Message('/oper/polar/direct_readout/'
                                       + self._station,
                                       "missing")
                    else:
                        if url.scheme in ["", "file"]: # data is locally stored.
                            resp = Message('/oper/polar/direct_readout/'
                                           + self._station,
                                           "scanline",
                                           self._holder.get_scanline(sat, utctime),
                                           binary=True)
                        else: # it's the address of a remote server.
                            resp = self.forward_request(urlunparse(url),
                                                        message)
                    self._socket.send(str(resp))

                # take in a new scanline
                elif(message.type == "notice" and
                     message.data["type"] == "scanline"):
                    sat = message.data["satellite"]
                    utctime = message.data["utctime"]
                    elevation = message.data["elevation"]
                    filename = message.data["filename"]
                    line_start = message.data["file_position"]
                    utctime = strp_isoformat(utctime)
                    self._holder.add_scanline(sat, utctime, elevation,
                                              line_start, filename)
                    resp = Message('/oper/polar/direct_readout/'
                                       + self._station,
                                       "notice",
                                       "ack")
                    self._socket.send(str(resp))
                elif(message.type == "ping"):
                    resp = Message('/oper/polar/direct_readout/'
                                   + self._station,
                                   "pong",
                                   {"station": self._station})
                    self._socket.send(str(resp))

                if resp.binary:
                    logger.debug("Response: " + " ".join(str(resp).split()[:6]))
                else:
                    logger.debug("Response: " + str(resp))
Example #46
0
class Subscriber(object):

    """Subscriber

    Subscribes to *addresses* for *topics*, and perform address translation of
    *translate* is true. The function *message_filter* can be used to
    discriminate some messages on the subscriber side. *topics* on the other
    hand performs filtering on the publishing side (from zeromq 3).

    Example::

        from posttroll.subscriber import Subscriber, get_pub_address

        addr = get_pub_address(service, timeout=2)
        sub = Subscriber([addr], 'my_topic')
        try:
            for msg in sub(timeout=2):
                print("Consumer got", msg)

        except KeyboardInterrupt:
            print("terminating consumer...")
            sub.close()

    """

    def __init__(self, addresses, topics='', message_filter=None,
                 translate=False):
        self._topics = self._magickfy_topics(topics)
        self._filter = message_filter
        self._translate = translate

        self.sub_addr = {}
        self.addr_sub = {}
        self.poller = None

        self._hooks = []
        self._hooks_cb = {}

        self.poller = Poller()
        self._lock = Lock()

        self.update(addresses)

        self._loop = True

    def add(self, address, topics=None):
        """Add *address* to the subscribing list for *topics*.

        It topics is None we will subscibe to already specified topics.
        """
        with self._lock:
            if address in self.addresses:
                return False

            topics = self._magickfy_topics(topics) or self._topics
            LOGGER.info("Subscriber adding address %s with topics %s",
                        str(address), str(topics))
            subscriber = get_context().socket(SUB)
            for t__ in topics:
                subscriber.setsockopt_string(SUBSCRIBE, six.text_type(t__))
            subscriber.connect(address)
            self.sub_addr[subscriber] = address
            self.addr_sub[address] = subscriber
            if self.poller:
                self.poller.register(subscriber, POLLIN)
            return True

    def remove(self, address):
        """Remove *address* from the subscribing list for *topics*.
        """
        with self._lock:
            try:
                subscriber = self.addr_sub[address]
            except KeyError:
                return False
            LOGGER.info("Subscriber removing address %s", str(address))
            if self.poller:
                self.poller.unregister(subscriber)
            del self.addr_sub[address]
            del self.sub_addr[subscriber]
            subscriber.close()
            return True

    def update(self, addresses):
        """Updating with a set of addresses.
        """
        if isinstance(addresses, six.string_types):
            addresses = [addresses, ]
        s0_, s1_ = set(self.addresses), set(addresses)
        sr_, sa_ = s0_.difference(s1_), s1_.difference(s0_)
        for a__ in sr_:
            self.remove(a__)
        for a__ in sa_:
            self.add(a__)
        return bool(sr_ or sa_)

    def add_hook_sub(self, address, topics, callback):
        """Specify a *callback* in the same stream (thread) as the main receive
        loop. The callback will be called with the received messages from the
        specified subscription.

        Good for operations, which is required to be done in the same thread as
        the main recieve loop (e.q operations on the underlying sockets).
        """
        LOGGER.info("Subscriber adding SUB hook %s for topics %s",
                    str(address), str(topics))
        socket = get_context().socket(SUB)
        for t__ in self._magickfy_topics(topics):
            socket.setsockopt_string(SUBSCRIBE, six.text_type(t__))
        socket.connect(address)
        self._add_hook(socket, callback)

    def add_hook_pull(self, address, callback):
        """Same as above, but with a PULL socket.
        (e.g good for pushed 'inproc' messages from another thread).
        """
        LOGGER.info("Subscriber adding PULL hook %s", str(address))
        socket = get_context().socket(PULL)
        socket.connect(address)
        self._add_hook(socket, callback)

    def _add_hook(self, socket, callback):
        """Generic hook. The passed socket has to be "receive only".
        """
        self._hooks.append(socket)
        self._hooks_cb[socket] = callback
        if self.poller:
            self.poller.register(socket, POLLIN)

    @property
    def addresses(self):
        """Get the addresses
        """
        return self.sub_addr.values()

    @property
    def subscribers(self):
        """Get the subscribers
        """
        return self.sub_addr.keys()

    def recv(self, timeout=None):
        """Receive, optionally with *timeout* in seconds.
        """
        if timeout:
            timeout *= 1000.

        for sub in list(self.subscribers) + self._hooks:
            self.poller.register(sub, POLLIN)
        self._loop = True
        try:
            while self._loop:
                sleep(0)
                try:
                    socks = dict(self.poller.poll(timeout=timeout))
                    if socks:
                        for sub in self.subscribers:
                            if sub in socks and socks[sub] == POLLIN:
                                m__ = Message.decode(sub.recv_string(NOBLOCK))
                                if not self._filter or self._filter(m__):
                                    if self._translate:
                                        url = urlsplit(self.sub_addr[sub])
                                        host = url[1].split(":")[0]
                                        m__.sender = (m__.sender.split("@")[0]
                                                      + "@" + host)
                                    yield m__

                        for sub in self._hooks:
                            if sub in socks and socks[sub] == POLLIN:
                                m__ = Message.decode(sub.recv_string(NOBLOCK))
                                self._hooks_cb[sub](m__)
                    else:
                        # timeout
                        yield None
                except ZMQError as err:
                    LOGGER.exception("Receive failed: %s", str(err))
        finally:
            for sub in list(self.subscribers) + self._hooks:
                self.poller.unregister(sub)

    def __call__(self, **kwargs):
        return self.recv(**kwargs)

    def stop(self):
        """Stop the subscriber.
        """
        self._loop = False

    def close(self):
        """Close the subscriber: stop it and close the local subscribers.
        """
        self.stop()
        for sub in list(self.subscribers) + self._hooks:
            try:
                sub.setsockopt(LINGER, 1)
                sub.close()
            except ZMQError:
                pass

    @staticmethod
    def _magickfy_topics(topics):
        """Add the magick to the topics if missing.
        """
        # If topic does not start with messages._MAGICK (pytroll:/), it will be
        # prepended.
        if topics is None:
            return None
        if isinstance(topics, six.string_types):
            topics = [topics, ]
        ts_ = []
        for t__ in topics:
            if not t__.startswith(_MAGICK):
                if t__ and t__[0] == '/':
                    t__ = _MAGICK + t__
                else:
                    t__ = _MAGICK + '/' + t__
            ts_.append(t__)
        return ts_

    def __del__(self):
        for sub in list(self.subscribers) + self._hooks:
            try:
                sub.close()
            except:
                pass
class PushRequester(object):

    """Base requester class.
    """

    request_retries = 3

    def __init__(self, host, port):
        self._socket = None
        self._reqaddress = "tcp://" + host + ":" + str(port)
        self._poller = Poller()
        self._lock = Lock()
        self.failures = 0
        self.jammed = False

        self.connect()

    def connect(self):
        """Connect to the server
        """
        self._socket = context.socket(REQ)
        self._socket.connect(self._reqaddress)
        self._poller.register(self._socket, POLLIN)

    def stop(self):
        """Close the connection to the server
        """
        self._socket.setsockopt(LINGER, 0)
        self._socket.close()
        self._poller.unregister(self._socket)

    def reset_connection(self):
        """Reset the socket
        """
        self.stop()
        self.connect()

    def __del__(self, *args, **kwargs):
        self.stop()

    def send_and_recv(self, msg, timeout=REQ_TIMEOUT):

        with self._lock:
            retries_left = self.request_retries
            request = str(msg)
            self._socket.send(request)
            rep = None
            while retries_left:
                socks = dict(self._poller.poll(timeout))
                if socks.get(self._socket) == POLLIN:
                    reply = self._socket.recv()
                    if not reply:
                        LOGGER.error("Empty reply!")
                        break
                    rep = Message(rawstr=reply)
                    self.failures = 0
                    self.jammed = False
                    break
                else:
                    LOGGER.warning("Timeout from " + str(self._reqaddress)
                                   + ", retrying...")
                    # Socket is confused. Close and remove it.
                    self.stop()
                    retries_left -= 1
                    if retries_left <= 0:
                        LOGGER.error("Server doesn't answer, abandoning... " +
                                     str(self._reqaddress))
                        self.connect()
                        self.failures += 1
                        if self.failures == 5:
                            LOGGER.critical("Server jammed ? %s",
                                            self._reqaddress)
                            self.jammed = True
                        break
                    LOGGER.info("Reconnecting and resending " + str(msg))
                    # Create new connection
                    self.connect()
                    self._socket.send(request)

        return rep
Example #48
0
class Requester(object):

    """Make a request connection, waiting to get scanlines .
    """
    
    def __init__(self, host, port):
        self._host = host
        self._port = port
        self._context = Context()
        self._socket = self._context.socket(REQ)
        self._socket.setsockopt(LINGER, 1)
        self._socket.connect("tcp://"+host+":"+str(port))
        self._poller = Poller()
        self._poller.register(self._socket, POLLIN)

    def stop(self):
        """Close the socket.
        """
        self._socket.close()

    def __del__(self, *args, **kwargs):
        self.stop()

    def send(self, msg):
        """Send a message.
        """
        return self._socket.send(str(msg))

    def recv(self, timeout=None):
        """Receive a message. *timeout* in ms.
        """
        if self._poller.poll(timeout):
            return Message(rawstr=self._socket.recv())
        else:
            raise IOError("Timeout from " + str(self._host) +
                          ":" + str(self._port))
        
    def get_line(self, satellite, utctime):
        """Get the scanline of *satellite* at *utctime*.
        """
        msg = Message('/oper/polar/direct_readout/norrköping',
                      'request',
                      {"type": "scanline",
                       "satellite": satellite,
                       "utctime": utctime.isoformat()})
        self.send(msg)
        return self.recv(1000).data


    def get_slice(self, satellite, start_time, end_time):
        """Get a slice of scanlines.
        """
        msg = Message('/oper/polar/direct_readout/norrköping',
                      'request',
                      {"type": 'scanlines',
                       "satellite": satellite,
                       "start_time": start_time.isoformat(),
                       "end_time": end_time.isoformat()})
        self.send(msg)
        return self.recv(1000).data

    def send_lineinfo(self, sat, utctime, elevation, filename, pos):
        """Send information to our own server.
        """

        msg = Message('/oper/polar/direct_readout/norrköping',
                      'notice',
                      {"type": 'scanline',
                       "satellite": sat,
                       "utctime": utctime.isoformat(),
                       "elevation": elevation,
                       "filename": filename,
                       "file_position": pos})
        self.send(msg)
        self._socket.recv()         
Example #49
0
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
    '''Connect to and marshall messages between broker and writer sockets.

    Parameters
    ----------

    bot : zio.Message

        The BOT message

    rule_object: dicionary 

        A ruleset rule object.

    writer_addr :: string

        The address of the writer's PULL socket to connect.

    broker_addr : string

        The address of the broker's SERVER socket to connect.

    '''
    # An HDF path to be added to every message we send to writer.
    mattr = message_to_dict(bot)
    rattr = dict(rule_object.get("attr",{}), **mattr)
    log.info(f'writer: attrs: {rattr}')
    try:
        base_path =  rule_object.get("grouppat","/").format(**rattr)
    except KeyError as e:
        log.error(f'writer: missing attribute: {e}')
        raise
    log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
    log.debug(bot)
    pipe.signal()

    push = ctx.socket(PUSH)
    push.connect(writer_addr)

    sock = ctx.socket(CLIENT)
    port = Port("write-handler", sock)
    port.connect(broker_addr)
    port.online(None)

    direction = mattr["direction"]
    if direction != "inject":
        raise RuntimeError(f'zio.flow.hdf.writer bad direction: "{direction}"')
    credit = mattr["credit"]
    flow = Flow(port, direction, credit)
    log.debug (f'writer({base_path}) send BOT to {broker_addr}')

    bot = flow.bot(bot)         # this introduces us to the server
    log.debug (f'writer({base_path}) got response:\n{bot}')
    flow.begin()

    def push_message(m):
        log.debug (f'write_handler({base_path}) push {m}')
        attr = message_to_dict(m)
        attr['hdfgroup'] = base_path
        m.label = json.dumps(attr)
        push.send(m.encode())

    #push_message(bot)

    poller = Poller()
    poller.register(pipe, POLLIN)
    poller.register(sock, POLLIN)
    while True:

        for which,_ in poller.poll():
            if not which:
                return

            if which == pipe: # signal exit
                log.debug ('write_handler pipe hit')
                return          

            # o.w. we have flow

            try:
                msg = flow.get()
            except TransmissionEnd as te:
                flow.eotsend()
                break
            push_message(msg)

            continue

    log.debug ('write_handler exiting')
    pipe.signal()
Example #50
0
    server_sock = context.socket(SUB)
    server_sock.connect('tcp://localhost:6667')
    server_sock.setsockopt(SUBSCRIBE, "")

    # this is public endpoint for subscribers
    client_sock = context.socket(PUB)
    client_sock.bind("tcp://*:6668")

    # socket to receive control messages on
    controller = context.socket(SUB)
    controller.connect('tcp://localhost:6670')
    controller.setsockopt(SUBSCRIBE, "")

    # poller for handling receiver and controller messages
    poller = Poller()
    poller.register(server_sock, POLLIN)
    poller.register(controller, POLLIN)

    print 'Weather proxy started ...'

    # shunt messages out to our own subscriber or handle incomming commandds
    while True:
        socks = dict(poller.poll())

        if socks.get(server_sock) == POLLIN:
            # process weather updates to clients
            msg = server_sock.recv()
            more = server_sock.getsockopt(RCVMORE)
            if more:
                client_sock.send(msg, SNDMORE)
            else:
Example #51
0
class ServiceMixin(Thread):
    def __init__(self, pipe, local_ep, local_uuid, config_svc):
        Thread.__init__(self, name='dcamp.service.{}'.format(self))
        self.ctx = Context.instance()
        self.__control_pipe = pipe

        assert isinstance(local_ep, EndpntSpec)
        self.__endpoint = local_ep

        assert isinstance(local_uuid, UUID)
        self.__uuid = local_uuid

        self.__cfgsvc = config_svc

        self.logger = getLogger('dcamp.service.%s' % self)

        self.poller = Poller()
        self.poller_timer = None

        self.poller.register(self.__control_pipe, POLLIN)

    def __str__(self):
        return self.__class__.__name__

    @property
    def cfgsvc(self):
        return self.__cfgsvc

    @property
    def endpoint(self):
        return self.__endpoint

    @property
    def uuid(self):
        return self.__uuid

    def __send_control(self, message):
        self.__control_pipe.send_string(message)

    def __recv_control(self):
        return self.__control_pipe.recv_string()

    def _cleanup(self):
        # tell role we're done (if we can)
        if not self.in_errored_state:
            # @todo: this might raise an exception / issue #38
            self.__send_control('STOPPED')
            self.logger.debug('sent STOPPED control reply')

        # shared context; will be term()'ed by caller
        self.__control_pipe.close()
        del self.__control_pipe

        self.logger.debug('service cleanup finished; exiting')

    def _pre_poll(self):
        pass

    def _post_poll(self, items):
        raise NotImplementedError('subclass must implement _post_poll()')

    def _do_control(self):
        """
        Process control command on the pipe.
        """
        msg = self.__recv_control()

        if 'STOP' == msg:
            self.logger.debug('received STOP control command')
            self.stop_state()
        else:
            self.__send_control('WTF')
            self.logger.error('unknown control command: %s' % msg)

    def run(self):
        self.run_state()

        # wait for configuration service to init
        if self.__cfgsvc is not None:
            self.__cfgsvc.wait_for_gogo()

        while self.in_running_state:
            try:
                self._pre_poll()
                items = dict(self.poller.poll(self.poller_timer))
                self._post_poll(items)
                if self.__control_pipe in items:
                    self._do_control()

            except ZMQError as e:
                if e.errno == ETERM:
                    self.logger.debug('received ETERM: %s' % self.__class__)
                    self.error_state()
                else:
                    raise

        # thread is stopping; cleanup and exit
        return self._cleanup()
Example #52
0
class Cornerstone(Scaffold):
    """ Cornerstone can be used to create a 0mq poll loop.

    Upon creation of a Cornerstone instance, the initial state of the instance
    internal xmq poll loop is passive. To start the loop call Cornerstone
    run(). To stop the Cornerstone instance call Cornerstone.kill().

    Cornerstone only allows for one zmq input port and one zmq output port.
    Cornerstone support respectively; Cornerstone.register_input_sock() and
    Cornerstone.register_output_sock() methods.

    Cornerstone implements an internal signal handler for detection of
    interrupt signals to handle shutdown of connection resources.

    Example Usage:
    >>> import threading
    >>> import time
    >>> from zmq import Context, SUB, SUBSCRIBE

    >>> # create, configure, and run a Cornerstone instance
    >>> foo = Cornerstone()
    >>> property_bag = foo.__create_property_bag__()
    >>> property_bag.heartbeat = 1
    >>> foo.configure(args=property_bag)
    >>> t = threading.Thread(target=foo.run)
    >>> t.start()
    >>> time.sleep(1)
    >>> assert t.is_alive()
    >>> foo.kill()
    >>> t.join(1)
    >>> assert not t.is_alive()

    >>> # register an input socket
    >>> ctx = foo.zmq_ctx
    >>> sock = ctx.socket(SUB)
    >>> sock.connect('tcp://localhost:6670')
    >>> sock.setsockopt(SUBSCRIBE, "")
    >>> foo.register_input_sock(sock)
    >>> t = threading.Thread(target=foo.run)
    >>> t.start()
    >>> time.sleep(1)
    >>> foo.kill()
    >>> t.join(1)
    >>> assert not t.is_alive()
    """


    def __init__(self, **kwargs):
        self._input_sock = None
        self._output_sock = None
        self._control_sock = None

        # determine if outgoing messages should enable NOBLOCK on send
        # default behaviour is to block on a send call till receiver is present
        self.no_block_send = False

        # configure the interrupt handling
        self._stop = True
        signal.signal(signal.SIGINT, self._signal_interrupt_handler)

        # a regular hearbeat interval must be set to the default.
        self.heartbeat = 3 # seconds

        # create the zmq context
        self.zmq_ctx = Context()

        # set the default input receive handler, if none has been assigned
        if not hasattr(self, 'input_recv_handler'):
            self.input_recv_handler = self._default_recv_handler

        # set the default handler, if none has been assigned.
        if not hasattr(self, '_command_handler'):
            self._command_handler = self._default_command_handler

        # construct the poller
        self._poll = Poller()

        # monitoring of message stream is off by default
        self.monitor_stream = False

        Scaffold.__init__(self, **kwargs)


    def configuration_options(self, arg_parser=None):
        """
        The configuration_options method utilizes the arg_parser parameter to
        add arguments that should be handled during configuration.

        Keyword Arguments:
        arg_parser - argparse.ArgumentParser object.

        Sample invocation:
        >>> import argparse
        >>> parser = argparse.ArgumentParser(prog='app.py')
        >>> foo = Cornerstone()
        >>> foo.configuration_options(arg_parser=parser)
        >>> args = parser.print_usage() # doctest: +NORMALIZE_WHITESPACE
        usage: app.py [-h] [--heartbeat HEARTBEAT] [--monitor_stream]
                  [--no_block_send]
        """
        assert arg_parser

        arg_parser.add_argument('--heartbeat',
                                type=int,
                                default=3,
                                help="Set the heartbeat rate in seconds of "
                                     "the core 0mq poller timeout.")
        arg_parser.add_argument('--monitor_stream',
                                action='store_true',
                                help='Enable the sampling of message flow.')
        arg_parser.add_argument('--no_block_send',
                                action='store_true',
                                help='Enable NOBLOCK on the sending of messages.'
                                     ' This will cause an message to be dropped '
                                     'if no receiver is present.')


    def configure(self, args=None):
        """
        The configure method configures a Cornerstone instance by
        prior to the invocation of start.

        Keyword Arguments:
        args - an object with attributes set to the argument values.e
        """
        assert args


    def register_input_sock(self, sock):
        """
        Register a given input socket as the ingest point for a Cornerstone
        instance.

        Keyward Arguments:
        sock - the input socket that is to be registered.

        Return: None

        Cornerstone does not support multiple input sockets, so any currently
        registered input socket will be discarded. This is a per instance
        limitation, in which case the primary concern is ipaddress collision.

        Example Usage:
        >>> from zmq import SUB, SUBSCRIBE
        >>> foo = Cornerstone()
        >>> ctx = foo.zmq_ctx
        >>> sock1 = ctx.socket(SUB)
        >>> sock1.connect('tcp://localhost:2880')
        >>> sock1.setsockopt(SUBSCRIBE, "")
        >>> assert foo._poll.sockets == {}
        >>> foo.register_input_sock(sock1)
        >>> assert foo._poll.sockets.has_key(sock1)
        >>> sock2 = ctx.socket(SUB)
        >>> sock2.connect('tcp://localhost:2881')
        >>> sock2.setsockopt(SUBSCRIBE, "")
        >>> foo.register_input_sock(sock2)
        >>> assert not foo._poll.sockets.has_key(sock1)
        >>> assert foo._poll.sockets.has_key(sock2)
        """
        # if there is an existing input socket, then it will be removed.
        if self._input_sock is not None:
            self._poll.unregister(self._input_sock)
            self._input_sock.close()
            self._input_sock = None

        self._input_sock = sock
        if self._input_sock is not None:
            self._poll.register(self._input_sock, POLLIN)


    def register_output_sock(self, sock):
        """
        Register a given output socket as the egress point for a Cornerstone
        instance.

        Keyward Arguments:
        sock - the output socket that is to be registered.

        Return: none

        Cornerstone does not support multiple output sockets,
        so any currently registered output socket will be discarded. This is
        a per instance limitation. In which case the primary concern is
        ipaddress collision.

        Example Usage:
        >>> from zmq import PUB
        >>> foo = Cornerstone()
        >>> ctx = foo.zmq_ctx
        >>> sock1 = ctx.socket(PUB)
        >>> sock1.bind('tcp://*:2880')
        >>> assert foo._output_sock == None
        >>> foo.register_output_sock(sock1)
        >>> assert foo._output_sock == sock1
        >>> sock2 = ctx.socket(PUB)
        >>> sock2.bind('tcp://*:28881')
        >>> foo.register_output_sock(sock2)
        >>> assert foo._output_sock == sock2
        """
        # if there is an existing output socket, then it will be removed.
        if self._output_sock is not None:
            self._output_sock.close()
            self._output_sock = None

        self._output_sock = sock


    def send(self, msg):
        assert msg
        if self.monitor_stream:
            self.log.info('o: %s', msg)

        if not self.no_block_send:
            self._output_sock.send(msg)
        else:
            try:
                self._output_sock.send(msg, NOBLOCK)
            except:
                self.log.error("Unexpected error:", sys.exc_info()[0])


    def setRun(self):
        self._stop = False


    def isStopped(self):
        return self._stop


    def run(self):
        """
        Comment: -- AAA --
        What needs to occur here si to see if there is a 0mq connection
        configured. If so, then we will simply push to that connector. This
        will be the default behavior, at least for now. There should be a
        mechanism for transmitting the data out to a registered handler.
        """
        self._stop = False

        self.log.info('Beginning run() with configuration: %s', self._args)

        #todo: raul - move this section to command configuraiton layer
        # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
        # of course this is when a command configuration layer get's added
        controller = self.zmq_ctx.socket(SUB)
        controller.connect('tcp://localhost:7885')
        controller.setsockopt(SUBSCRIBE, "")
        self._control_sock = controller
        self._poll.register(self._control_sock)
        # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

        loop_count = 0
        input_count = 0
        while True:
            try:
                socks = dict(self._poll.poll(timeout=self.heartbeat))
                loop_count += 1
                if self.monitor_stream and (loop_count % 1000) == 0:
                    sys.stdout.write('loop(%s)' % loop_count)
                    sys.stdout.flush()

                if self._input_sock and socks.get(self._input_sock) == POLLIN:
                    #todo: raul - this whole section needs to be redone,
                    # see additional comment AAA above.
                    msg = self.input_recv_handler(self._input_sock)
                    input_count += 1
                    if self.monitor_stream: # and (input_count % 10) == 0:
                        self.log.info('i:%s- %s', input_count, msg)

                if (self._control_sock and
                    socks.get(self._control_sock) == POLLIN):
                    msg = self._control_sock.recv()
                    if self._command_handler is not None:
                        self._command_handler(msg)

                if self._stop:
                    self.log.info('Stop flag triggered ... shutting down.')
                    break

            except ZMQError, ze:
                if ze.errno == 4: # Known exception due to keyboard ctrl+c
                    self.log.info('System interrupt call detected.')
                else: # exit hard on unhandled exceptions
                    self.log.error('Unhandled exception in run execution:%d - %s'
                                   % (ze.errno, ze.strerror))
                    exit(-1)

        # close the sockets held by the poller
        self._control_sock.close()
        self.register_input_sock(sock=None)
        self.register_output_sock(sock=None)

        self.log.info('Run terminated for %s', self.name)
Example #53
0
class Broker(Thread):
    def __init__(self, faddr, baddr):
        Thread.__init__(self)
        self._faddr = faddr
        self._baddr = baddr
        self._init()

    def _log(self, text):
        if LOG_BROKER:
            log_debug(self, text)

    def _init(self):
        self._context = Context(1)
        self._queue = BrokerQueue()
        self._init_frontend()
        self._init_backend()
        self._init_pollers()
        self._active = True

    def _init_frontend(self):
        self._frontend = self._context.socket(ROUTER)
        self._frontend.bind(zmqaddr(self._faddr, GATEWAY_PORT))

    def _init_backend(self):
        self._backend = self._context.socket(ROUTER)
        self._backend.bind(zmqaddr(self._baddr, BROKER_PORT))

    def _init_pollers(self):
        self._poller1 = Poller()
        self._poller2 = Poller()
        self._poller1.register(self._backend, POLLIN)
        self._poller2.register(self._backend, POLLIN)
        self._poller2.register(self._frontend, POLLIN)

    def run(self):
        self._log('start ...')
        timeout = time.time() + PPP_HEARTBEAT_INTERVAL
        while self._active:
            if len(self._queue.queue) > 0:
                poller = self._poller2
            else:
                poller = self._poller1
            socks = dict(poller.poll(PPP_HEARTBEAT_INTERVAL * 1000))
            if socks.get(self._backend) == POLLIN:
                frames = self._backend.recv_multipart()
                if not frames:
                    break
                identity = frames[0]
                self._queue.add(BrokerItem(identity))
                msg = frames[1:]
                if len(msg) == 1:
                    if msg[0] not in (PPP_READY, PPP_HEARTBEAT):
                        log_err(self, "invalid message")
                else:
                    self._frontend.send_multipart(msg)

                if time.time() >= timeout:
                    for identity in self._queue.queue:
                        msg = [identity, PPP_HEARTBEAT]
                        self._backend.send_multipart(msg)
                    timeout = time.time() + PPP_HEARTBEAT_INTERVAL

            if socks.get(self._frontend) == POLLIN:
                frames = self._frontend.recv_multipart()
                if not frames:
                    break
                frames.insert(0, self._queue.pop())
                self._backend.send_multipart(frames)

            self._queue.purge()

    def stop(self):
        self._active = False
Example #54
0
class RouterQueue(Process, Component):
    """`RouterQueue` class represents a router between pipeline steps, and it
    manages queue for prev step.
    It is derived from Process class.
    RouterQueue class is connected to one or more input steps and to one
    or more output steps thanks to zmq.ROUTER sockets.
    If inputs arrive quickers than output are sent (because next stage have not
    enough time to compute) these inputs are queued in RouterQueue.
    RouterQueue send output the next steps in LRU(last recently used) pattern.
    If a queue limit is reach for a step, RouterQueue replies "FULL"
    without taking new job in account, so the prev step have to send it again
    later.
    """
    def __init__(self, connexions=dict(), gui_address=None):
        """
        Parameters
        ----------
        connexions: dict {'STEP_NANE' : (STEP port in, STEP port out,
                                            STEP queue lengh max )}
            Port in, port out  for socket for each next steps.
            Max queue lengh (-1 menans no maximum)
        gui_address : str
            GUI port for ZMQ 'hostname': + 'port'
        """
        Process.__init__(self)
        Component.__init__(self, parent=None)
        self.gui_address = gui_address
        # list of available stages which receive next job
        # This list allow to use next_stage in a LRU (Last recently used)
        # pattern
        self.next_available_stages = dict()
        # queue jobs to be send to next_stage
        self.queue_jobs = dict()
        self.router_sockets = dict()
        self.dealer_sockets = dict()
        self.queue_limit = dict()
        self.connexions = connexions
        self.done = False
        self._stop = Value('i', 0)
        self._total_queue_size = Value('i', 0)

    def run(self):
        """
        Method representing the processus’s activity.
        It sends a job present in its queue (FIFO) to an available stager
        (if exist). Then it polls its sockets (in and out).
        When received new input from input socket, it appends contains to
        its queue.
        When received a signal from its output socket, it append sender
        (a pipeline step) to availble stagers list
        """
        if self.init_connexions():
            nb_job_remains = 0
            while not self.stop or nb_job_remains > 0:
                for name in self.connexions:
                    queue = self.queue_jobs[name]
                    next_available = self.next_available_stages[name]
                    if queue and next_available:
                        # get that oldest job and remove it form list
                        job = self.queue_jobs[name].pop(0)
                        if self.gui_address: self.update_gui(name)
                        # Get the next_stage for new job, and remove it from
                        # available list
                        next_stage = self.next_available_stages[name].pop(0)
                        # send new job
                        self.dealer_sockets[name].send_multipart(
                            [next_stage, b"", dumps(job)])
                    # check if new socket message arrive. Or skip after timeout
                    # (100 s)
                sockets = dict(self.poller.poll(100))
                # Test if message arrived from next_stages
                for n, socket_dealer in self.dealer_sockets.items():
                    if (socket_dealer in sockets
                            and sockets[socket_dealer] == POLLIN):
                        request = socket_dealer.recv_multipart()
                        # Get next_stage identity(to responde) and message
                        next_stage, _, _ = request[:3]
                        # add stager to next_available_stages
                        self.next_available_stages[n].append(next_stage)
                # Test if message arrived from prev_stage (stage or producer)
                for n, socket_router in self.router_sockets.items():
                    if (socket_router in sockets
                            and sockets[socket_router] == POLLIN):
                        # Get next prev_stage request
                        address, empty, request = socket_router.recv_multipart(
                        )
                        # store it to job queue
                        queue = self.queue_jobs[n]
                        if (len(queue) > self.queue_limit[n]
                                and self.queue_limit[n] != -1):
                            socket_router.send_multipart(
                                [address, b"", b"FULL"])
                        else:
                            queue.append(loads(request))
                            if self.gui_address: self.update_gui(n)
                            socket_router.send_multipart([address, b"", b"OK"])
                nb_job_remains = 0
                for n, queue in self.queue_jobs.items():
                    nb_job_remains += len(queue)
                self._total_queue_size.value = nb_job_remains
            for socket in self.router_sockets.values():
                socket.close()
            for socket in self.dealer_sockets.values():
                socket.close()
        self.done = True

    def isQueueEmpty(self, stage_name=None):
        """
        Get status of steps' queue
        Parameters
        ----------
        stage_name: str
            router_name corresponding to stager name or consumer name
            If stage_name=None, it take in account all queues
        Returns
        -------
        True is corresponding queue is empty, otherwise False
        If stage_name = None it returns True if sum of all queues is 0
        """
        if stage_name:
            val = stage_name.find("$$processus_number$$")
            if val != -1:
                name_to_search = stage_name[0:val]
            else:
                name_to_search = stage_name
            for name, queue in self.queue_jobs.items():
                if name.find("_router"):
                    pos = name.find("_router")
                    name = name[:pos]
                if name == name_to_search:
                    if not queue:
                        return True
            return False
        else:
            for queue in self.queue_jobs.values():
                if queue:
                    return False
            return True

    def init_connexions(self):
        """
        Initialise zmq sockets, poller and queues.
        Because this class is s Process, This method must be call in the run
         method to be hold by the correct processus.
        """
        # Prepare our context and sockets
        context = Context()
        # Socket to talk to prev_stages
        for name, connexions in self.connexions.items():
            self.queue_limit[name] = connexions[2]
            sock_router = context.socket(ROUTER)
            try:
                sock_router.bind('tcp://*:' + connexions[0])
            except ZMQError as e:
                self.log.error('{} : tcp://localhost:{}'.format(
                    e, connexions[0]))
                return False
            self.router_sockets[name] = sock_router
            # Socket to talk to next_stages
            sock_dealer = context.socket(ROUTER)
            try:
                sock_dealer.bind("tcp://*:" + connexions[1])
            except ZMQError as e:
                self.log.error('{} : tcp://localhost:{}'.format(
                    e, connexions[1]))
                return False
            self.dealer_sockets[name] = sock_dealer
            self.next_available_stages[name] = list()
            self.queue_jobs[name] = list()
        # Use a ZMQ Pool to get multichannel message
        self.poller = Poller()
        # Register dealer socket to next_stage
        for n, dealer in self.dealer_sockets.items():
            self.poller.register(dealer, POLLIN)
        for n, router in self.router_sockets.items():
            self.poller.register(router, POLLIN)
        # Register router socket to prev_stages or producer
        self.socket_pub = context.socket(PUB)
        if self.gui_address is not None:
            try:
                self.socket_pub.connect("tcp://" + self.gui_address)
            except ZMQError as e:
                self.log.error("".format(e, self.gui_address))
                return False
        # This flag stop this current processus
        return True

    def update_gui(self, name):
        """
        send status to GUI
        name: str
            step name
        """
        msg = [name, str(len(self.queue_jobs[name]))]
        self.socket_pub.send_multipart([b'GUI_ROUTER_CHANGE', dumps(msg)])

    @property
    def stop(self):
        return self._stop.value

    @stop.setter
    def stop(self, value):
        self._stop.value = value

    @property
    def total_queue_size(self):
        return self._total_queue_size.value
Example #55
0
class RequestManager(Thread):
    """Manage requests.
    """
    def __init__(self, port, attrs=None):
        Thread.__init__(self)

        self._loop = True
        self.out_socket = get_context().socket(ROUTER)
        self.out_socket.bind("tcp://*:" + str(port))
        self.port = port
        self.in_socket = get_context().socket(PULL)
        self.in_socket.bind("inproc://replies" + str(port))

        self._poller = Poller()
        self._poller.register(self.out_socket, POLLIN)
        self._poller.register(self.in_socket, POLLIN)
        self._attrs = attrs
        try:
            # Checking the validity of the file pattern
            _pattern = globify(attrs["origin"])
        except ValueError as err:
            raise ConfigError('Invalid file pattern: ' + str(err))
        except KeyError:
            if 'listen' not in attrs:
                raise
        self._deleter = Deleter()

        try:
            self._station = self._attrs["station"]
        except (KeyError, TypeError):
            LOGGER.warning("Station is not defined in config file")
            self._station = "unknown"
        LOGGER.debug("Station is '%s'" % self._station)

    def start(self):
        self._deleter.start()
        Thread.start(self)

    def pong(self, message):
        """Reply to ping
        """
        return Message(message.subject, "pong", {"station": self._station})

    def push(self, message):
        """Reply to push request
        """
        for the_dict in gen_dict_contains(message.data, 'uri'):
            uri = urlparse(the_dict['uri'])
            rel_path = the_dict.get('path', '')
            pathname = uri.path
            # FIXME: check against file_cache
            if 'origin' in self._attrs and not fnmatch.fnmatch(
                    os.path.basename(pathname),
                    os.path.basename(globify(self._attrs["origin"]))):
                LOGGER.warning('Client trying to get invalid file: %s',
                               pathname)
                return Message(message.subject,
                               "err",
                               data="{0:s} not reachable".format(pathname))
            try:
                move_it(pathname,
                        message.data['destination'],
                        self._attrs,
                        rel_path=rel_path)
            except Exception as err:
                return Message(message.subject, "err", data=str(err))
            else:
                if (self._attrs.get('compression')
                        or self._attrs.get('delete', 'False').lower()
                        in ["1", "yes", "true", "on"]):
                    self._deleter.add(pathname)

            if 'dataset' in message.data:
                mtype = 'dataset'
            elif 'collection' in message.data:
                mtype = 'collection'
            elif 'uid' in message.data:
                mtype = 'file'
            else:
                raise KeyError('No known metadata in message.')

        new_msg = Message(message.subject, mtype, data=message.data.copy())
        new_msg.data['destination'] = clean_url(new_msg.data['destination'])
        return new_msg

    def ack(self, message):
        """Reply with ack to a publication
        """
        for url in gen_dict_extract(message.data, 'uri'):
            uri = urlparse(url)
            pathname = uri.path

            if 'origin' in self._attrs and not fnmatch.fnmatch(
                    os.path.basename(pathname),
                    os.path.basename(globify(self._attrs["origin"]))):
                LOGGER.warning('Client trying to get invalid file: %s',
                               pathname)
                return Message(message.subject,
                               "err",
                               data="{0:s} not reacheable".format(pathname))

            if (self._attrs.get('compression') or self._attrs.get(
                    'delete', 'False').lower() in ["1", "yes", "true", "on"]):
                self._deleter.add(pathname)
        new_msg = Message(message.subject, "ack", data=message.data.copy())
        try:
            new_msg.data['destination'] = clean_url(
                new_msg.data['destination'])
        except KeyError:
            pass
        return new_msg

    def info(self, message):
        topic = message.subject
        max_count = 2256  # Let's set a (close to arbitrary) limit on messages size.
        try:
            max_count = min(message.data.get("max_count", max_count),
                            max_count)
        except AttributeError:
            pass
        uptime = datetime.datetime.utcnow() - START_TIME
        files = []
        with file_cache_lock:
            for i in file_cache:
                if i.startswith(topic):
                    files.append(i)
                    if len(files) == max_count:
                        break
        return Message(message.subject,
                       "info",
                       data={
                           "files": files,
                           "max_count": max_count,
                           "uptime": str(uptime)
                       })

    def unknown(self, message):
        """Reply to any unknown request.
        """
        return Message(message.subject, "unknown")

    def reply_and_send(self, fun, address, message):
        in_socket = get_context().socket(PUSH)
        in_socket.connect("inproc://replies" + str(self.port))

        reply = Message(message.subject, "error")
        try:
            reply = fun(message)
        except Exception:
            LOGGER.exception(
                "Something went wrong"
                " when processing the request: %s", str(message))
        finally:
            LOGGER.debug("Response: " + str(reply))
            try:
                in_socket.send_multipart([address, b'', str(reply)])
            except TypeError:
                in_socket.send_multipart(
                    [address, b'', bytes(str(reply), 'utf-8')])

    def run(self):
        while self._loop:
            try:
                socks = dict(self._poller.poll(timeout=2000))
            except ZMQError:
                LOGGER.info("Poller interrupted.")
                continue
            if socks.get(self.out_socket) == POLLIN:
                LOGGER.debug("Received a request")
                address, empty, payload = self.out_socket.recv_multipart(
                    NOBLOCK)
                message = Message(rawstr=payload)
                fake_msg = Message(rawstr=str(message))
                try:
                    urlparse(message.data['destination'])
                except (KeyError, TypeError):
                    pass
                else:
                    fake_msg.data['destination'] = clean_url(
                        message.data['destination'])

                LOGGER.debug("processing request: " + str(fake_msg))
                if message.type == "ping":
                    Thread(target=self.reply_and_send,
                           args=(self.pong, address, message)).start()
                elif message.type == "push":
                    Thread(target=self.reply_and_send,
                           args=(self.push, address, message)).start()
                elif message.type == "ack":
                    Thread(target=self.reply_and_send,
                           args=(self.ack, address, message)).start()
                elif message.type == "info":
                    Thread(target=self.reply_and_send,
                           args=(self.info, address, message)).start()
                else:  # unknown request
                    Thread(target=self.reply_and_send,
                           args=(self.unknown, address, message)).start()
            elif socks.get(self.in_socket) == POLLIN:
                self.out_socket.send_multipart(
                    self.in_socket.recv_multipart(NOBLOCK))

            else:  # timeout
                pass

    def stop(self):
        """Stop the request manager."""
        self._loop = False
        self._deleter.stop()
        self.out_socket.close(1)
        self.in_socket.close(1)
Example #56
0
class ZmqSelector(BaseSelector):
    """A selector that can be used with asyncio's selector base event loops."""

    def __init__(self):
        # this maps file descriptors to keys
        self._fd_to_key = {}
        # read-only mapping returned by get_map()
        self._map = _SelectorMapping(self)
        self._poller = ZMQPoller()

    def _fileobj_lookup(self, fileobj):
        """Return a file descriptor from a file object.

        This wraps _fileobj_to_fd() to do an exhaustive search in case
        the object is invalid but we still have it in our map.  This
        is used by unregister() so we can unregister an object that
        was previously registered even if it is closed.  It is also
        used by _SelectorMapping.
        """
        try:
            return _fileobj_to_fd(fileobj)
        except ValueError:
            # Do an exhaustive search.
            for key in self._fd_to_key.values():
                if key.fileobj is fileobj:
                    return key.fd
            # Raise ValueError after all.
            raise

    def register(self, fileobj, events, data=None):
        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
            raise ValueError("Invalid events: {!r}".format(events))

        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)

        if key.fd in self._fd_to_key:
            raise KeyError("{!r} (FD {}) is already registered"
                           .format(fileobj, key.fd))

        z_events = 0
        if events & EVENT_READ:
            z_events |= POLLIN
        if events & EVENT_WRITE:
            z_events |= POLLOUT
        try:
            self._poller.register(key.fd, z_events)
        except ZMQError as exc:
            raise OSError(exc.errno, exc.strerror) from exc

        self._fd_to_key[key.fd] = key
        return key

    def unregister(self, fileobj):
        try:
            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
        except KeyError:
            raise KeyError("{!r} is not registered".format(fileobj)) from None
        try:
            self._poller.unregister(key.fd)
        except ZMQError as exc:
            self._fd_to_key[key.fd] = key
            raise OSError(exc.errno, exc.strerror) from exc
        return key

    def modify(self, fileobj, events, data=None):
        try:
            fd = self._fileobj_lookup(fileobj)
            key = self._fd_to_key[fd]
        except KeyError:
            raise KeyError("{!r} is not registered".format(fileobj)) from None
        if data == key.data and events == key.events:
            return key
        if events != key.events:
            z_events = 0
            if events & EVENT_READ:
                z_events |= POLLIN
            if events & EVENT_WRITE:
                z_events |= POLLOUT
            try:
                self._poller.modify(fd, z_events)
            except ZMQError as exc:
                raise OSError(exc.errno, exc.strerror) from exc

        key = key._replace(data=data, events=events)
        self._fd_to_key[key.fd] = key
        return key

    def close(self):
        self._fd_to_key.clear()
        self._poller = None

    def get_map(self):
        return self._map

    def _key_from_fd(self, fd):
        """Return the key associated to a given file descriptor.

        Parameters:
        fd -- file descriptor

        Returns:
        corresponding key, or None if not found
        """
        try:
            return self._fd_to_key[fd]
        except KeyError:
            return None

    def select(self, timeout=None):
        if timeout is None:
            timeout = None
        elif timeout <= 0:
            timeout = 0
        else:
            # poll() has a resolution of 1 millisecond, round away from
            # zero to wait *at least* timeout seconds.
            timeout = math.ceil(timeout * 1e3)

        ready = []
        try:
            z_events = self._poller.poll(timeout)
        except ZMQError as exc:
            if exc.errno == EINTR:
                return ready
            else:
                raise OSError(exc.errno, exc.strerror) from exc

        for fd, evt in z_events:
            events = 0
            if evt & POLLIN:
                events |= EVENT_READ
            if evt & POLLOUT:
                events |= EVENT_WRITE
            if evt & POLLERR:
                events = EVENT_READ | EVENT_WRITE

            key = self._key_from_fd(fd)
            if key:
                ready.append((key, events & key.events))

        return ready
Example #57
0
class PushRequester(object):
    """Base requester class.
    """

    request_retries = 3

    def __init__(self, host, port):
        self._socket = None
        self._reqaddress = "tcp://" + host + ":" + str(port)
        self._poller = Poller()
        self._lock = Lock()
        self.failures = 0
        self.jammed = False
        self.running = True

        self.connect()

    def connect(self):
        """Connect to the server
        """
        self._socket = context.socket(REQ)
        self._socket.connect(self._reqaddress)
        self._poller.register(self._socket, POLLIN)

    def stop(self):
        """Close the connection to the server
        """
        self.running = False
        self._socket.setsockopt(LINGER, 0)
        self._socket.close()
        self._poller.unregister(self._socket)

    def reset_connection(self):
        """Reset the socket
        """
        self.stop()
        self.connect()

    def __del__(self, *args, **kwargs):
        self.stop()

    def send_and_recv(self, msg, timeout=DEFAULT_REQ_TIMEOUT):

        with self._lock:
            retries_left = self.request_retries
            request = str(msg)
            self._socket.send(request)
            rep = None
            small_timeout = 0.1
            while retries_left and self.running:
                now = time.time()
                while time.time() < now + timeout:
                    if not self.running:
                        return rep
                    socks = dict(self._poller.poll(small_timeout))
                    if socks.get(self._socket) == POLLIN:
                        reply = self._socket.recv()
                        if not reply:
                            LOGGER.error("Empty reply!")
                            break
                        try:
                            rep = Message(rawstr=reply)
                        except MessageError as err:
                            LOGGER.error('Message error: %s', str(err))
                            break
                        LOGGER.debug("Receiving (REQ) %s", str(rep))
                        self.failures = 0
                        self.jammed = False
                        return rep
                    # During big file transfers, give some time to a friend.
                    time.sleep(0.1)

                LOGGER.warning("Timeout from " + str(self._reqaddress) +
                               ", retrying...")
                # Socket is confused. Close and remove it.
                self.stop()
                retries_left -= 1
                if retries_left <= 0:
                    LOGGER.error("Server doesn't answer, abandoning... " +
                                 str(self._reqaddress))
                    self.connect()
                    self.failures += 1
                    if self.failures == 5:
                        LOGGER.critical("Server jammed ? %s", self._reqaddress)
                        self.jammed = True
                    break
                LOGGER.info("Reconnecting and resending " + str(msg))
                # Create new connection
                self.connect()
                self._socket.send(request)

        return rep