def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = get_context().socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
logger.debug('Connecting to %s',
"tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send_string(six.text_type(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv_string(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds." %
timeout)
finally:
socket.close()
def __init__(self, **kwargs):
self._input_sock = None
self._output_sock = None
self._control_sock = None
# determine if outgoing messages should enable NOBLOCK on send
# default behaviour is to block on a send call till receiver is present
self.no_block_send = False
# configure the interrupt handling
self._stop = True
signal.signal(signal.SIGINT, self._signal_interrupt_handler)
# a regular hearbeat interval must be set to the default.
self.heartbeat = 3 # seconds
# create the zmq context
self.zmq_ctx = Context()
# set the default input receive handler, if none has been assigned
if not hasattr(self, 'input_recv_handler'):
self.input_recv_handler = self._default_recv_handler
# set the default handler, if none has been assigned.
if not hasattr(self, '_command_handler'):
self._command_handler = self._default_command_handler
# construct the poller
self._poll = Poller()
# monitoring of message stream is off by default
self.monitor_stream = False
Scaffold.__init__(self, **kwargs)
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(max_age=self._max_age)
arec.start()
port = PORT
try:
self.listener = context.socket(REP)
self.listener.bind("tcp://*:" + str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
self.listener.send_unicode(str(get_active_address(
msg.data["service"], arec)))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.listener.close()
def __init__(self,
server_host,
server_port,
timeout=3,
encoding="utf-8",
skip_translations=False,
translation_cache=None):
""" Create a client instance which can be used to get translations.
A client will close all resources on exit. If resource should be freed
before that use the close function.
:type server_host: str
:type server_port: int
:param timeout: Timeout for a request, in seconds.
:type timeout: int
:type encoding: str
"""
self._timeout = timeout or 0.001 # seconds
self._encoding = encoding
self._context = Context(1)
self._socket = self._context.socket(DEALER)
self._socket.set(LINGER, 0) # do not wait for anything when closing
self._socket.connect("tcp://{}:{}".format(server_host, server_port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
self._skip_translations = skip_translations
self._translation_cache = translation_cache
if self._translation_cache and not isinstance(self._translation_cache,
TranslationCache):
raise TypeError(
"The cache needs to be a subclass of TranslationCache")
def __init__(self, port, attrs=None):
Thread.__init__(self)
self._loop = True
self.out_socket = get_context().socket(ROUTER)
self.out_socket.bind("tcp://*:" + str(port))
self.port = port
self.in_socket = get_context().socket(PULL)
self.in_socket.bind("inproc://replies" + str(port))
self._poller = Poller()
self._poller.register(self.out_socket, POLLIN)
self._poller.register(self.in_socket, POLLIN)
self._attrs = attrs
try:
# Checking the validity of the file pattern
_pattern = globify(attrs["origin"])
except ValueError as err:
raise ConfigError('Invalid file pattern: ' + str(err))
except KeyError:
if 'listen' not in attrs:
raise
self._deleter = Deleter()
try:
self._station = self._attrs["station"]
except (KeyError, TypeError):
LOGGER.warning("Station is not defined in config file")
self._station = "unknown"
LOGGER.debug("Station is '%s'" % self._station)
def __init__(self, impl=None):
self._impl = impl or Poller()
if hasattr(self._impl, 'fileno'):
self._set_close_exec(self._impl.fileno())
self._handlers = {}
self._events = {}
self._callbacks = set()
self._timeouts = []
self._running = False
self._stopped = False
self._blocking_log_threshold = None
# Create a pipe that we send bogus data to when we want to wake
# the I/O loop when it is idle
if os.name != 'nt':
r, w = os.pipe()
self._set_nonblocking(r)
self._set_nonblocking(w)
self._set_close_exec(r)
self._set_close_exec(w)
self._waker_reader = os.fdopen(r, "r", 0)
self._waker_writer = os.fdopen(w, "w", 0)
else:
self._waker_reader = self._waker_writer = win32_support.Pipe()
r = self._waker_writer.reader_fd
self.add_handler(r, self._read_waker, self.READ)
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
def __init__(self, options):
self.ctx = Context.instance()
self.sck = self.ctx.socket(REQ)
self.poller = Poller()
self.opts = options
self.sck.setsockopt(LINGER, 0)
self.poller.register(self.sck, POLLIN)
def __init__(self, host, port):
self._host = host
self._port = port
self._context = Context()
self._socket = self._context.socket(REQ)
self._socket.setsockopt(LINGER, 1)
self._socket.connect("tcp://" + host + ":" + str(port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
def start_poll(pull, push, sub, hash_checker):
global INPUT_DATA
seq_no = 0
poller = Poller()
poller.register(pull, zmq.POLLIN)
poller.register(sub, zmq.POLLIN)
while True:
try:
socks = dict(poller.poll())
except KeyboardInterrupt:
return
if sub in socks:
log.debug('recv -> {}'.format(sub.recv_multipart()))
try:
log.info('Concatting...')
concat_string = process()
concat_string_hash = HASHER(concat_string).digest()
# Send final hash
push.send_multipart([
MessageType.INPUT.value,
seq_no.to_bytes(2, byteorder='big'),
concat_string_hash,
])
log.debug('send -> {} | {} | {}', MessageType.INPUT.name,
seq_no, concat_string_hash.hex())
# Send original concatted string
push.send_multipart([
MessageType.COMMIT.value,
seq_no.to_bytes(2, byteorder='big'),
concat_string,
])
log.debug('send -> {} | {} | {}...', MessageType.COMMIT.name,
seq_no, concat_string[:20])
INPUT_DATA.clear()
log.info("Reset concat")
if seq_no == 0xFFFF:
seq_no = 0
else:
seq_no += 1
except Exception as e:
log.error("Could not concat inputs -> {}".format(e))
if pull in socks:
inp = pull.recv()
log.debug('recv -> {}'.format(inp.hex()))
try:
hash_checker.check(inp)
except:
log.warn('Bad hash received! -> {}', inp)
continue
INPUT_DATA.append(inp)
log.info('Input added | Elements: {}'.format(len(INPUT_DATA)))
def __init__(self, host, port):
self._socket = None
self._reqaddress = "tcp://" + host + ":" + str(port)
self._poller = Poller()
self._lock = Lock()
self.failures = 0
self.jammed = False
self.connect()
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
def refresh_serum_index(self):
try:
rawData = []
for i in range(3):
socket = self.context.socket(REQ)
socket.connect("tcp://serum-server:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send(dumps({"endpoint": "list"}))
responses = poller.poll(10000)
if len(responses) != 0:
response = socket.recv()
socket.close()
rawData = loads(response)
break
else:
socket.close()
for market in rawData["markets"]:
base, quote = market["name"].split("/", 1)
if base not in self.serumIndex:
self.serumIndex[base] = []
self.serumIndex[base].append({"id": market["address"], "name": base, "base": base, "quote": quote, "image": None, "program": market["programId"]})
for token in rawData["tokenList"]:
symbol = token["symbol"].upper()
if symbol not in self.serumIndex:
self.serumIndex[symbol] = []
processed = []
for market in self.serumIndex[symbol]:
processed.append(market["quote"])
market["name"] = token["name"]
market["image"] = token.get("logoURI")
for extension, address in token.get("extensions", {}).items():
if extension.startswith("serumV3"):
quote = extension.removeprefix("serumV3").upper()
if quote not in processed:
processed.append(quote)
self.serumIndex[symbol].append({"id": address, "name": token["name"], "base": symbol, "quote": quote, "image": token.get("logoURI"), "program": "9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin"})
if len(self.serumIndex[symbol]) == 0:
self.serumIndex.pop(symbol)
elif self.serumIndex[symbol][0]["quote"] != "USDC":
usdcMarket = None
for index, market in enumerate(self.serumIndex[symbol]):
if market["quote"] == "USDC":
usdcMarket = self.serumIndex[symbol].pop(index)
break
if usdcMarket is not None:
self.serumIndex[symbol].insert(0, usdcMarket)
except Exception:
print(format_exc())
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def __init__(self, host, port):
"""Initialize pish request."""
self._socket = None
self._reqaddress = "tcp://" + host + ":" + str(port)
self._poller = Poller()
self._lock = Lock()
self.failures = 0
self.jammed = False
self.running = True
self.connect()
def __init__(self):
super(Listener, self).__init__(name="Listener")
self._shutdown = False
self.context = Context()
self.sub = self.context.socket(SUB)
self.sub.bind('tcp://*:7000')
self.sub.setsockopt(SUBSCRIBE, "")
self.poller = Poller()
self.poller.register(self.sub, POLLIN)
def __init__(self, port):
self._ctx = Context()
self._socket = self._ctx.socket(DEALER)
self._socket.bind('tcp://*:%s' % port)
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
self._clients = {}
self._actions = {'register': self._register_client}
def __init__(self, holder, port, station):
Thread.__init__(self)
self._holder = holder
self._loop = True
self._port = port
self._station = station
self._lock = Lock()
self._socket = get_context().socket(REP)
self._socket.bind("tcp://*:" + str(self._port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
def __init__(self, context, *args, **kw):
self.context = context
self.recv = self.context.socket(PAIR)
self.recv.connect("inproc://#1")
self.pub = self.context.socket(PUB)
self.pub.connect('tcp://localhost:7003')
self.pub.setsockopt(HWM, 1000)
self.poller = Poller()
self.poller.register(self.recv, POLLIN)
super(Stethoscope, self).__init__(*args, **kw)
def __init__(self, addresses, translate=False):
self._addresses = addresses
self._translate = translate
self.subscribers = []
self._poller = Poller()
for addr in self._addresses:
subscriber = context.socket(SUB)
subscriber.setsockopt(SUBSCRIBE, "pytroll")
subscriber.connect(addr)
self.subscribers.append(subscriber)
self._poller.register(subscriber)
self._lock = Lock()
self._loop = True
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def listen(self, topics: Iterable[str] = ("log|", )):
"""
Listens to incoming log messages on the ZeroMQ socket(s).
Blocks until `Ctrl + C` is pressed,
or ``SIGINT`` or ``SIGTERM`` is received by the process.
Returns immediately if either a bind address nor at least
one connect address is set.
:param topics:
The topic to subscribe with.
Can be used for selecting log messages from specific source.
Defaults to ``["log|"]``.
:type topics: Iterable[str]
"""
if not self._bind_address and not self._connect_addresses:
return
poller = Poller()
if self._bind_address:
self._pull_socket = self._ctx.socket(PULL)
self._pull_socket.bind('tcp://' + self._bind_address)
poller.register(self._pull_socket, POLLIN)
if self._connect_addresses:
self._sub_socket = self._ctx.socket(SUB)
for topic in topics:
self._sub_socket.setsockopt(
SUBSCRIBE, topic.encode(encoding=_TOPIC_ENCODING))
for address in self._connect_addresses:
self._sub_socket.connect('tcp://' + address)
poller.register(self._sub_socket, POLLIN)
try:
while not is_stopped():
events = dict(poller.poll(self._interval * 1000))
if self._pull_socket in events \
and events[self._pull_socket] == POLLIN:
data = self._pull_socket.recv_multipart(flags=NOBLOCK)
self._handle_message(data)
if self._sub_socket in events \
and events[self._sub_socket] == POLLIN:
data = self._sub_socket.recv_multipart(flags=NOBLOCK)
self._handle_message(data)
finally:
if self._pull_socket:
self._pull_socket.close()
self._pull_socket = None
if self._sub_socket:
self._sub_socket.close()
self._sub_socket = None
def __init__(self, holder, host, pubport, reqport, sched):
Thread.__init__(self)
self._holder = holder
self._pubaddress = "tcp://" + host + ":" + str(pubport)
self._reqaddress = "tcp://" + host + ":" + str(reqport)
self._req = SimpleRequester(host, reqport)
self._subsocket = get_context().socket(SUB)
self._subsocket.connect(self._pubaddress)
self._subsocket.setsockopt_string(SUBSCRIBE, "pytroll")
self._poller = Poller()
self._poller.register(self._subsocket, POLLIN)
self._lock = Lock()
self._loop = True
self._sched = sched
def init_connexions(self):
"""
Initialise zmq sockets, poller and queues.
Because this class is s Process, This method must be call in the run
method to be hold by the correct processus.
"""
# Prepare our context and sockets
context = Context()
# Socket to talk to prev_stages
for name, connexions in self.connexions.items():
self.queue_limit[name] = connexions[2]
sock_router = context.socket(ROUTER)
try:
sock_router.bind('tcp://*:' + connexions[0])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[0]))
return False
self.router_sockets[name] = sock_router
# Socket to talk to next_stages
sock_dealer = context.socket(ROUTER)
try:
sock_dealer.bind("tcp://*:" + connexions[1])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[1]))
return False
self.dealer_sockets[name] = sock_dealer
self.next_available_stages[name] = list()
self.queue_jobs[name] = list()
# Use a ZMQ Pool to get multichannel message
self.poller = Poller()
# Register dealer socket to next_stage
for n, dealer in self.dealer_sockets.items():
self.poller.register(dealer, POLLIN)
for n, router in self.router_sockets.items():
self.poller.register(router, POLLIN)
# Register router socket to prev_stages or producer
self.socket_pub = context.socket(PUB)
if self.gui_address is not None:
try:
self.socket_pub.connect("tcp://" + self.gui_address)
except ZMQError as e:
self.log.error("".format(e, self.gui_address))
return False
# This flag stop this current processus
return True
def _request_quote(cls, request, ticker, context=None):
try:
socket = context.socket(REQ)
socket.connect("tcp://serum-server:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send(bytes(dumps({"endpoint": "quote", "marketAddress": ticker.get("id"), "program": ticker.get("symbol")}), encoding='utf8'))
responses = poller.poll(5000)
if len(responses) != 0:
response = socket.recv()
socket.close()
rawData = loads(response)
else:
socket.close()
return [{}, ""]
except:
return [{}, ""]
coinThumbnail = static_storage.icon if ticker.get("image") is None else ticker.get("image")
price = [float(rawData["price"])]
priceChange = 0
volume = 0
tokenImage = static_storage.icon
base = "USD" if ticker.get("base") in AbstractProvider.stableCoinTickers else ticker.get("base")
quote = "USD" if ticker.get("quote") in AbstractProvider.stableCoinTickers else ticker.get("quote")
payload = {
"quotePrice": "{:,.10f}".format(price[0]).rstrip('0').rstrip('.') + " " + quote,
"quoteVolume": "{:,.4f}".format(volume).rstrip('0').rstrip('.') + " " + base,
"title": ticker.get("name"),
"thumbnailUrl": coinThumbnail,
"messageColor": "amber" if priceChange == 0 else ("green" if priceChange > 0 else "red"),
"sourceText": "Data from Serum DEX",
"platform": "Serum",
"raw": {
"quotePrice": [price[0]],
"quoteVolume": [volume],
"timestamp": time()
}
}
return [payload, ""]
def __init__(self, timeout=-1):
self.controllerSocket = None
self.timecodeSocket = None
self.requestSocket = None
self.subscribers = {}
self.subscriberCallbacks = {}
self.subscriberTopicFilters = {}
self.periodicPublisher = None
self.commandCb = None
self.requestCb = None
self.timeout = timeout
self.poller = Poller()
self.running = False
self.pollCounter = 0
def init_connections(self):
"""
Initialise zmq sockets.
Because this class is s Process, This method must be call in the run
method to be hold by the correct process.
"""
Connections.init_connections(self)
context = Context()
self.sock_for_me = context.socket(REQ)
self.sock_for_me.connect(self.sock_job_for_me_url)
# Use a ZMQ Pool to get multichannel message
self.poll = Poller()
# Register sockets
self.poll.register(self.sock_for_me, POLLIN)
# Send READY to next_router to inform about my capacity to compute new
# job
self.sock_for_me.send_pyobj("READY")
return True
def _server(self, response):
""" Wait for a client request, record it and send the response. """
context = Context()
try:
socket = context.socket(REP)
try:
socket.set(LINGER, 0)
socket.bind("tcp://*:{}".format(_PORT))
poller = Poller()
poller.register(socket, POLLIN)
sockets = dict(poller.poll(_TIMEOUT))
if socket in sockets:
self._client_request = socket.recv_multipart()
if response:
socket.send_multipart(response)
finally:
socket.close()
finally:
context.destroy(linger=0)
def __init__(self, addresses, topics='', message_filter=None,
translate=False):
self._topics = self._magickfy_topics(topics)
self._filter = message_filter
self._translate = translate
self.sub_addr = {}
self.addr_sub = {}
self.poller = None
self._hooks = []
self._hooks_cb = {}
self.poller = Poller()
self._lock = Lock()
self.update(addresses)
self._loop = True
def execute_parser_request(endpoint, parameters, timeout=5):
socket = TickerParser.zmqContext.socket(REQ)
payload, responseText = None, None
socket.connect("tcp://parser:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send_multipart([endpoint] + parameters)
responses = poller.poll(timeout * 1000)
if len(responses) != 0:
response = socket.recv_multipart()
socket.close()
return response
else:
socket.close()
raise Exception("time out")
return None
