class WorkerRep(object): """Helper class to represent a worker in the broker. Instances of this class are used to track the state of the attached worker and carry the timers for incomming and outgoing heartbeats. :param proto: the worker protocol id. :type wid: str :param wid: the worker id. :type wid: str :param service: service this worker serves :type service: str :param stream: the ZMQStream used to send messages :type stream: ZMQStream """ def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return def send_hb(self): """Called on every HB_INTERVAL. Decrements the current liveness by one. Sends heartbeat to worker. """ self.curr_liveness -= 1 msg = [self.id, b'', self.proto, b'\x04'] self.stream.send_multipart(msg) return def on_heartbeat(self): """Called when a heartbeat message from the worker was received. Sets current liveness to HB_LIVENESS. """ self.curr_liveness = HB_LIVENESS return def is_alive(self): """Returns True when the worker is considered alive. """ return self.curr_liveness > 0 def shutdown(self): """Cleanup worker. Stops timer. """ self.hb_out_timer.stop() self.hb_out_timer = None self.stream = None return
class BufferModule(base.ZmqProcess): def __init__(self, recv_addr, send_addr, recv_title, send_title, state_handler, period=1000): super().__init__() self.sub_stream = None self.timer = None self.recv_addr = recv_addr self.recv_title = recv_title self.state_handler = state_handler self.period = period self.callback = Callback(Publisher(send_addr, send_title), self.state_handler) def setup(self): super().setup() self.sub_stream, _ = self.stream(zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8')) self.sub_stream.on_recv(SubStreamHandler(self.sub_stream, self.stop, self.state_handler)) self.timer = PeriodicCallback(self.callback, self.period, self.loop) def run(self): self.setup() print('Start loop!') self.timer.start() self.loop.start() def stop(self): self.loop.stop()
class DeviceRep(object): """ Helper class to represent a device to a worker """ def __init__(self, device_id, state='unknown'): self.id = device_id self.state = state self.curr_liveness = CLIENT_HB_LIVENESS self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL) self.hb_timer.start() return def heartbeat(self): if self.curr_liveness > 0: self.curr_liveness -= 1 if self.curr_liveness == 0: self.state = 'dead' return def on_message_received(self): self.curr_liveness = CLIENT_HB_LIVENESS return def is_alive(self): return self.curr_liveness > 0 def get_state(self): return self.state def shutdown(self): self.hb_timer.stop() self.hb_timer = None
class AppServer(object): # listen = "127.0.0.1" listen = "*" port = 5555 def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client_identities = {} self.server = self.ctx.socket(zmq.ROUTER) bind_addr = "tcp://%s:%s" % (self.listen, self.port) print("bind_addr=", bind_addr) self.server.bind(bind_addr) print("Server listening for new client connections at", bind_addr) self.server = ZMQStream(self.server) self.server.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def periodictask(self): stale_clients = [] for client_id, last_seen in self.client_identities.items(): if last_seen + timedelta(seconds=10) < datetime.utcnow(): stale_clients.append(client_id) else: msg = HelloMessage() msg.send(self.server, client_id) for client_id in stale_clients: print( "Haven't received a HELO from cliient %s recently. Dropping from list of connected clients." % client_id) del self.client_identities[client_id] sys.stdout.write(".") sys.stdout.flush() def on_recv(self, msg): identity = msg[0] self.client_identities[identity] = datetime.utcnow() msg_type = msg[1] print("Received message of type %s from client ID %s!" % (msg_type, identity))
def main(pat): fname = find_connection_file(pat) with open(fname) as f: cfg = json.load(f) url = "%s://%s:%s" % (cfg.get('transport', 'tcp'), cfg['ip'], cfg['iopub_port']) session = Session(key=cfg['key']) ctx = zmq.Context.instance() sub = ctx.socket(zmq.SUB) sub.subscribe = b'' sub.connect(url) # import IPython # IPython.embed() # return stream = ZMQStream(sub) stream.on_recv(lambda msg_list: log_msg(session, msg_list)) pc = PeriodicCallback(print_time, 5 * 60 * 1000) pc.start() IOLoop.instance().start()
class WorkerRep(object): """Helper class to represent a worker in the broker. Instances of this class are used to track the state of the attached worker and carry the timers for incomming and outgoing heartbeats. :type wid: str :param wid: the worker id. :param service: service this worker serves :type service: str :param stream: the ZMQStream used to send messages :type stream: ZMQStream """ def __init__(self, wid, service, stream): self.id = wid self.service = service self.multicasts = [] self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return def send_hb(self): """Called on every HB_INTERVAL. Decrements the current liveness by one. Sends heartbeat to worker. """ self.curr_liveness -= 1 msg = [ self.id, b'', MDP_WORKER_VERSION, b'\x05' ] self.stream.send_multipart(msg) return def on_heartbeat(self): """Called when a heartbeat message from the worker was received. Sets current liveness to HB_LIVENESS. """ self.curr_liveness = HB_LIVENESS return def is_alive(self): """Returns True when the worker is considered alive. """ return self.curr_liveness > 0 def shutdown(self): """Cleanup worker. Stops timer. """ self.hb_out_timer.stop() self.hb_out_timer = None self.stream = None return
def run(self): _period = 1000 * self.configreader["services"][self.name]["checkmail_interval"] beat = PeriodicCallback(self.cmd_processmails, _period, io_loop=self.ioloop) beat.start() Service.run(self)
class TaskState (object): """ Tracks task state (with help of watchdog) """ log = skytools.getLogger ('d:TaskState') def __init__ (self, uid, name, info, ioloop, cc, xtx): self.uid = uid self.name = name self.info = info self.pidfile = info['config']['pidfile'] self.ioloop = ioloop self.cc = cc self.xtx = xtx self.timer = None self.timer_tick = 1 self.heartbeat = False self.start_time = None self.dead_since = None def start (self): self.start_time = time.time() self.timer = PeriodicCallback (self.watchdog, self.timer_tick * 1000, self.ioloop) self.timer.start() def stop (self): try: self.log.info ('Signalling %s', self.name) skytools.signal_pidfile (self.pidfile, signal.SIGINT) except: self.log.exception ('signal_pidfile failed: %s', self.pidfile) def watchdog (self): live = skytools.signal_pidfile (self.pidfile, 0) if live: self.log.debug ('%s is alive', self.name) if self.heartbeat: self.send_reply ('running') else: self.log.info ('%s is over', self.name) self.dead_since = time.time() self.timer.stop() self.timer = None self.send_reply ('stopped') def ccpublish (self, msg): assert isinstance (msg, TaskReplyMessage) cmsg = self.xtx.create_cmsg (msg) cmsg.send_to (self.cc) def send_reply (self, status, feedback = {}): msg = TaskReplyMessage( req = 'task.reply.%s' % self.uid, handler = self.info['task']['task_handler'], task_id = self.info['task']['task_id'], status = status, feedback = feedback) self.ccpublish (msg)
class TaskState(object): """ Tracks task state (with help of watchdog) """ log = skytools.getLogger('d:TaskState') def __init__(self, uid, name, info, ioloop, cc, xtx): self.uid = uid self.name = name self.info = info self.pidfile = info['config']['pidfile'] self.ioloop = ioloop self.cc = cc self.xtx = xtx self.timer = None self.timer_tick = 1 self.heartbeat = False self.start_time = None self.dead_since = None def start(self): self.start_time = time.time() self.timer = PeriodicCallback(self.watchdog, self.timer_tick * 1000, self.ioloop) self.timer.start() def stop(self): try: self.log.info('Signalling %s', self.name) skytools.signal_pidfile(self.pidfile, signal.SIGINT) except: self.log.exception('signal_pidfile failed: %s', self.pidfile) def watchdog(self): live = skytools.signal_pidfile(self.pidfile, 0) if live: self.log.debug('%s is alive', self.name) if self.heartbeat: self.send_reply('running') else: self.log.info('%s is over', self.name) self.dead_since = time.time() self.timer.stop() self.timer = None self.send_reply('stopped') def ccpublish(self, msg): assert isinstance(msg, TaskReplyMessage) cmsg = self.xtx.create_cmsg(msg) cmsg.send_to(self.cc) def send_reply(self, status, feedback={}): msg = TaskReplyMessage(req='task.reply.%s' % self.uid, handler=self.info['task']['task_handler'], task_id=self.info['task']['task_id'], status=status, feedback=feedback) self.ccpublish(msg)
class AppClient(object): # where to connect to - point to another IP or hostname:5556 if running server on another machine endpoint = "tcp://127.0.0.1:5556" # crypto = True means 'use CurveZMQ'. False means don't. Must match server. crypto = True def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) if self.crypto: self.keymonkey = KeyMonkey("client") self.client = self.keymonkey.setupClient(self.client, self.endpoint, "server") self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def periodictask(self): if not self.last_recv or self.last_recv + timedelta( seconds=5) < datetime.utcnow(): print( "Hmmm... haven't heard from the server in 5 seconds... Server unresponsive." ) print("Sending HELLO to server") msg = HelloMessage() msg.send(self.client) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
def start(self): loop = self.loop loop.add_handler(self.udp.handle.fileno(), self.handle_beacon, loop.READ) stream = ZMQStream(self.pipe, loop) stream.on_recv(self.control_message) pc = PeriodicCallback(self.send_ping, PING_INTERVAL * 1000, loop) pc.start() pc = PeriodicCallback(self.reap_peers, PING_INTERVAL * 1000, loop) pc.start() loop.start()
def start(self): loop = self.loop loop.add_handler(self.udp.handle.fileno(), self.handle_beacon, loop.READ) stream = ZMQStream(self.pipe, loop) stream.on_recv(self.control_message) pc = PeriodicCallback(self.send_ping, PING_INTERVAL * 1000, loop) pc.start() pc = PeriodicCallback(self.reap_peers, PING_INTERVAL * 1000, loop) pc.start() loop.start()
class AppClient(object): def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.endpoint = "tcp://127.0.0.1:5556" self.client = self.ctx.socket(zmq.DEALER) self.client.setsockopt( zmq.LINGER, 0 ) # Without linger and timeouts you might have problems when closing context self.client.setsockopt(zmq.RCVTIMEO, 5000) # 5s self.client.setsockopt(zmq.SNDTIMEO, 5000) print("Connecting to", self.endpoint) self.client.connect(self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def disconnect(self): if self.ctx is not None: try: self.periodic.stop() print("Closing socket and context") self.client.close() self.ctx.term() except Exception as e: print(e) def periodictask(self): if self.client is None: return if not self.last_recv or self.last_recv + timedelta( seconds=5) < datetime.utcnow(): print("No data from remote (5s)... [ping]") print("Sending HELLO to server") msg = HelloMessage() msg.send(self.client) def start(self): try: self.periodic.start() self.loop.start() msg = HelloMessage() msg.send(self.client) except KeyboardInterrupt: print("\n\nCtrl+C detected\n") except Exception as E: print("Error detected") print(str(E)) finally: self.disconnect() def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
class AppServer(object): def __init__(self): self.listen = "127.0.0.1" self.port = 5556 self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client_identities = {} self.server = self.ctx.socket(zmq.ROUTER) self.server.setsockopt( zmq.LINGER, 0 ) # Without linger and timeouts you might have problems when closing context self.server.setsockopt(zmq.RCVTIMEO, 5000) # 5s self.server.setsockopt(zmq.SNDTIMEO, 5000) bind_addr = "tcp://%s:%s" % (self.listen, self.port) self.server.bind(bind_addr) print("Server listening for new client connections at", bind_addr) self.server = ZMQStream(self.server) self.server.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: self.periodic.stop() print("\nClosing socket and context\n") self.server.close() self.ctx.term() def periodictask(self): stale_clients = [] for client_id, last_seen in self.client_identities.items(): if last_seen + timedelta(seconds=10) < datetime.utcnow(): stale_clients.append(client_id) else: msg = HelloMessage() msg.send(self.server, client_id) for client_id in stale_clients: print( "\nHaven't received a HELO from client %s recently. Dropping from list of connected clients." % client_id) del self.client_identities[client_id] sys.stdout.write(".") sys.stdout.flush() def on_recv(self, msg): identity = msg[ 0] # This contains client id (socket handle), use to reply it back self.client_identities[identity] = datetime.utcnow() msg_type = msg[1] print("Received message of type %s from client ID %s!" % (msg_type, identity))
def loop(): self.__ioloop = IOLoop() queue_socket = self.__context.socket(zmq.PULL) queue_socket.connect(self.message_address) queue_stream = ZMQStream(queue_socket, self.__ioloop) worker_socket = self.__context.socket(zmq.DEALER) for address in self.active_connections: worker_socket.connect(address) worker_stream = ZMQStream(worker_socket, self.__ioloop) def receive_response(message, response_override=None): self.__queued_messages.pop(message[1], None) self.__message_timeouts.pop(message[1], None) callback = self.__callbacks.pop(message[1], None) if callback: try: callback(response_override or self.loads(self.decompress(message[2]))) except Exception as e: self.log_error(e) callback({'error': e}) worker_stream.on_recv(receive_response) def queue_message(message): if message[0]: if message[0] == WORKER_SOCKET_CONNECT and message[2] not in self.active_connections: self.active_connections.add(message[2]) worker_stream.socket.connect(message[2]) elif message[0] == WORKER_SOCKET_DISCONNECT and message[2] in self.active_connections: self.active_connections.remove(message[2]) worker_stream.socket.disconnect(message[2]) return self.__queued_messages[message[1]] = (time(), message) try: worker_stream.send_multipart(message) except Exception as e: self.log_error(e) queue_stream.on_recv(queue_message) def timeout_message(): now = time() for message, retry in [(item[1], self.__message_auto_retry.get(item[1][1], self.__auto_retry)) for item, t in ((i, self.__message_timeouts.get(i[1][1], self.__timeout)) for i in self.__queued_messages.itervalues()) if t >= 0 and (item[0] + t < now)]: if retry: logging.info('Worker timeout, requeuing ' + message[1]) queue_message(message) else: receive_response(('', message[1]), {'error': 'timeout'}) timeout_callback = PeriodicCallback(timeout_message, int(abs(self.__timeout * 1000.0)), io_loop = self.__ioloop) timeout_callback.start() self.__ioloop.start() self.__thread = None
class DeviceApp(object): # endpoint = "tcp://127.0.0.1:5556" # endpoint = "tcp://localhost:5556" endpoint = "tcp://165.227.24.226:5556" def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def periodictask(self): if not (self.last_recv) or (self.last_recv + timedelta(seconds=5)) < datetime.utcnow(): print( "Hmmm... haven't heard from the server in 5 seconds... Server unresponsive." ) print("Sending HELLO to server") msg = HelloMessage() print("msg=", msg) msg.send(self.client) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
def on_run(self, ctx, io_loop, socks, streams): z = ZmqRpcProxy(self._uris['manager_rpc']) self._data = self._initial_data() self._remote_handlers = set(z.available_handlers()) env = OrderedDict([ ('ctx', ctx), ('io_loop', io_loop), ('socks', socks), ('streams', streams), ]) f = functools.partial(self.timer__monitor_heartbeats, env) callback = PeriodicCallback(f, 1000, io_loop=io_loop) callback.start() f = functools.partial(self.timer__grim_reaper, env) callback = PeriodicCallback(f, 5000, io_loop=io_loop) callback.start()
class Delay (CCHandler): """ Delays all received messages, then dispatches them to another handler. """ CC_ROLES = ['local', 'remote'] log = skytools.getLogger ('h:Delay') tick = 250 # ms def __init__ (self, hname, hcf, ccscript): super(Delay, self).__init__(hname, hcf, ccscript) self.fwd_hname = self.cf.get ('forward-to') self.delay = self.cf.getint ('delay', 0) self.fwd_handler = ccscript.get_handler (self.fwd_hname) self.queue = collections.deque() self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop) self.timer.start() def handle_msg (self, cmsg): """ Got message from client -- queue it """ self.queue.append ((time.time() + self.delay, cmsg)) def process_queue (self): now = time.time() try: while (self.queue[0][0] <= now): at, cmsg = self.queue.popleft() size = cmsg.get_size() try: self.fwd_handler.handle_msg (cmsg) stat = 'ok' except Exception: self.log.exception ('crashed, dropping msg: %s', cmsg.get_dest()) stat = 'crashed' self.stat_inc ('delay.count') self.stat_inc ('delay.bytes', size) self.stat_inc ('delay.count.%s' % stat) self.stat_inc ('delay.bytes.%s' % stat, size) except IndexError: pass
def start(self): """Initialize and start the event loop. Listen for ZMQ control messages.""" ctx = zmq.Context() socket = ctx.socket(zmq.PAIR) socket.bind("tcp://*:{}".format(settings.ZMQ_CONTROL_PORT)) logserver = LogServer() logserver.listen(settings.TCP_LOGGING_PORT) loop = self.loop stream = ZMQStream(socket, loop) stream.on_recv(self.handle_ctrl_msg) # TODO: Stop loop and restart on CONFIG reread just to get possible new # EVENT_POLL_INTERVAL setting? pc = PeriodicCallback(self.chime, self.EVENT_POLL_INTERVAL * 1E3, loop) pc.start() loop.start()
def run_sock_configs(sock_configs, on_run=None, control_pipe=None): ctx, io_loop, socks, streams = get_run_context(sock_configs) def _on_run(): on_run(ctx, io_loop, socks, streams) if on_run: io_loop.add_callback(_on_run) def watchdog(): if control_pipe.poll(): io_loop.stop() if control_pipe: callback = PeriodicCallback(watchdog, 500, io_loop=io_loop) callback.start() try: io_loop.start() except KeyboardInterrupt: pass
class _HeartbeatHelper: def __init__(self, messaging): self.messaging = messaging self.last_message_time = time.time() self._heart_beat = PeriodicCallback(self._send_state, 1000) name = self.messaging._service_name + '.messaging.heartbeat' self.logger = logging.getLogger(name) def start(self): self.logger.info(' Start Heart Beat') self._heart_beat.start() def message_recieved(self): self.last_message_time = time.time() def _send_state(self): time_now = time.time() delta_time = time_now - self.last_message_time if delta_time > 1.5: msg = create_vex_message('', self.messaging._service_name, self.messaging.uuid) self.messaging.add_callback(self.messaging.subscription_socket.send_multipart, msg) self.last_message_time = time_now
def start(root): global loop, path, checkup_periodic, control_port path = root print('Starting.') # add HOSTS to /etc/hosts add_hosts(HOSTS) # define context ctx = zmq.Context() # create ioloop loop = zmq.eventloop.ioloop.IOLoop() # connect req to mongrel2 control port c = ctx.socket(zmq.REQ) c.connect(M2_CONTROL_PORT) control_port = ZMQStream(c, io_loop=loop) # define 'checkup' interval checkup_periodic = PeriodicCallback(send_checkup, CHECKUP_INTERVAL, io_loop=loop) # load mongrel2 config load_mongrel() # kill PID if server didn't get shut down at close of last run kill_mongrel_with_pid(M2_PID_PATH) # start mongrel2 with m2sh start_mongrel() # start the loop checkup_periodic.start() loop.start()
class Receive: def __init__(self): self.stop = False self.conx = Context.instance() self.socket = self.conx.socket(zmq.ROUTER) self.socket.bind('tcp://*:5555') self.periodic = PeriodicCallback(self.timer, 4000) self.periodic.start() def prt(self, message): print('id=', message[0]) print('data=', message[1]) if (message[1] == b'exit'): self.stop = True async def recv(self): while (self.stop == False): msg = await self.socket.recv_multipart() self.prt(msg) self.socket.close() async def timer(self): print('OMG asynchronicity!')
class BufferModule(base.ZmqProcess): def __init__(self, recv_addr, send_addr, recv_title, send_title, state_handler, period=1000): super().__init__() self.sub_stream = None self.timer = None self.recv_addr = recv_addr self.recv_title = recv_title self.state_handler = state_handler self.period = period self.callback = Callback(Publisher(send_addr, send_title), self.state_handler) def setup(self): super().setup() self.sub_stream, _ = self.stream( zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8')) self.sub_stream.on_recv( SubStreamHandler(self.sub_stream, self.stop, self.state_handler)) self.timer = PeriodicCallback(self.callback, self.period, self.loop) def run(self): self.setup() print('Start loop!') self.timer.start() self.loop.start() def stop(self): self.loop.stop()
def loop(): self.__ioloop = IOLoop() queue_socket = self.__context.socket(zmq.PULL) queue_socket.connect(self.message_address) queue_stream = ZMQStream(queue_socket, self.__ioloop) worker_socket = self.__context.socket(zmq.DEALER) worker_socket.connect(self.address) worker_stream = ZMQStream(worker_socket, self.__ioloop) def receive_response(message): self.__queued_messages.pop(message[1], None) self.__message_timeouts.pop(message[1], None) callback = self.__callbacks.pop(message[1], None) if callback: try: callback(self.loads(self.decompress(message[2]))) except Exception as e: self.log_error(e) worker_stream.on_recv(receive_response) def queue_message(message): self.__queued_messages[message[1]] = (time() * 1000, message) try: worker_stream.send_multipart(message) except Exception as e: self.log_error(e) queue_stream.on_recv(queue_message) def requeue_message(): now = time() * 1000 for message in (item[1] for item in self.__queued_messages.itervalues() if item[0] + self.__message_timeouts.get(item[1][1], self.__retry_ms) < now): queue_message(message) requeue_callback = PeriodicCallback(requeue_message, self.__retry_ms, io_loop = self.__ioloop) requeue_callback.start() self.__ioloop.start() self.__thread = None
class DemoApp(ZMQProcess): def __init__ (self): super(DemoApp, self).__init__() def setup(self): super(DemoApp, self).setup() self.pub, self.pub_addr = self.stream(zmq.PUB, 'tcp://127.0.0.1:%(port)s', True) self.sub, sub_addr = self.stream(zmq.SUB, self.pub_addr, False, callback=DemoHandler()) self.heartbeat = PeriodicCallback(self.ping, 1000, self.loop) def ping(self): print 'SEND PING' self.pub.send_multipart(['ping', json.dumps(['ping', time.time()])]) def local_run(self): print 'START HEARTBEAT' self.heartbeat.start() def stop(self): self.heartbeat.stop() self.loop.stop()
def run(self): consumer = Process(target=Consumer(self.uris['backend_rep'], self.uris['consumer_push_be'], self.uris['consumer_pull_be']).run ) producer = Process(target=self.producer_class( self.uris['frontend_rep_uri'], self.uris['frontend_pub_uri'], self.uris['consumer_pull_be'], self.uris['consumer_push_be']).run ) self.io_loop = IOLoop() periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop) periodic_callback.start() try: consumer.start() producer.start() self.io_loop.start() except KeyboardInterrupt: pass producer.terminate() consumer.terminate() logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have ' 'been terminated')
def run(self): self.received_count = 0 ctx, io_loop, self.socks, streams = get_run_context(self.sock_configs) self.io_loop = io_loop if self.request_delay > 0: periodic_callback = PeriodicCallback(self.do_request, self.request_delay * 1000., io_loop) io_loop.add_timeout(timedelta(seconds=0.1), lambda: periodic_callback.start()) else: for i in range(self.target_count): io_loop.add_callback(self.do_request) try: io_loop.start() except KeyboardInterrupt: pass
class Broker(object): """This is implementation of broker You don't need to override any methods in this class. It works immediately. Just call start_listening() method :type context: Context :param context: instance of zmq.Context :param endpoint: listening address :type endpoint: str """ def __init__(self, context, endpoint): socket = context.socket(zmq.ROUTER) socket.bind(endpoint) self.stream = ZMQStream(socket) self.stream.on_recv(self.on_message) # services, workers and multicast groups self._workers = {} self._services = {} self._multicasts = {} self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() return def start_listening(self): """Start listening to new messages """ IOLoop.instance().start() def stop_listening(self): """Stop listening """ IOLoop.instance().stop() def on_message(self, msg): """Processes given message. Decides what kind of message it is -- client or worker -- and calls the appropriate method. If unknown, the message is ignored. :param msg: message parts :type msg: list of str :rtype: None """ return_addresses, msg = self.split_address(msg) # dispatch on first frame after path method_to_call = None try: t = msg.pop(0) if t.startswith(b'MDPW'): method_to_call = self.on_worker elif t.startswith(b'MDPC'): method_to_call = self.on_client else: # Unknown protocol pass except (AttributeError, IndexError): # Wrong incoming msg format pass if method_to_call is not None: method_to_call(return_addresses, msg) return def on_client(self, return_addresses, message): """Method called on client message. Frame 0 of msg is the command id Frame 1 of msg is the requested service. The remaining frames are the request to forward to the worker. .. note:: If the service is unknown to the broker the message is ignored. If a worker is available for the requested service, the message is repackaged and sent to the worker. If the service name starts with `mmi.`, the message is passed to the internal MMI_ handler. .. _MMI: http://rfc.zeromq.org/spec:8 If the service name starts with `multicast.`, the message is sent to all worker in that group. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ cmd = message.pop(0) # always 0x01 service = message.pop(0) # mmi requests if service.startswith(b'mmi.'): self.on_client_mmi(return_addresses, service, message) return # multicast requests if service.startswith(b'multicast.'): self.on_client_multicast(return_addresses, service, message) return # worker requests try: available_workers = self._services[service] random_worker = choice(available_workers) # TODO: loadbalancing to_send = [random_worker.id, b'', MDP_WORKER_VERSION, b'\x02'] to_send.extend(return_addresses) to_send.append(b'') to_send.extend(message) self.stream.send_multipart(to_send) except KeyError: # unknwon service self.client_response(return_addresses, b'broker', b'No worker available', error=True) return def on_client_multicast(self, return_addresses, service, message): """Handling multicast messages from client :param return_addresses: return address stack :type return_addresses: list of str :param service: name of mmi service :type service: str :param message: message parts :type message: list of str """ target = service[10:] # remove 'multicast.' try: # first, prepare list of workers in target multicast grouped_by_names = {} for worker in self._multicasts[target]: if worker.service in grouped_by_names: grouped_by_names[worker.service].append(worker) else: grouped_by_names[worker.service] = [worker] # send message to one worker per service sent_messages = [] for name, workers in grouped_by_names.items(): random_worker = choice(workers) # TODO: loadbalancing to_send = [random_worker.id, b'', MDP_WORKER_VERSION, b'\x02'] to_send.extend(return_addresses) to_send.append(b'') to_send.extend(message) self.stream.send_multipart(to_send) sent_messages.append(random_worker.service) # notify client with list of services in multicast group client_msg = return_addresses[:] client_msg.extend([b'', MDP_WORKER_VERSION, b'\x05']) client_msg.extend(sent_messages) self.stream.send_multipart(client_msg) except KeyError: # unknwon service self.client_response(return_addresses, b'broker', b'No services available in this multicast', error=True) return def on_client_mmi(self, return_addresses, service, message): """Handling MMI messages from client :param return_addresses: return address stack :type return_addresses: list of str :param service: name of mmi service :type service: str :param message: message parts :type message: list of str """ if service == b'mmi.service': return self.on_client_mmi_service(return_addresses, service, message) elif service == b'mmi.services': return self.on_client_mmi_services(return_addresses, service, message) elif service == b'mmi.workers': return self.on_client_mmi_workers(return_addresses, service, message) elif service == b'mmi.multicasts': return self.on_client_mmi_multicasts(return_addresses, service, message) else: # unknown mmi service - notify client self.client_response(return_addresses, b'broker', b'Service not found', error=True) def on_client_mmi_service(self, return_addresses, service, message): """Check if services exists """ return self.client_response_pack(return_addresses, b'broker', message[0] in self._services.keys()) def on_client_mmi_services(self, return_addresses, service, message): """List of all services """ return self.client_response_pack(return_addresses, b'broker', [k for k in self._services]) def on_client_mmi_workers(self, return_addresses, service, message): """Number of workers per service """ s = {} for se in self._services: s[se] = len(self._services[se]) return self.client_response_pack(return_addresses, b'broker', s) def on_client_mmi_multicasts(self, return_addresses, service, message): """List of available multicast groups """ m = {} for se in self._multicasts: m[se] = [s.service for s in self._multicasts[se]] return self.client_response_pack(return_addresses, b'broker', m) def on_worker(self, return_addresses, message): """Method called on worker message. Frame 0 of msg is the command id. The remaining frames depend on the command. This method determines the command sent by the worker and calls the appropriate method. If the command is unknown the message is ignored and a DISCONNECT is sent. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ cmd = message.pop(0) worker_cmds = { b'\x01': self.on_worker_ready, b'\x03': self.on_worker_partial_reply, b'\x04': self.on_worker_final_reply, b'\x05': self.on_worker_heartbeat, b'\x06': self.on_worker_disconnect, b'\x07': self.on_worker_multicast_add, # this is not part of the Majordomo Protocol 0.2 ! b'\x08': self.on_worker_exception, # this is not part of the Majordomo Protocol 0.2 ! b'\x09': self.on_worker_error, # this is not part of the Majordomo Protocol 0.2 ! } if cmd in worker_cmds: fnc = worker_cmds[cmd] fnc(return_addresses, message) else: # ignore unknown command pass return def on_worker_ready(self, return_addresses, message): """Called when new worker is ready to receive messages. Register worker to list of available services. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ # Frame 0 of msg is a service. service = message.pop(0) wid = return_addresses[0] return self.register_worker(wid, service) def on_worker_partial_reply(self, return_addresses, message): """Process worker PARTIAL REPLY command. Route the `message` to the client given by the address(es) in front of `message`. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ ret_id = return_addresses[0] try: wrep = self._workers[ret_id] except KeyError: return # worker is gone, ignore this message return_addresses, msg = self.split_address(message) self.client_response(return_addresses, wrep.service, msg, partial=True) return def on_worker_final_reply(self, return_addresses, message): """Process worker FINAL REPLY command. Route the `message` to the client given by the address(es) in front of `message`. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ ret_id = return_addresses[0] try: wrep = self._workers[ret_id] except KeyError: return # worker is gone, ignore this message return_addresses, msg = self.split_address(message) self.client_response(return_addresses, wrep.service, msg) return def on_worker_heartbeat(self, return_addresses, message): """Process worker HEARTBEAT command. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ ret_id = return_addresses[0] try: worker = self._workers[ret_id] if worker.is_alive(): worker.on_heartbeat() except KeyError: # ignore HB for unknown worker pass return def on_worker_disconnect(self, return_addresses, message): """Process worker DISCONNECT command. Remove worker from list of services. :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ wid = return_addresses[0] return self.unregister_worker(wid) def on_worker_multicast_add(self, return_addresses, message): """Process worker MULTICAST ADD command. Add worker to list of multicasts This is not part of the Majordomo Protocol 0.2 ! :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ multicast_name = message.pop(0) wid = return_addresses[0] return self.register_multicast(wid, multicast_name) def on_worker_exception(self, return_addresses, message): """Process worker EXCEPTION command. Route the `message` to the client given by the address(es) in front of `message`. This is not part of the Majordomo Protocol 0.2 ! :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ ret_id = return_addresses[0] try: wrep = self._workers[ret_id] except KeyError: return # worker is gone, ignore this message return_addresses, msg = self.split_address(message) self.client_response(return_addresses, wrep.service, msg, exception=True) return def on_worker_error(self, return_addresses, message): """Process worker ERROR command. Route the `message` to the client given by the address(es) in front of `message`. This is not part of the Majordomo Protocol 0.2 ! :param return_addresses: return address stack :type return_addresses: list of str :param message: message parts :type message: list of str :rtype: None """ ret_id = return_addresses[0] try: wrep = self._workers[ret_id] except KeyError: return # worker is gone, ignore this message return_addresses, msg = self.split_address(message) self.client_response(return_addresses, wrep.service, msg, error=True) return def on_timer(self): """Method called on timer expiry. Checks which workers are dead and unregisters them. :rtype: None """ for wrep in self._workers.values(): if not wrep.is_alive(): self.on_log_event('worker.connection_timeout', "Worker connection timeout for service '%s'." % wrep.service) self.unregister_worker(wrep.id) return def client_response(self, return_addresses, service, msg, partial=False, exception=False, error=False): """Package and send reply to client. :param return_addresses: return address stack :type return_addresses: list of str :param service: name of service :type service: str :param msg: message parts :type msg: list of str | str :rtype: None """ to_send = return_addresses[:] if error: t = b'\x04' elif exception: t = b'\x06' elif partial: t = b'\x02' else: t = b'\x03' to_send.extend([b'', MDP_WORKER_VERSION, t, service]) if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def client_response_pack(self, return_addresses, service, msg, partial=False): """Send message to client and pack it (msg) in msgpack format Exception and error messages are not allowed here. :param return_addresses: return address stack :type return_addresses: list of str :param service: name of service :type service: str :param msg: message to pack and send :type msg: mixed :param partial: if message is partial of final, default False :type partial: bool :rtype: None """ packed = msgpack.Packer().pack(msg) self.client_response(return_addresses, service, packed, partial=partial) def unregister_worker(self, wid): """Unregister the worker with the given id. If the worker id is not registered, nothing happens. Will stop all timers for the worker. :param wid: the worker id. :type wid: str :rtype: None """ try: wrep = self._workers[wid] except KeyError: # not registered, ignore return wrep.shutdown() service = wrep.service # remove worker from service list if service in self._services: worker_list = self._services[service] for worker in worker_list: if worker.id == wid: worker_list.remove(worker) if not worker_list: del self._services[service] # remove worker from multicasts for m_name in self._multicasts: mw = self._multicasts[m_name] for w in [w for w in mw if w.id == wid]: mw.remove(w) # delete empty rows empty_keys = [k for k, v in self._multicasts.items() if len(v) == 0] for k in empty_keys: del self._multicasts[k] del self._workers[wid] self.on_log_event('worker.unregister', "Worker for service '%s' disconnected." % service) return def register_worker(self, wid, service): """Register the worker id and add it to the given service. Does nothing if worker is already known. :param wid: the worker id. :type wid: str :param service: the service name. :type service: str :rtype: None """ if wid in self._workers: return worker = WorkerRep(wid, service, self.stream) self._workers[wid] = worker if service in self._services: s = self._services[service] s.append(worker) else: self._services[service] = [worker] self.on_log_event('worker.register', "Worker for service '%s' is connected." % service) return def register_multicast(self, wid, multicast_name): """Add worker to multicast group :type wid: str :param wid: the worker id. :type multicast_name: str :param multicast_name: group name """ if wid not in self._workers: return worker = self._workers[wid] if multicast_name in self._multicasts: m = self._multicasts[multicast_name] m.append(worker) else: self._multicasts[multicast_name] = [worker] self.on_log_event('worker.register_multicast', "Service '%s' added to multicast group '%s'." % (worker.service, multicast_name)) return def shutdown(self): """Shutdown broker. Will unregister all workers, stop all timers and ignore all further messages. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ self.stream.on_recv(None) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.socket.close() self.stream.close() self.stream = None self._workers = {} self._services = {} self._multicasts = {} return def on_log_event(self, event, message): """Override this method if you want to log events from broker :type event: str :param event: event type - used for filtering :type message: str :param message: log message :rtype: None """ pass # helpers: def split_address(self, msg): """Function to split return Id and message received by ROUTER socket. Returns 2-tuple with return Id and remaining message parts. Empty frames after the Id are stripped. """ ret_ids = [] for i, p in enumerate(msg): if p: ret_ids.append(p) else: break return ret_ids, msg[i + 1:]
class LogWatch_HandleStats(PgLogForwardPlugin): LOG_FORMATS = ['netstr'] def init(self, log_fmt): super(LogWatch_HandleStats, self).init(log_fmt) # depends on pg_settings.log_function_calls self.parse_statements = self.cf.getbool('parse_statements', True) self.msg_suffix = self.cf.get('msg-suffix', 'confdb') if self.msg_suffix and not is_msg_req_valid(self.msg_suffix): self.log.error("invalid msg-suffix: %s", self.msg_suffix) self.msg_suffix = None self.hostname = socket.gethostname() self.stat_queue_name = self.cf.get('stat_queue_name', '') self.max_stat_items = self.cf.get('max_stat_items', 10000) self.stat_dump_interval = self.cf.getint('stat_interval', 3600) self.last_stat_dump = time.time() self.client_stats = {} self.timer = PeriodicCallback(self.save_stats, self.stat_dump_interval * 1000) self.timer.start() def process_netstr(self, data): """ Process contents of collected log chunk. This might be a SQL statement or a connect/disconnect entry. """ if not self.stat_queue_name: return if data['remotehost'] == "[local]": data['remotehost'] = "127.0.0.1" action = None action_duration = 0 statement_duration = 0 call_count = 0 if data['message'].startswith("connection authorized:"): action = "connect" elif data['message'].startswith("disconnection"): action = "disconnect" m = rc_disconnect.match(data['message']) if m: action_duration = (int(m.group('hours')) * 3600 + int(m.group('minutes')) * 60 + float(m.group('seconds'))) * 1000 elif not self.parse_statements: # we have function logging enabled, see if we can use it m = rc_logged_func.search(data['message']) if m: # a logged function call, definitely prefer this to parsing action = m.group('func_name') action_duration = float(m.group('time')) / 1000 call_count = int(m.group('calls')) if not action: # we have to parse function call m = rc_sql.search(data['message']) if m: if self.parse_statements: # attempt to parse the function name and parameters #action = self.get_sql_action (m.group('sql')) call_count = 1 # count the overall statement duration action_duration = float(m.group('duration')) statement_duration = action_duration self._update_stats(data, action, action_duration, call_count) self._update_stats(data, "SQL statements", statement_duration, call_count) def _update_stats(self, data, action, duration, call_count): if action: key = (data['database'], data['username'], data['remotehost'], action) cs = self.client_stats.get(key) if cs: cs.update(duration, call_count) elif len(self.client_stats) > self.max_stat_items: self.log.error("Max stat items exceeded: %i", self.max_stat_items) else: cs = ClientStats(data['database'], data['username'], data['remotehost'], action, duration, call_count) self.client_stats[key] = cs def save_stats(self): """ Dump client stats to database. Scheduled to be called periodically. """ # do not send stats if stats is missing or stats queue is missing if not self.client_stats or not self.stat_queue_name: return now = time.time() time_passed = now - self.last_stat_dump self.log.info("Sending usage stats to repository [%i]", len(self.client_stats)) # post role usage usage = [] for client in self.client_stats.values(): self.log.trace("client: %s", client) usage.append(client.to_dict()) params = skytools.db_urlencode( dict(hostname=self.hostname, sample_length='%d seconds' % time_passed, snap_time=datetime.datetime.now().isoformat())) confdb_funcargs = ('username=discovery', params, skytools.make_record_array(usage)) funcargs = [ None, self.stat_queue_name, 'dba.set_role_usage', skytools.db_urlencode(dict(enumerate(confdb_funcargs))) ] msg = DatabaseMessage(function='pgq.insert_event', params=cc.json.dumps(funcargs)) if self.msg_suffix: msg.req += '.' + self.msg_suffix self.main.ccpublish(msg) self.client_stats = {} self.last_stat_dump = now def stop(self): self.timer.stop()
class Shell(PromptSession): def __init__(self, history_filepath=None): self._logger = logging.getLogger(__name__) if history_filepath is None: self._logger.info(' getting default history filepath.') history_filepath = _get_default_history_filepath() self._logger.info(' history filepath %s', history_filepath) self.history = FileHistory(history_filepath) self.messaging = _Messaging('shell', run_control_loop=True) # TODO: Cleanup this API access self.messaging._heartbeat_reciever.identity_callback = self._identity_callback self._thread = _Thread(target=self.messaging.start, daemon=True) self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state, 1000) self.shebangs = [ '!', ] self.command_observer = CommandObserver(self.messaging, prompt=self) commands = self.command_observer._get_commands() self._word_completer = WordCompleter(commands, WORD=_WORD) self._services_completer = ServiceCompleter(self._word_completer) self.author_interface = AuthorInterface(self._word_completer, self.messaging) self.service_interface = ServiceInterface(self._word_completer, self.messaging) self.entity_interface = EntityInterface(self.author_interface, self.service_interface) self._bot_status = '' super().__init__(message='vexbot: ', history=self.history, completer=self._services_completer, enable_system_prompt=True, enable_suspend=True, enable_open_in_editor=True, complete_while_typing=False) self.print_observer = PrintObserver(self.app) self._print_subscription = self.messaging.chatter.subscribe( self.print_observer) self.messaging.chatter.subscribe(LogObserver()) self.messaging.command.subscribe(self.command_observer) self.messaging.command.subscribe( ServicesObserver(self._identity_setter, self._set_service_completion)) def _identity_setter(self, services: list) -> None: try: services.remove(self.messaging._service_name) except ValueError: pass for service in services: self.service_interface.add_service(service) self.messaging.send_command('REMOTE', remote_command='commands', target=service, suppress=True) def _set_service_completion(self, service: str, commands: list) -> None: shebang = self.shebangs[0] commands = [shebang + command for command in commands] completer = WordCompleter(commands, WORD=_WORD) self._services_completer.set_service_completer(service, completer) # FIXME: `!commands` does not update these as it if service == 'vexbot': for command in commands: self._word_completer.words.add(command) def _identity_callback(self): self.messaging.send_command('services', suppress=True) self.messaging.send_command('get_commands', suppress=True) def _monitor_bot_state(self): time_now = time.time() last_message = self.messaging._heartbeat_reciever._last_message_time # TODO: put in a countdown since last contact? delta_time = time_now - last_message # NOTE: Bot is set to send a heartbeat every 1.5 seconds if time_now - last_message > 3.4: self._bot_status = '<NO BOT>' else: if self._bot_status == '': return self._bot_status = '' self.app.invalidate() def is_command(self, text: str) -> bool: """ checks for presence of shebang in the first character of the text """ if text[0] in self.shebangs: return True return False def run(self): self._thread.start() self._bot_status_monitor.start() with patch_stdout(raw=True): while True: # Get our text try: text = self.prompt(rprompt=self._get_rprompt_tokens) # KeyboardInterrupt continues except KeyboardInterrupt: continue # End of file returns except EOFError: return # Clean text text = text.lstrip() # Handle empty text if text == '': self._logger.debug(' empty string found') continue # Program specific handeling. Currently either first word # or second word can be commands for string in text.split('&&'): self._handle_text(string) def _get_rprompt_tokens(self): return self._bot_status def _handle_text(self, text: str): """ Check to see if text is a command. Otherwise, check to see if the second word is a command. Commands get handeled by the `_handle_command` method If not command, check to see if first word is a service or an author. Default program is to send message replies. However, we will also check to see if the second word is a command and handle approparitly This method does simple string parsing and high level program control """ # If first word is command if self.is_command(text): self._logger.debug(' first word is a command') # get the command, args, and kwargs out using `shlex` command, args, kwargs = _get_cmd_args_kwargs(text) self._logger.info(' command: %s, %s %s', command, args, kwargs) # hand off to the `handle_command` method result = self._handle_command(command, args, kwargs) if result: if isinstance(result, str): print(result) else: _pprint.pprint(result) # Exit the method here if in this block return # Else first word is not a command else: self._logger.debug(' first word is not a command') # get the first word and then the rest of the text. try: first_word, second_word = text.split(' ', 1) self._logger.debug(' first word: %s', first_word) except ValueError: self._logger.debug('No second word in chain!') return self._handle_NLP(text) # check if second word/string is a command if self.is_command(second_word): self._logger.info(' second word is a command') # get the command, args, and kwargs out using `shlex` command, args, kwargs = _get_cmd_args_kwargs(second_word) self._logger.debug(' second word: %s', command) self._logger.debug(' command %s', command) self._logger.debug('args %s ', args) self._logger.debug('kwargs %s', kwargs) return self._first_word_not_cmd(first_word, command, args, kwargs) # if second word is not a command, default to NLP else: self._logger.info(' defaulting to message since second word ' 'isn\'t a command') return self._handle_NLP(text) def _handle_NLP(self, text: str): entities = self.entity_interface.get_entities(text) self.messaging.send_command('NLP', text=text, entities=entities) def _handle_command(self, command: str, args: tuple, kwargs: dict): if kwargs.get('remote', False): self._logger.debug(' `remote` in kwargs, sending command') self.messaging.send_command(command, *args, **kwargs) return if self.command_observer.is_command(command): self._logger.debug(' `%s` is a command', command) # try: return self.command_observer.handle_command( command, *args, **kwargs) """ except Exception as e: self.command_observer.on_error(e, command, args, kwargs) return """ else: self._logger.debug('command not found! Sending to bot') self.messaging.send_command(command, *args, **kwargs) def _first_word_not_cmd(self, first_word: str, command: str, args: tuple, kwargs: dict) -> None: """ check to see if this is an author or service. This method does high level control handling """ if self.service_interface.is_service(first_word): self._logger.debug(' first word is a service') kwargs = self.service_interface.get_metadata(first_word, kwargs) self._logger.debug(' service transform kwargs: %s', kwargs) elif self.author_interface.is_author(first_word): self._logger.debug(' first word is an author') kwargs = self.author_interface.get_metadata(first_word, kwargs) self._logger.debug(' author transform kwargs: %s', kwargs) if not kwargs.get('remote'): kwargs['remote_command'] = command command = 'REMOTE' self.messaging.send_command(command, *args, **kwargs) return else: self.messaging.send_command(command, *args, **kwargs)
class BinaryStar(object): ctx = None # Our private context loop = None # Reactor loop statepub = None # State publisher statesub = None # State subscriber state = None # Current state event = None # Current event peer_expiry = 0 # When peer is considered 'dead' voter_callback = None # Voting socket handler master_callback = None # Call when become master slave_callback = None # Call when become slave heartbeat = None # PeriodicCallback for def __init__(self, primary, local, remote): # initialize the Binary Star self.ctx = zmq.Context() self.loop = IOLoop.instance() self.state = STATE_PRIMARY if primary else STATE_BACKUP # Create publisher for state going to peer self.statepub = self.ctx.socket(zmq.PUB) self.statepub.bind(local) # Create subscriber for state coming from peer self.statesub = self.ctx.socket(zmq.SUB) self.statesub.setsockopt(zmq.SUBSCRIBE, '') self.statesub.connect(remote) # wrap statesub in ZMQStream for event triggers self.statesub = ZMQStream(self.statesub, self.loop) # setup basic reactor events self.heartbeat = PeriodicCallback(self.send_state, HEARTBEAT, self.loop) self.statesub.on_recv(self.recv_state) # setup log formmater def update_peer_expiry(self): """Update peer expiry time to be 2 heartbeats from now.""" self.peer_expiry = time.time() + 2e-3 * HEARTBEAT def start(self): self.update_peer_expiry() self.heartbeat.start() return self.loop.start() def execute_fsm(self): """Binary Star finite state machine (applies event to state) returns True if connections should be accepted, False otherwise. """ accept = True if (self.state == STATE_PRIMARY): # Primary server is waiting for peer to connect # Accepts CLIENT_REQUEST events in this state if (self.event == PEER_BACKUP): print ("I: connected to backup (slave), ready as master") self.state = STATE_ACTIVE if (self.master_callback): self.loop.add_callback(self.master_callback) elif (self.event == PEER_ACTIVE): print ("I: connected to backup (master), ready as slave") self.state = STATE_PASSIVE if (self.slave_callback): self.loop.add_callback(self.slave_callback) elif (self.event == CLIENT_REQUEST): if (time.time () >= self.peer_expiry): print ("I: request from client, ready as master") self.state = STATE_ACTIVE if (self.master_callback): self.loop.add_callback(self.master_callback) else: # don't respond to clients yet - we don't know if # the backup is currently Active as a result of # a successful failover accept = False elif (self.state == STATE_BACKUP): # Backup server is waiting for peer to connect # Rejects CLIENT_REQUEST events in this state if (self.event == PEER_ACTIVE): print ("I: connected to primary (master), ready as slave") self.state = STATE_PASSIVE if (self.slave_callback): self.loop.add_callback(self.slave_callback) elif (self.event == CLIENT_REQUEST): accept = False elif (self.state == STATE_ACTIVE): # Server is active # Accepts CLIENT_REQUEST events in this state # The only way out of ACTIVE is death if (self.event == PEER_ACTIVE): # Two masters would mean split-brain print ("E: fatal error - dual masters, aborting") raise FSMError("Dual Masters") elif (self.state == STATE_PASSIVE): # Server is passive # CLIENT_REQUEST events can trigger failover if peer looks dead if (self.event == PEER_PRIMARY): # Peer is restarting - become active, peer will go passive print ("I: primary (slave) is restarting, ready as master") self.state = STATE_ACTIVE elif (self.event == PEER_BACKUP): # Peer is restarting - become active, peer will go passive print ("I: backup (slave) is restarting, ready as master") self.state = STATE_ACTIVE elif (self.event == PEER_PASSIVE): # Two passives would mean cluster would be non-responsive print ("E: fatal error - dual slaves, aborting") raise FSMError("Dual slaves") elif (self.event == CLIENT_REQUEST): # Peer becomes master if timeout has passed # It's the client request that triggers the failover assert (self.peer_expiry > 0) if (time.time () >= self.peer_expiry): # If peer is dead, switch to the active state print ("I: failover successful, ready as master") self.state = STATE_ACTIVE else: # If peer is alive, reject connections accept = False # Call state change handler if necessary if (self.state == STATE_ACTIVE and self.master_callback): self.loop.add_callback(self.master_callback) return accept # --------------------------------------------------------------------- # Reactor event handlers... def send_state (self): """Publish our state to peer""" self.statepub.send("%d" % self.state) def recv_state (self, msg): """Receive state from peer, execute finite state machine""" state = msg[0] if state: self.event = int(state) self.update_peer_expiry() self.execute_fsm() def voter_ready(self, msg): """Application wants to speak to us, see if it's possible""" # If server can accept input now, call appl handler self.event = CLIENT_REQUEST if self.execute_fsm(): print "CLIENT REQUEST" self.voter_callback(self.voter_socket, msg) else: # Message will be ignored pass # ------------------------------------------------------------------------- # def register_voter(self, endpoint, type, handler): """Create socket, bind to local endpoint, and register as reader for voting. The socket will only be available if the Binary Star state machine allows it. Input on the socket will act as a "vote" in the Binary Star scheme. We require exactly one voter per bstar instance. handler will always be called with two arguments: (socket,msg) where socket is the one we are creating here, and msg is the message that triggered the POLLIN event. """ assert self.voter_callback is None socket = self.ctx.socket(type) socket.bind(endpoint) self.voter_socket = socket self.voter_callback = handler stream = ZMQStream(socket, self.loop) stream.on_recv(self.voter_ready)
class CryptoTransportLayer(TransportLayer): def __init__(self, ob_ctx, db): self.ob_ctx = ob_ctx self.log = logging.getLogger( '[%s] %s' % (ob_ctx.market_id, self.__class__.__name__) ) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) self.db = db self.bitmessage_api = None if (ob_ctx.bm_user, ob_ctx.bm_pass, ob_ctx.bm_port) != (None, None, -1): if not self._connect_to_bitmessage(): self.log.info('Bitmessage not installed or started') self.market_id = ob_ctx.market_id self.nick_mapping = {} self.uri = network_util.get_peer_url(ob_ctx.server_ip, ob_ctx.server_port) self.ip = ob_ctx.server_ip self.nickname = "" self.dev_mode = ob_ctx.dev_mode self.all_messages = ( 'hello', 'findNode', 'findNodeResponse', 'store' ) self._setup_settings() ob_ctx.market_id = self.market_id self.dht = DHT(self, self.market_id, self.settings, self.db) TransportLayer.__init__(self, ob_ctx, self.guid, self.nickname) self.start_listener() if ob_ctx.enable_ip_checker and not ob_ctx.seed_mode and not ob_ctx.dev_mode: self.start_ip_address_checker() def start_listener(self): self.add_callbacks([ ( msg, { 'cb': getattr(self, 'on_%s' % msg), 'validator_cb': getattr(self, 'validate_on_%s' % msg) } ) for msg in self.all_messages ]) self.listener = connection.CryptoPeerListener( self.ip, self.port, self.pubkey, self.secret, self.ctx, self.guid, self._on_message ) self.listener.set_ok_msg({ 'type': 'ok', 'senderGUID': self.guid, 'pubkey': self.pubkey, 'senderNick': self.nickname }) self.listener.listen() def start_ip_address_checker(self): '''Checks for possible public IP change''' if self.ob_ctx.enable_ip_checker: self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() self.log.info("IP_CHECKER_ENABLED: Periodic IP Address Checker started.") def _ip_updater_periodic_callback(self): if self.ob_ctx.enable_ip_checker: new_ip = network_util.get_my_ip() if not new_ip or new_ip == self.ip: return self.ob_ctx.server_ip = new_ip self.ip = new_ip if self.listener is not None: self.listener.set_ip_address(new_ip) self.dht._iterativeFind(self.guid, [], 'findNode') def save_peer_to_db(self, peer_tuple): uri = peer_tuple[0] pubkey = peer_tuple[1] guid = peer_tuple[2] nickname = peer_tuple[3] # Update query self.db.deleteEntries("peers", {"uri": uri, "guid": guid}, "OR") if guid is not None: self.db.insertEntry("peers", { "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname, "market_id": self.market_id }) def _connect_to_bitmessage(self): # Get bitmessage going # First, try to find a local instance result = False bm_user = self.ob_ctx.bm_user bm_pass = self.ob_ctx.bm_pass bm_port = self.ob_ctx.bm_port try: self.log.info( '[_connect_to_bitmessage] Connecting to Bitmessage on port %s', bm_port ) self.bitmessage_api = xmlrpclib.ServerProxy( "http://{}:{}@localhost:{}/".format(bm_user, bm_pass, bm_port), verbose=0 ) result = self.bitmessage_api.add(2, 3) self.log.info( "[_connect_to_bitmessage] Bitmessage API is live: %s", result ) # If we failed, fall back to starting our own except Exception as e: self.log.info("Failed to connect to bitmessage instance: %s", e) self.bitmessage_api = None return result def validate_on_hello(self, msg): self.log.debug('Validating ping message.') return True def on_hello(self, msg): self.log.info('Pinged %s', json.dumps(msg, ensure_ascii=False)) def validate_on_store(self, msg): self.log.debug('Validating store value message.') return True def on_store(self, msg): self.dht._on_storeValue(msg) def validate_on_findNode(self, msg): self.log.debug('Validating find node message.') return True def on_findNode(self, msg): self.dht.on_find_node(msg) def validate_on_findNodeResponse(self, msg): self.log.debug('Validating find node response message.') return True def on_findNodeResponse(self, msg): self.dht.on_findNodeResponse(self, msg) def _setup_settings(self): try: self.settings = self.db.selectEntries("settings", {"market_id": self.market_id}) except (OperationalError, DatabaseError) as e: print e raise SystemExit("database file %s corrupt or empty - cannot continue" % self.db.db_path) if len(self.settings) == 0: self.settings = {"market_id": self.market_id, "welcome": "enable"} self.db.insertEntry("settings", self.settings) else: self.settings = self.settings[0] # Generate PGP key during initial setup or if previous PGP gen failed if not self.settings.get('PGPPubKey'): try: self.log.info('Generating PGP keypair. This may take several minutes...') print 'Generating PGP keypair. This may take several minutes...' gpg = gnupg.GPG() input_data = gpg.gen_key_input(key_type="RSA", key_length=2048, name_email='*****@*****.**', name_comment="Autogenerated by Open Bazaar", passphrase="P@ssw0rd") assert input_data is not None key = gpg.gen_key(input_data) assert key is not None pubkey_text = gpg.export_keys(key.fingerprint) newsettings = {"PGPPubKey": pubkey_text, "PGPPubkeyFingerprint": key.fingerprint} self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) self.log.info('PGP keypair generated.') except Exception as e: sys.exit("Encountered a problem with GPG: %s" % e) if not self.settings.get('pubkey'): # Generate Bitcoin keypair self._generate_new_keypair() if not self.settings.get('nickname'): newsettings = {'nickname': 'Default'} self.db.updateEntries('settings', newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) self.nickname = self.settings.get('nickname', '') self.secret = self.settings.get('secret', '') self.pubkey = self.settings.get('pubkey', '') self.privkey = self.settings.get('privkey') self.btc_pubkey = privkey_to_pubkey(self.privkey) self.guid = self.settings.get('guid', '') self.sin = self.settings.get('sin', '') self.bitmessage = self.settings.get('bitmessage', '') if not self.settings.get('bitmessage'): # Generate Bitmessage address if self.bitmessage_api is not None: self._generate_new_bitmessage_address() self.cryptor = Cryptor(pubkey_hex=self.pubkey, privkey_hex=self.secret) # In case user wants to override with command line passed bitmessage values if self.ob_ctx.bm_user is not None and \ self.ob_ctx.bm_pass is not None and \ self.ob_ctx.bm_port is not None: self._connect_to_bitmessage() def _generate_new_keypair(self): secret = str(random.randrange(2 ** 256)) self.secret = hashlib.sha256(secret).hexdigest() self.pubkey = privkey_to_pubkey(self.secret) self.privkey = random_key() self.btc_pubkey = privkey_to_pubkey(self.privkey) print 'PUBLIC KEY: ', self.btc_pubkey # Generate SIN sha_hash = hashlib.sha256() sha_hash.update(self.pubkey) ripe_hash = hashlib.new('ripemd160') ripe_hash.update(sha_hash.digest()) self.guid = ripe_hash.hexdigest() self.sin = obelisk.EncodeBase58Check('\x0F\x02%s' % ripe_hash.digest()) newsettings = { "secret": self.secret, "pubkey": self.pubkey, "privkey": self.privkey, "guid": self.guid, "sin": self.sin } self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) def _generate_new_bitmessage_address(self): # Use the guid generated previously as the key self.bitmessage = self.bitmessage_api.createRandomAddress( self.guid.encode('base64'), False, 1.05, 1.1111 ) newsettings = {"bitmessage": self.bitmessage} self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) def join_network(self, seeds=None, callback=None): if seeds is None: seeds = [] self.log.info('Joining network') # Connect up through seed servers for idx, seed in enumerate(seeds): seeds[idx] = network_util.get_peer_url(seed, "12345") # Connect to persisted peers db_peers = self.get_past_peers() known_peers = list(set(seeds).union(db_peers)) for known_peer in known_peers: self.dht.add_peer(self, known_peer) # Populate routing table by searching for self if known_peers: # Check every one second if we are connected # We could use a PeriodicCallback but I think this is simpler # since this will be repeated in most cases less than 10 times def join_callback(): # If we are not connected to any node, reschedule a check if not self.dht.activePeers: ioloop.IOLoop.instance().call_later(1, join_callback) else: self.search_for_my_node() join_callback() if callback is not None: callback('Joined') def get_past_peers(self): result = self.db.selectEntries("peers", {"market_id": self.market_id}) return [peer['uri'] for peer in result] def search_for_my_node(self): self.log.info('Searching for myself') self.dht._iterativeFind(self.guid, self.dht.knownNodes, 'findNode') def get_crypto_peer(self, guid=None, uri=None, pubkey=None, nickname=None): if guid == self.guid: self.log.error('Cannot get CryptoPeerConnection for your own node') return self.log.debug( 'Getting CryptoPeerConnection' '\nGUID: %s' '\nURI: %s' '\nPubkey:%s' '\nNickname:%s', guid, uri, pubkey, nickname ) return connection.CryptoPeerConnection( self, uri, pubkey, guid=guid, nickname=nickname ) def respond_pubkey_if_mine(self, nickname, ident_pubkey): if ident_pubkey != self.pubkey: self.log.info("Public key does not match your identity") return # Return signed pubkey pubkey = self.cryptor.pubkey # XXX: A Cryptor does not have such a field. ec_key = obelisk.EllipticCurveKey() ec_key.set_secret(self.secret) digest = obelisk.Hash(pubkey) signature = ec_key.sign(digest) # Send array of nickname, pubkey, signature to transport layer self.send(proto_response_pubkey(nickname, pubkey, signature)) def send(self, data, send_to=None, callback=None): self.log.debug("Outgoing Data: %s %s", data, send_to) # Directed message if send_to is not None: peer = self.dht.routingTable.getContact(send_to) if peer is None: for activePeer in self.dht.activePeers: if activePeer.guid == send_to: peer = activePeer break if peer: self.log.debug('Directed Data (%s): %s', send_to, data) try: peer.send(data, callback=callback) except Exception as e: self.log.error('Not sending message directly to peer %s', e) else: self.log.error('No peer found') else: # FindKey and then send for peer in self.dht.activePeers: try: routing_peer = self.dht.routingTable.getContact(peer.guid) if routing_peer is None: self.dht.routingTable.addContact(peer) routing_peer = peer data['senderGUID'] = self.guid data['pubkey'] = self.pubkey def cb(msg): self.log.debug('Message Back: \n%s', pformat(msg)) routing_peer.send(data, cb) except Exception: self.log.info("Error sending over peer!") traceback.print_exc() def _on_message(self, msg): # here goes the application callbacks # we get a "clean" msg which is a dict holding whatever pubkey = msg.get('pubkey') uri = msg.get('uri') guid = msg.get('senderGUID') nickname = msg.get('senderNick')[:120] self.log.info('On Message: %s', json.dumps(msg, ensure_ascii=False)) self.dht.add_peer(self, uri, pubkey, guid, nickname) t = Thread(target=self.trigger_callbacks, args=(msg['type'], msg,)) t.start() def store(self, *args, **kwargs): """ Store or republish data. Refer to the dht module (iterativeStore()) for further details. """ self.dht.iterativeStore(*args, **kwargs) def shutdown(self): print "CryptoTransportLayer.shutdown()!" print "Notice: explicit DHT Shutdown not implemented." try: if self.bitmessage_api is not None: self.bitmessage_api.close() except Exception as e: # It might not even be open; we can't do much more on our # way out if exception is thrown here. self.log.error( "Could not shutdown bitmessage_api's ServerProxy: %s", e.message )
class ProxyHandler(BaseProxyHandler): """ Simply proxies further """ log = skytools.getLogger('h:ProxyHandler') ping_tick = 1 def __init__(self, hname, hcf, ccscript): super(ProxyHandler, self).__init__(hname, hcf, ccscript) self.ping_remote = self.cf.getbool("ping", False) if self.ping_remote: self.echo_stats = EchoState(self.remote_url) self.echo_timer = PeriodicCallback(self.ping, self.ping_tick * 1000, self.ioloop) self.echo_timer.start() self.log.debug("will ping %s", self.remote_url) def on_recv(self, zmsg): """ Got message from remote CC, process it. """ try: # pongs to our pings should come back w/o any routing info if self.ping_remote and zmsg[0] == '': self.log.trace("%r", zmsg) cmsg = CCMessage(zmsg) req = cmsg.get_dest() if req == "echo.response": self._recv_pong(cmsg) else: self.log.warn("unknown msg: %s", req) except: self.log.exception("crashed") finally: super(ProxyHandler, self).on_recv(zmsg) def _recv_pong(self, cmsg): """ Pong received, evaluate it. """ msg = cmsg.get_payload(self.xtx) if not msg: return if msg.orig_target != self.remote_url: self.log.warn("unknown pong: %s", msg.orig_target) return echo = self.echo_stats echo.update_pong(msg) rtt = echo.time_pong - msg.orig_time if msg.orig_time == echo.time_ping: self.log.trace("echo time: %f s (%s)", rtt, self.remote_url) elif rtt <= 5 * self.ping_tick: self.log.debug("late pong: %f s (%s)", rtt, self.remote_url) else: self.log.info("too late pong: %f s (%s)", rtt, self.remote_url) def _send_ping(self): """ Send ping to remote CC. """ msg = EchoRequestMessage(target=self.remote_url) cmsg = self.xtx.create_cmsg(msg) self.stream.send_cmsg(cmsg) self.echo_stats.update_ping(msg) self.log.trace("%r", msg) def ping(self): """ Echo requesting and monitoring. """ self.log.trace("") echo = self.echo_stats if echo.time_ping - echo.time_pong > 5 * self.ping_tick: self.log.warn("no pong from %s for %f s", self.remote_url, echo.time_ping - echo.time_pong) self._send_ping() def stop(self): super(ProxyHandler, self).stop() self.log.info("stopping") if hasattr(self, "echo_timer"): self.echo_timer.stop()
class CryptoTransportLayer(TransportLayer): def __init__(self, ob_ctx, db): self.ob_ctx = ob_ctx self.log = logging.getLogger( '[%s] %s' % (ob_ctx.market_id, self.__class__.__name__)) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) self.db = db self.bitmessage_api = None if (ob_ctx.bm_user, ob_ctx.bm_pass, ob_ctx.bm_port) != (None, None, -1): if not self._connect_to_bitmessage(): self.log.info('Bitmessage not installed or started') self.market_id = ob_ctx.market_id self.nick_mapping = {} self.uri = network_util.get_peer_url(ob_ctx.server_ip, ob_ctx.server_port) self.ip = ob_ctx.server_ip self.nickname = "" self.dev_mode = ob_ctx.dev_mode self.all_messages = ('hello', 'findNode', 'findNodeResponse', 'store') self._setup_settings() ob_ctx.market_id = self.market_id self.dht = DHT(self, self.market_id, self.settings, self.db) TransportLayer.__init__(self, ob_ctx, self.guid, self.nickname) self.start_listener() if ob_ctx.enable_ip_checker and not ob_ctx.seed_mode and not ob_ctx.dev_mode: self.start_ip_address_checker() def start_listener(self): self.add_callbacks([(msg, { 'cb': getattr(self, 'on_%s' % msg), 'validator_cb': getattr(self, 'validate_on_%s' % msg) }) for msg in self.all_messages]) self.listener = connection.CryptoPeerListener(self.ip, self.port, self.pubkey, self.secret, self.ctx, self.guid, self._on_message) self.listener.set_ok_msg({ 'type': 'ok', 'senderGUID': self.guid, 'pubkey': self.pubkey, 'senderNick': self.nickname }) self.listener.listen() def start_ip_address_checker(self): '''Checks for possible public IP change''' if self.ob_ctx.enable_ip_checker: self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() self.log.info( "IP_CHECKER_ENABLED: Periodic IP Address Checker started.") def _ip_updater_periodic_callback(self): if self.ob_ctx.enable_ip_checker: new_ip = network_util.get_my_ip() if not new_ip or new_ip == self.ip: return self.ob_ctx.server_ip = new_ip self.ip = new_ip if self.listener is not None: self.listener.set_ip_address(new_ip) self.dht._iterativeFind(self.guid, [], 'findNode') def save_peer_to_db(self, peer_tuple): uri = peer_tuple[0] pubkey = peer_tuple[1] guid = peer_tuple[2] nickname = peer_tuple[3] # Update query self.db.deleteEntries("peers", {"uri": uri, "guid": guid}, "OR") if guid is not None: self.db.insertEntry( "peers", { "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname, "market_id": self.market_id }) def _connect_to_bitmessage(self): # Get bitmessage going # First, try to find a local instance result = False bm_user = self.ob_ctx.bm_user bm_pass = self.ob_ctx.bm_pass bm_port = self.ob_ctx.bm_port try: self.log.info( '[_connect_to_bitmessage] Connecting to Bitmessage on port %s', bm_port) self.bitmessage_api = xmlrpclib.ServerProxy( "http://{}:{}@localhost:{}/".format(bm_user, bm_pass, bm_port), verbose=0) result = self.bitmessage_api.add(2, 3) self.log.info( "[_connect_to_bitmessage] Bitmessage API is live: %s", result) # If we failed, fall back to starting our own except Exception as e: self.log.info("Failed to connect to bitmessage instance: %s", e) self.bitmessage_api = None return result def validate_on_hello(self, msg): self.log.debug('Validating ping message.') return True def on_hello(self, msg): self.log.info('Pinged %s', json.dumps(msg, ensure_ascii=False)) def validate_on_store(self, msg): self.log.debug('Validating store value message.') return True def on_store(self, msg): self.dht._on_storeValue(msg) def validate_on_findNode(self, msg): self.log.debug('Validating find node message.') return True def on_findNode(self, msg): self.dht.on_find_node(msg) def validate_on_findNodeResponse(self, msg): self.log.debug('Validating find node response message.') return True def on_findNodeResponse(self, msg): self.dht.on_findNodeResponse(self, msg) def _setup_settings(self): try: self.settings = self.db.selectEntries( "settings", {"market_id": self.market_id}) except (OperationalError, DatabaseError) as e: print e raise SystemExit( "database file %s corrupt or empty - cannot continue" % self.db.db_path) if len(self.settings) == 0: self.settings = {"market_id": self.market_id, "welcome": "enable"} self.db.insertEntry("settings", self.settings) else: self.settings = self.settings[0] # Generate PGP key during initial setup or if previous PGP gen failed if not self.settings.get('PGPPubKey'): try: self.log.info( 'Generating PGP keypair. This may take several minutes...') print 'Generating PGP keypair. This may take several minutes...' gpg = gnupg.GPG() input_data = gpg.gen_key_input( key_type="RSA", key_length=2048, name_email='*****@*****.**', name_comment="Autogenerated by Open Bazaar", passphrase="P@ssw0rd") assert input_data is not None key = gpg.gen_key(input_data) assert key is not None pubkey_text = gpg.export_keys(key.fingerprint) newsettings = { "PGPPubKey": pubkey_text, "PGPPubkeyFingerprint": key.fingerprint } self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) self.log.info('PGP keypair generated.') except Exception as e: sys.exit("Encountered a problem with GPG: %s" % e) if not self.settings.get('pubkey'): # Generate Bitcoin keypair self._generate_new_keypair() if not self.settings.get('nickname'): newsettings = {'nickname': 'Default'} self.db.updateEntries('settings', newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) self.nickname = self.settings.get('nickname', '') self.secret = self.settings.get('secret', '') self.pubkey = self.settings.get('pubkey', '') self.privkey = self.settings.get('privkey') self.btc_pubkey = privkey_to_pubkey(self.privkey) self.guid = self.settings.get('guid', '') self.sin = self.settings.get('sin', '') self.bitmessage = self.settings.get('bitmessage', '') if not self.settings.get('bitmessage'): # Generate Bitmessage address if self.bitmessage_api is not None: self._generate_new_bitmessage_address() self.cryptor = Cryptor(pubkey_hex=self.pubkey, privkey_hex=self.secret) # In case user wants to override with command line passed bitmessage values if self.ob_ctx.bm_user is not None and \ self.ob_ctx.bm_pass is not None and \ self.ob_ctx.bm_port is not None: self._connect_to_bitmessage() def _generate_new_keypair(self): secret = str(random.randrange(2**256)) self.secret = hashlib.sha256(secret).hexdigest() self.pubkey = privkey_to_pubkey(self.secret) self.privkey = random_key() self.btc_pubkey = privkey_to_pubkey(self.privkey) print 'PUBLIC KEY: ', self.btc_pubkey # Generate SIN sha_hash = hashlib.sha256() sha_hash.update(self.pubkey) ripe_hash = hashlib.new('ripemd160') ripe_hash.update(sha_hash.digest()) self.guid = ripe_hash.hexdigest() self.sin = obelisk.EncodeBase58Check('\x0F\x02%s' % ripe_hash.digest()) newsettings = { "secret": self.secret, "pubkey": self.pubkey, "privkey": self.privkey, "guid": self.guid, "sin": self.sin } self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) def _generate_new_bitmessage_address(self): # Use the guid generated previously as the key self.bitmessage = self.bitmessage_api.createRandomAddress( self.guid.encode('base64'), False, 1.05, 1.1111) newsettings = {"bitmessage": self.bitmessage} self.db.updateEntries("settings", newsettings, {"market_id": self.market_id}) self.settings.update(newsettings) def join_network(self, seeds=None, callback=None): if seeds is None: seeds = [] self.log.info('Joining network') # Connect up through seed servers for idx, seed in enumerate(seeds): seeds[idx] = network_util.get_peer_url(seed, "12345") # Connect to persisted peers db_peers = self.get_past_peers() known_peers = list(set(seeds).union(db_peers)) for known_peer in known_peers: self.dht.add_peer(self, known_peer) # Populate routing table by searching for self if known_peers: # Check every one second if we are connected # We could use a PeriodicCallback but I think this is simpler # since this will be repeated in most cases less than 10 times def join_callback(): # If we are not connected to any node, reschedule a check if not self.dht.activePeers: ioloop.IOLoop.instance().call_later(1, join_callback) else: self.search_for_my_node() join_callback() if callback is not None: callback('Joined') def get_past_peers(self): result = self.db.selectEntries("peers", {"market_id": self.market_id}) return [peer['uri'] for peer in result] def search_for_my_node(self): self.log.info('Searching for myself') self.dht._iterativeFind(self.guid, self.dht.knownNodes, 'findNode') def get_crypto_peer(self, guid=None, uri=None, pubkey=None, nickname=None): if guid == self.guid: self.log.error('Cannot get CryptoPeerConnection for your own node') return self.log.debug( 'Getting CryptoPeerConnection' '\nGUID: %s' '\nURI: %s' '\nPubkey:%s' '\nNickname:%s', guid, uri, pubkey, nickname) return connection.CryptoPeerConnection(self, uri, pubkey, guid=guid, nickname=nickname) def respond_pubkey_if_mine(self, nickname, ident_pubkey): if ident_pubkey != self.pubkey: self.log.info("Public key does not match your identity") return # Return signed pubkey pubkey = self.cryptor.pubkey # XXX: A Cryptor does not have such a field. ec_key = obelisk.EllipticCurveKey() ec_key.set_secret(self.secret) digest = obelisk.Hash(pubkey) signature = ec_key.sign(digest) # Send array of nickname, pubkey, signature to transport layer self.send(proto_response_pubkey(nickname, pubkey, signature)) def send(self, data, send_to=None, callback=None): self.log.debug("Outgoing Data: %s %s", data, send_to) # Directed message if send_to is not None: peer = self.dht.routingTable.getContact(send_to) if peer is None: for activePeer in self.dht.activePeers: if activePeer.guid == send_to: peer = activePeer break if peer: self.log.debug('Directed Data (%s): %s', send_to, data) try: peer.send(data, callback=callback) except Exception as e: self.log.error('Not sending message directly to peer %s', e) else: self.log.error('No peer found') else: # FindKey and then send for peer in self.dht.activePeers: try: routing_peer = self.dht.routingTable.getContact(peer.guid) if routing_peer is None: self.dht.routingTable.addContact(peer) routing_peer = peer data['senderGUID'] = self.guid data['pubkey'] = self.pubkey def cb(msg): self.log.debug('Message Back: \n%s', pformat(msg)) routing_peer.send(data, cb) except Exception: self.log.info("Error sending over peer!") traceback.print_exc() def _on_message(self, msg): # here goes the application callbacks # we get a "clean" msg which is a dict holding whatever pubkey = msg.get('pubkey') uri = msg.get('uri') guid = msg.get('senderGUID') nickname = msg.get('senderNick')[:120] self.log.info('On Message: %s', json.dumps(msg, ensure_ascii=False)) self.dht.add_peer(self, uri, pubkey, guid, nickname) t = Thread(target=self.trigger_callbacks, args=( msg['type'], msg, )) t.start() def store(self, *args, **kwargs): """ Store or republish data. Refer to the dht module (iterativeStore()) for further details. """ self.dht.iterativeStore(*args, **kwargs) def shutdown(self): print "CryptoTransportLayer.shutdown()!" print "Notice: explicit DHT Shutdown not implemented." try: if self.bitmessage_api is not None: self.bitmessage_api.close() except Exception as e: # It might not even be open; we can't do much more on our # way out if exception is thrown here. self.log.error( "Could not shutdown bitmessage_api's ServerProxy: %s", e.message)
class MDPBroker(object): """The MDP broker class. The broker routes messages from clients to appropriate workers based on the requested service. This base class defines the overall functionality and the API. Subclasses are ment to implement additional features (like logging). The broker uses ØMQ ROUTER sockets to deal witch clients and workers. These sockets are wrapped in pyzmq streams to fit well into IOLoop. .. note:: The workers will *always* be served by the `main_ep` endpoint. In a two-endpoint setup clients will be handled via the `opt_ep` endpoint. :param context: the context to use for socket creation. :type context: zmq.Context :param main_ep: the primary endpoint for workers and clients. :type main_ep: str :param opt_ep: is an optional 2nd endpoint. :type opt_ep: str :param worker_q: the class to be used for the worker-queue. :type worker_q: class """ CLIENT_PROTO = b'MDPC01' #: Client protocol identifier WORKER_PROTO = b'MDPW01' #: Worker protocol identifier def __init__(self, context, main_ep, opt_ep=None): """Init MDPBroker instance. """ l = logger.Logger('mq_broker') self.log = l.get_logger() self.log.info("MDP broker startup...") socket = ZmqSocket(context, zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = ZmqSocket(context, zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self.log.debug("Socket created...") self._workers = {} # services contain the worker queue and the request queue self._services = {} self._worker_cmds = { b'\x01': self.on_ready, b'\x03': self.on_reply, b'\x04': self.on_heartbeat, b'\x05': self.on_disconnect, } self.log.debug("Launch the timer...") self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() self.log.info("MDP broker started") return def register_worker(self, wid, service): """Register the worker id and add it to the given service. Does nothing if worker is already known. :param wid: the worker id. :type wid: str :param service: the service name. :type service: str :rtype: None """ self.log.debug( "Try to register a worker : wid={0}, service={1}".format( wid, service)) try: if wid in self._workers: self.log.debug("Worker %s already registered" % service) return self._workers[wid] = WorkerRep(self.WORKER_PROTO, wid, service, self.main_stream) if service in self._services: wq, wr = self._services[service] wq.put(wid) else: q = ServiceQueue() q.put(wid) self._services[service] = (q, []) self.log.info("Registered worker : wid={0}, service={1}".format( wid, service)) except: self.log.error( "Error while registering a worker : wid={0}, service={1}, trace={2}" .format(wid, service, traceback.format_exc())) return def unregister_worker(self, wid): """Unregister the worker with the given id. If the worker id is not registered, nothing happens. Will stop all timers for the worker. :param wid: the worker id. :type wid: str :rtype: None """ self.log.debug("Try to unregister a worker : wid={0}".format(wid)) try: try: wrep = self._workers[wid] except KeyError: # not registered, ignore self.log.warning( "The worker wid={0} is not registered, ignoring the unregister request" .format(wid)) return wrep.shutdown() service = wrep.service if service in self._services: wq, wr = self._services[service] wq.remove(wid) del self._workers[wid] self.log.info("Unregistered worker : wid={0}".format(wid)) except: self.log.error( "Error while unregistering a worker : wid={0}, trace={1}". format(wid, traceback.format_exc())) return def disconnect(self, wid): """Send disconnect command and unregister worker. If the worker id is not registered, nothing happens. :param wid: the worker id. :type wid: str :rtype: None """ self.log.debug("Try to disconnect a worker : wid={0}".format(wid)) try: try: wrep = self._workers[wid] except KeyError: # not registered, ignore self.log.warning( "The worker wid={0} service={1} is not registered, ignoring the disconnect request" .format(wid, wrep.service)) return to_send = [wid, self.WORKER_PROTO, b'\x05'] self.main_stream.send_multipart(to_send) self.log.info( "Request to unregister a worker : wid={0} service={1}".format( wid, wrep.service)) except: self.log.error( "Error while disconnecting a worker : wid={0}, trace={1}". format(wid, traceback.format_exc())) self.unregister_worker(wid) return def client_response(self, rp, service, msg): """Package and send reply to client. :param rp: return address stack :type rp: list of str :param service: name of service :type service: str :param msg: message parts :type msg: list of str :rtype: None """ to_send = rp[:] to_send.extend([b'', self.CLIENT_PROTO, service]) to_send.extend(msg) self.client_stream.send_multipart(to_send) return def shutdown(self): """Shutdown broker. Will unregister all workers, stop all timers and ignore all further messages. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ self.log.info("Shutdown starting...") try: self.log.debug("Closing the socket...") if self.client_stream == self.main_stream: self.client_stream = None self.main_stream.on_recv(None) self.main_stream.socket.setsockopt(zmq.LINGER, 0) self.main_stream.socket.close() self.main_stream.close() self.main_stream = None if self.client_stream: self.client_stream.on_recv(None) self.client_stream.socket.setsockopt(zmq.LINGER, 0) self.client_stream.socket.close() self.client_stream.close() self.client_stream = None self.log.debug("Clean workers and services...") self._workers = {} self._services = {} except: self.log.error("Error during shutdown : trace={0}".format( traceback.format_exc())) return def on_timer(self): """Method called on timer expiry. Checks which workers are dead and unregisters them. :rtype: None """ self.log.debug("Check for dead workers...") for wrep in list(self._workers.values()): if not wrep.is_alive(): self.log.info( "A worker seems to be dead : wid={0} service={1}".format( wrep.id, wrep.service)) self.unregister_worker(wrep.id) return def on_ready(self, rp, msg): """Process worker READY command. Registers the worker for a service. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] self.register_worker(ret_id, msg[0]) return def on_reply(self, rp, msg): """Process worker REPLY command. Route the `msg` to the client given by the address(es) in front of `msg`. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] # make worker available again try: wrep = self._workers[ret_id] service = wrep.service wq, wr = self._services[service] cp, msg = split_address(msg) self.client_response(cp, service, msg) wq.put(wrep.id) if wr: proto, rp, msg = wr.pop(0) self.on_client(proto, rp, msg) except KeyError: # unknown service self.disconnect(ret_id) return def on_heartbeat(self, rp, msg): """Process worker HEARTBEAT command. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] try: worker = self._workers[ret_id] if worker.is_alive(): worker.on_heartbeat() except KeyError: # ignore HB for unknown worker pass return def on_disconnect(self, rp, msg): """Process worker DISCONNECT command. Unregisters the worker who sent this message. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ wid = rp[0] self.log.info("A worker disconnects itself : wid={0}".format(wid)) self.unregister_worker(wid) return def on_mmi(self, rp, service, msg): """Process MMI request. For now only mmi.service is handled. :param rp: return address stack :type rp: list of str :param service: the protocol id sent :type service: str :param msg: message parts :type msg: list of str :rtype: None """ if service == b'mmi.service': s = msg[0] ret = b'404' for wr in list(self._workers.values()): if s == wr.service: ret = b'200' break self.client_response(rp, service, [ret]) elif service == b'mmi.services': ret = [] for wr in list(self._workers.values()): ret.append(wr.service) self.client_response(rp, service, [b', '.join(ret)]) else: self.client_response(rp, service, [b'501']) return def on_client(self, proto, rp, msg): """Method called on client message. Frame 0 of msg is the requested service. The remaining frames are the request to forward to the worker. .. note:: If the service is unknown to the broker the message is ignored. .. note:: If currently no worker is available for a known service, the message is queued for later delivery. If a worker is available for the requested service, the message is repackaged and sent to the worker. The worker in question is removed from the pool of available workers. If the service name starts with `mmi.`, the message is passed to the internal MMI_ handler. .. _MMI: http://rfc.zeromq.org/spec:8 :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ service = msg.pop(0) if service.startswith(b'mmi.'): self.on_mmi(rp, service, msg) return try: wq, wr = self._services[service] wid = wq.get() if not wid: # no worker ready # queue message msg.insert(0, service) wr.append((proto, rp, msg)) return wrep = self._workers[wid] to_send = [wrep.id, b'', self.WORKER_PROTO, b'\x02'] to_send.extend(rp) to_send.append(b'') to_send.extend(msg) self.main_stream.send_multipart(to_send) except KeyError: # unknwon service # ignore request msg = "broker has no service {0}".format(service) print(msg) self.log.warning(msg) return def on_worker(self, proto, rp, msg): """Method called on worker message. Frame 0 of msg is the command id. The remaining frames depend on the command. This method determines the command sent by the worker and calls the appropriate method. If the command is unknown the message is ignored and a DISCONNECT is sent. :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ cmd = msg.pop(0) if cmd in self._worker_cmds: fnc = self._worker_cmds[cmd] fnc(rp, msg) else: # ignore unknown command # DISCONNECT worker self.log.warning( "Unknown command from worker (it will be disconnect) : wid={0}, cmd={1}" .format(rp[0], cmd)) self.disconnect(rp[0]) return def on_message(self, msg): """Processes given message. Decides what kind of message it is -- client or worker -- and calls the appropriate method. If unknown, the message is ignored. :param msg: message parts :type msg: list of str :rtype: None """ self.log.debug("Message received: {0}".format(msg)) rp, msg = split_address(msg) # dispatch on first frame after path t = msg.pop(0) if t.startswith(b'MDPW'): self.on_worker(t, rp, msg) elif t.startswith(b'MDPC'): self.on_client(t, rp, msg) else: self.log.warning("Broker unknown Protocol: {0}".format(t)) return
class MDPBroker(object): """The MDP broker class. The broker routes messages from clients to appropriate workers based on the requested service. This base class defines the overall functionality and the API. Subclasses are meant to implement additional features (like logging). The broker uses ZMQ ROUTER sockets to deal with clients and workers. These sockets are wrapped in pyzmq streams to fit well into IOLoop. .. note:: The workers will *always* be served by the `main_ep` endpoint. In a two-endpoint setup clients will be handled via the `opt_ep` endpoint. :param context: the context to use for socket creation. :type context: zmq.Context :param main_ep: the primary endpoint for workers. :type main_ep: str :param client_ep: the clients endpoint :type client_ep: str :param hb_ep: the heart beat endpoint for workers. :type hb_ep: str :param service_q: the class to be used for the service worker-queue. :type service_q: class """ CLIENT_PROTO = C_CLIENT #: Client protocol identifier WORKER_PROTO = W_WORKER #: Worker protocol identifier def __init__(self, context, main_ep, client_ep, hb_ep, service_q=None): """Init MDPBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q # # Setup the zmq sockets. # socket = context.socket(zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(client_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(hb_ep) self.hb_stream = ZMQStream(socket) self.hb_stream.on_recv(self.on_message) self._workers = {} # # services contain the service queue and the request queue # self._services = {} # # Mapping of worker commands and callbacks. # self._worker_cmds = { W_READY: self.on_ready, W_REPLY: self.on_reply, W_HEARTBEAT: self.on_heartbeat, W_DISCONNECT: self.on_disconnect, } # # 'Cleanup' timer for workers without heartbeat. # self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() def register_worker(self, wid, service): """Register the worker id and add it to the given service. Does nothing if worker is already known. :param wid: the worker id. :type wid: str :param service: the service name. :type service: str :rtype: None """ if wid in self._workers: logging.info('Worker {} already registered'.format(service)) return logging.info('Registering new worker {}'.format(service)) self._workers[wid] = WorkerRep(self.WORKER_PROTO, wid, service, self.main_stream) if service in self._services: wq, wr = self._services[service] wq.put(wid) else: q = self.service_q() q.put(wid) self._services[service] = (q, []) def unregister_worker(self, wid): """Unregister the worker with the given id. If the worker id is not registered, nothing happens. Will stop all timers for the worker. :param wid: the worker id. :type wid: str :rtype: None """ try: wrep = self._workers[wid] except KeyError: # # Not registered, ignore # return logging.info('Unregistering worker {}'.format(wrep.service)) wrep.shutdown() service = wrep.service if service in self._services: wq, wr = self._services[service] wq.remove(wid) del self._workers[wid] def disconnect(self, wid): """Send disconnect command and unregister worker. If the worker id is not registered, nothing happens. :param wid: the worker id. :type wid: str :rtype: None """ try: wrep = self._workers[wid] except KeyError: # # Not registered, ignore # return logging.info('Disconnecting worker {}'.format(wrep.service)) to_send = [wid, self.WORKER_PROTO, W_DISCONNECT] self.main_stream.send_multipart(to_send) self.unregister_worker(wid) def client_response(self, rp, service, msg): """Package and send reply to client. :param rp: return address stack :type rp: list of str :param service: name of service :type service: str :param msg: message parts :type msg: list of str :rtype: None """ if service == MMI_SERVICE: logging.debug('Send reply to client from worker {}'.format(service)) else: logging.info('Send reply to client from worker {}'.format(service)) to_send = rp[:] to_send.extend([EMPTY_FRAME, self.CLIENT_PROTO, service]) to_send.extend(msg) self.client_stream.send_multipart(to_send) def shutdown(self): """Shutdown broker. Will unregister all workers, stop all timers and ignore all further messages. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ logging.debug('Shutting down') self.main_stream.on_recv(None) self.main_stream.socket.setsockopt(zmq.LINGER, 0) self.main_stream.socket.close() self.main_stream.close() self.main_stream = None self.client_stream.on_recv(None) self.client_stream.socket.setsockopt(zmq.LINGER, 0) self.client_stream.socket.close() self.client_stream.close() self.client_stream = None self.hb_stream.on_recv(None) self.hb_stream.socket.setsockopt(zmq.LINGER, 0) self.hb_stream.socket.close() self.hb_stream.close() self.hb_stream = None self._workers = {} self._services = {} def on_timer(self): """Method called on timer expiry. Checks which workers are dead and unregisters them. :rtype: None """ # # Remove 'dead' (not responding to heartbeats) workers. # for wrep in self._workers.values(): if not wrep.is_alive(): self.unregister_worker(wrep.id) def on_ready(self, rp, msg): """Process worker READY command. Registers the worker for a service. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] logging.debug('Worker sent ready msg: {} ,{}'.format(rp, msg)) self.register_worker(ret_id, msg[0]) def on_reply(self, rp, msg): """Process worker REPLY command. Route the `msg` to the client given by the address(es) in front of `msg`. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] wrep = self._workers.get(ret_id) if not wrep: # # worker not found, ignore message # logging.error( "Worker with return id {} not found. Ignore message.".format( ret_id)) return service = wrep.service logging.info("Worker {} sent reply.".format(service)) try: wq, wr = self._services[service] # # Send response to client # cp, msg = split_address(msg) self.client_response(cp, service, msg) # # make worker available again # wq.put(wrep.id) if wr: logging.info("Sending queued message to worker {}".format(service)) proto, rp, msg = wr.pop(0) self.on_client(proto, rp, msg) except KeyError: # # unknown service # self.disconnect(ret_id) def on_heartbeat(self, rp, msg): """Process worker HEARTBEAT command. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ # # Note: # The modified heartbeat of the worker is sent over a separate socket # stream (self.hb_stream). Therefore the ret_id is wrong. Instead the # worker sends its id in the message. # if len(msg) > 0: ret_id = msg[0] else: ret_id = rp[0] try: worker = self._workers[ret_id] if worker.is_alive(): worker.on_heartbeat() except KeyError: # # Ignore HB for unknown worker # pass def on_disconnect(self, rp, msg): """Process worker DISCONNECT command. Unregisters the worker who sent this message. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ wid = rp[0] self.unregister_worker(wid) def on_mmi(self, rp, service, msg): """Process MMI request. mmi.service is used for querying if a specific service is available. mmi.services is used for querying the list of services available. :param rp: return address stack :type rp: list of str :param service: the protocol id sent :type service: str :param msg: message parts :type msg: list of str :rtype: None """ if service == MMI_SERVICE: s = msg[0] ret = [UNKNOWN_SERVICE] for wr in self._workers.values(): if s == wr.service: ret = [KNOWN_SERVICE] break elif service == MMI_SERVICES: # # Return list of services # ret = [wr.service for wr in self._workers.values()] elif service == MMI_TUNNELS: # # Read the tunnel files, and send back the network info. # tunnel_paths = glob.glob(os.path.expanduser("~/tunnel_port_*.txt")) tunnels_data = {} for path in tunnel_paths: filename = os.path.split(path)[-1] service_name = filename[-7:-4] with open(path, 'r') as f: tunnels_data[service_name] = json.load(f) ret = [cPickle.dumps(tunnels_data)] else: # # Unknown command. # ret = [UNKNOWN_COMMAND] self.client_response(rp, service, ret) def on_client(self, proto, rp, msg): """Method called on client message. Frame 0 of msg is the requested service. The remaining frames are the request to forward to the worker. .. note:: If the service is unknown to the broker the message is ignored. .. note:: If currently no worker is available for a known service, the message is queued for later delivery. If a worker is available for the requested service, the message is repackaged and sent to the worker. The worker in question is removed from the pool of available workers. If the service name starts with `mmi.`, the message is passed to the internal MMI_ handler. .. _MMI: http://rfc.zeromq.org/spec:8 :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ service = msg.pop(0) if service.startswith(b'mmi.'): logging.debug("Got MMI message from client.") self.on_mmi(rp, service, msg) return logging.info("Client sends message (possibly queued) to worker {}".format(service)) try: wq, wr = self._services[service] wid = wq.get() if not wid: # # No worker ready. Queue message # logging.info("Worker {} missing. Queuing message.".format(service)) msg.insert(0, service) wr.append((proto, rp, msg)) return wrep = self._workers[wid] to_send = [wrep.id, EMPTY_FRAME, self.WORKER_PROTO, W_REQUEST] to_send.extend(rp) to_send.append(EMPTY_FRAME) to_send.extend(msg) self.main_stream.send_multipart(to_send) except KeyError: # # Unknwon service. Ignore request # logging.info('broker has no service "{}"'.format(service)) def on_worker(self, proto, rp, msg): """Method called on worker message. Frame 0 of msg is the command id. The remaining frames depend on the command. This method determines the command sent by the worker and calls the appropriate method. If the command is unknown the message is ignored and a DISCONNECT is sent. :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ cmd = msg.pop(0) if cmd in self._worker_cmds: fnc = self._worker_cmds[cmd] fnc(rp, msg) else: # # Ignore unknown command. Disconnect worker. # logging.error("Unknown worker command: {}".format(cmd)) self.disconnect(rp[0]) def on_message(self, msg): """Processes given message. Decides what kind of message it is -- client or worker -- and calls the appropriate method. If unknown, the message is ignored. :param msg: message parts :type msg: list of str :rtype: None """ rp, msg = split_address(msg) try: # # Dispatch on first frame after path # t = msg.pop(0) if t.startswith(b'MDPW'): logging.debug('Recieved message from worker {}'.format(rp)) self.on_worker(t, rp, msg) elif t.startswith(b'MDPC'): logging.debug('Recieved message from client {}'.format(rp)) self.on_client(t, rp, msg) else: logging.error('Broker unknown Protocol: "{}"'.format(t)) except: logging.error( "An error occured while trying to process message: rp: {}, msg: {}\n{}".format( rp, msg, traceback.format_exc() ) )
class WorkerRep(object): """Helper class to represent a worker in the broker. Instances of this class are used to track the state of the attached worker and carry the timers for incoming and outgoing heartbeats. :param proto: the worker protocol id. :type wid: str :param wid: the worker id. :type wid: str :param service: service this worker serves :type service: str :param stream: the ZMQStream used to send messages :type stream: ZMQStream """ def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_liveness = HB_LIVENESS self.stream = stream self.send_uniqueid() self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() def send_uniqueid(self): """Called on W_READY from worker. Sends unique id to worker. """ logging.debug('Broker to Worker {} sending unique id: {}'.format( self.service, self.id)) msg = [self.id, EMPTY_FRAME, self.proto, W_READY, self.id] self.stream.send_multipart(msg) def send_hb(self): """Called on every HB_INTERVAL. Decrements the current liveness by one. Sends heartbeat to worker. """ self.curr_liveness -= 1 logging.debug('Broker to Worker {} HB tick, current liveness: {}'.format( self.service, self.curr_liveness)) msg = [self.id, EMPTY_FRAME, self.proto, W_HEARTBEAT] self.stream.send_multipart(msg) def on_heartbeat(self): """Called when a heartbeat message from the worker was received. Sets current liveness to HB_LIVENESS. """ logging.debug('Received HB from worker {}'.format(self.service)) self.curr_liveness = HB_LIVENESS def is_alive(self): """Returns True when the worker is considered alive. """ return self.curr_liveness > 0 def shutdown(self): """Cleanup worker. Stops timer. """ logging.info('Shuting down worker {}'.format(self.service)) self.hb_out_timer.stop() self.hb_out_timer = None self.stream = None
class CCServer(skytools.BaseScript): """Listens on single ZMQ sockets, dispatches messages to handlers. Config:: ## Parameters for CCServer ## # listening socket for this CC instance cc-socket = tcp://127.0.0.1:22632 # zmq customization: #zmq_nthreads = 1 #zmq_linger = 500 #zmq_hwm = 100 #zmq_tcp_keepalive = 1 #zmq_tcp_keepalive_intvl = 15 #zmq_tcp_keepalive_idle = 240 #zmq_tcp_keepalive_cnt = 4 """ extra_ini = """ Extra segments:: # map req prefix to handler segment [routes] log = h:locallog # segment for specific handler [h:locallog] handler = cc.handler.locallogger """ log = skytools.getLogger('CCServer') cf_defaults = { 'logfmt_console': LOG.fmt, 'logfmt_file': LOG.fmt, 'logfmt_console_verbose': LOG.fmt_v, 'logfmt_file_verbose': LOG.fmt_v, 'logdatefmt_console': LOG.datefmt, 'logdatefmt_file': LOG.datefmt, 'logdatefmt_console_verbose': LOG.datefmt_v, 'logdatefmt_file_verbose': LOG.datefmt_v, } __version__ = __version__ stat_level = 1 zmq_nthreads = 1 zmq_linger = 500 zmq_hwm = 100 zmq_rcvbuf = 0 # means no change zmq_sndbuf = 0 # means no change zmq_tcp_keepalive = 1 zmq_tcp_keepalive_intvl = 15 zmq_tcp_keepalive_idle = 4 * 60 zmq_tcp_keepalive_cnt = 4 def reload(self): super(CCServer, self).reload() self.zmq_nthreads = self.cf.getint('zmq_nthreads', self.zmq_nthreads) self.zmq_hwm = self.cf.getint('zmq_hwm', self.zmq_hwm) self.zmq_linger = self.cf.getint('zmq_linger', self.zmq_linger) self.zmq_rcvbuf = hsize_to_bytes( self.cf.get('zmq_rcvbuf', str(self.zmq_rcvbuf))) self.zmq_sndbuf = hsize_to_bytes( self.cf.get('zmq_sndbuf', str(self.zmq_sndbuf))) self.zmq_tcp_keepalive = self.cf.getint('zmq_tcp_keepalive', self.zmq_tcp_keepalive) self.zmq_tcp_keepalive_intvl = self.cf.getint( 'zmq_tcp_keepalive_intvl', self.zmq_tcp_keepalive_intvl) self.zmq_tcp_keepalive_idle = self.cf.getint( 'zmq_tcp_keepalive_idle', self.zmq_tcp_keepalive_idle) self.zmq_tcp_keepalive_cnt = self.cf.getint('zmq_tcp_keepalive_cnt', self.zmq_tcp_keepalive_cnt) def print_ini(self): super(CCServer, self).print_ini() self._print_ini_frag(self.extra_ini) def startup(self): """Setup sockets and handlers.""" super(CCServer, self).startup() self.log.info("C&C server version %s starting up..", self.__version__) self.xtx = CryptoContext(self.cf) self.zctx = zmq.Context(self.zmq_nthreads) self.ioloop = IOLoop.instance() self.local_url = self.cf.get('cc-socket') self.cur_role = self.cf.get('cc-role', 'insecure') if self.cur_role == 'insecure': self.log.warning( 'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config' ) self.stat_level = self.cf.getint('cc-stats', 1) if self.stat_level < 1: self.log.warning('CC statistics level too low: %d', self.stat_level) # initialize local listen socket s = self.zctx.socket(zmq.XREP) s.setsockopt(zmq.LINGER, self.zmq_linger) s.setsockopt(zmq.HWM, self.zmq_hwm) if self.zmq_rcvbuf > 0: s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf) if self.zmq_sndbuf > 0: s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf) if self.zmq_tcp_keepalive > 0: if getattr(zmq, 'TCP_KEEPALIVE', -1) > 0: s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive) s.setsockopt(zmq.TCP_KEEPALIVE_INTVL, self.zmq_tcp_keepalive_intvl) s.setsockopt(zmq.TCP_KEEPALIVE_IDLE, self.zmq_tcp_keepalive_idle) s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt) else: self.log.info("TCP_KEEPALIVE not available") s.bind(self.local_url) self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm) self.local.on_recv(self.handle_cc_recv) self.handlers = {} self.routes = {} rcf = skytools.Config('routes', self.cf.filename, ignore_defs=True) for r, hnames in rcf.cf.items('routes'): self.log.info('New route: %s = %s', r, hnames) for hname in [hn.strip() for hn in hnames.split(',')]: h = self.get_handler(hname) self.add_handler(r, h) self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop) self.stimer.start() def send_stats(self): if self.stat_level == 0: return # make sure we have something to send self.stat_increase('count', 0) # combine our stats with global stats self.combine_stats(reset_stats()) super(CCServer, self).send_stats() def combine_stats(self, other): for k, v in other.items(): self.stat_inc(k, v) def get_handler(self, hname): if hname in self.handlers: h = self.handlers[hname] else: hcf = self.cf.clone(hname) # renamed option: plugin->handler htype = hcf.get('plugin', '?') if htype == '?': htype = hcf.get('handler') cls = cc_handler_lookup(htype, self.cur_role) h = cls(hname, hcf, self) self.handlers[hname] = h return h def add_handler(self, rname, handler): """Add route to handler""" if rname == '*': r = () else: r = tuple(rname.split('.')) self.log.debug('New route for handler: %r -> %s', r, handler.hname) rhandlers = self.routes.setdefault(r, []) rhandlers.append(handler) def handle_cc_recv(self, zmsg): """Got message from client, pick handler.""" start = time.time() self.stat_inc('count') self.log.trace('got msg: %r', zmsg) try: cmsg = CCMessage(zmsg) except: self.log.exception('Invalid CC message') self.stat_increase('count.invalid') return try: dst = cmsg.get_dest() size = cmsg.get_size() route = tuple(dst.split('.')) # find and run all handlers that match cnt = 0 for n in range(0, 1 + len(route)): p = route[:n] for h in self.routes.get(p, []): self.log.trace('calling handler %s', h.hname) h.handle_msg(cmsg) cnt += 1 if cnt == 0: self.log.warning('dropping msg, no route: %s', dst) stat = 'dropped' else: stat = 'ok' except Exception: self.log.exception('crashed, dropping msg: %s', dst) stat = 'crashed' # update stats taken = time.time() - start self.stat_inc('bytes', size) self.stat_inc('seconds', taken) self.stat_inc('count.%s' % stat) self.stat_inc('bytes.%s' % stat, size) self.stat_inc('seconds.%s' % stat, taken) if self.stat_level > 1: self.stat_inc('count.%s.msg.%s' % (stat, dst)) self.stat_inc('bytes.%s.msg.%s' % (stat, dst), size) self.stat_inc('seconds.%s.msg.%s' % (stat, dst), taken) def work(self): """Default work loop simply runs ioloop.""" self.set_single_loop(1) self.log.info('Starting IOLoop') try: self.ioloop.start() except zmq.ZMQError, d: # ZMQ gets surprised by EINTR if d.errno == errno.EINTR: return 1 raise
class CloneServer(object): # Our server is defined by these properties ctx = None # Context wrapper kvmap = None # Key-value store loop = None # IOLoop reactor port = None # Main port we're working on sequence = 0 # How many updates we're at snapshot = None # Handle snapshot requests publisher = None # Publish updates to clients collector = None # Collect updates from clients def __init__(self, port=5556): self.port = port self.ctx = zmq.Context() self.kvmap = {} self.loop = IOLoop.instance() # Set up our clone server sockets self.snapshot = self.ctx.socket(zmq.ROUTER) self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.PULL) self.snapshot.bind("tcp://*:%d" % self.port) self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Wrap sockets in ZMQStreams for IOLoop handlers self.snapshot = ZMQStream(self.snapshot) self.publisher = ZMQStream(self.publisher) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.snapshot.on_recv(self.handle_snapshot) self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) def start(self): # Run reactor until process interrupted self.flush_callback.start() try: self.loop.start() except KeyboardInterrupt: pass def handle_snapshot(self, msg): """snapshot requests""" if len(msg) != 3 or msg[1] != "ICANHAZ?": print "E: bad request, aborting" dump(msg) self.loop.stop() return identity, request, subtree = msg if subtree: # Send state snapshot to client route = Route(self.snapshot, identity, subtree) # For each entry in kvmap, send kvmsg to client for k,v in self.kvmap.items(): send_single(k,v,route) # Now send END message with sequence number logging.info("I: Sending state shapshot=%d" % self.sequence) self.snapshot.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree kvmsg.send(self.snapshot) def handle_collect(self, msg): """Collect updates from clients""" kvmsg = KVMsg.from_msg(msg) self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = float(kvmsg.get('ttl', 0)) if ttl: kvmsg['ttl'] = time.time() + ttl kvmsg.store(self.kvmap) logging.info("I: publishing update=%d", self.sequence) def flush_ttl(self): """Purge ephemeral values that have expired""" for key,kvmsg in self.kvmap.items(): self.flush_single(kvmsg) def flush_single(self, kvmsg): """If key-value pair has expired, delete it and publish the fact to listening clients.""" ttl = float(kvmsg.get('ttl', 0)) if ttl and ttl <= time.time(): kvmsg.body = "" self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) del self.kvmap[kvmsg.key] logging.info("I: publishing delete=%d", self.sequence)
class CloneServer(object): # Our server is defined by these properties ctx = None # Context wrapper kvmap = None # Key-value store loop = None # IOLoop reactor port = None # Main port we're working on sequence = 0 # How many updates we're at snapshot = None # Handle snapshot requests publisher = None # Publish updates to clients collector = None # Collect updates from clients def __init__(self, port=5556): self.port = port self.ctx = zmq.Context() self.kvmap = {} self.loop = IOLoop.instance() # Set up our clone server sockets self.snapshot = self.ctx.socket(zmq.ROUTER) self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.PULL) self.snapshot.bind(f"tcp://*:{self.port:d}") self.publisher.bind(f"tcp://*:{self.port + 1:d}") self.collector.bind(f"tcp://*:{self.port + 2:d}") # Wrap sockets in ZMQStreams for IOLoop handlers self.snapshot = ZMQStream(self.snapshot) self.publisher = ZMQStream(self.publisher) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.snapshot.on_recv(self.handle_snapshot) self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) def start(self): # Run reactor until process interrupted self.flush_callback.start() try: self.loop.start() except KeyboardInterrupt: pass def handle_snapshot(self, msg): """snapshot requests""" if len(msg) != 3 or msg[1] != b"ICANHAZ?": print("E: bad request, aborting") dump(msg) self.loop.stop() return [identity, request, subtree] = msg if subtree: # Send state snapshot to client route = Route(self.snapshot, identity, subtree) # For each entry in kvmap, send kvmsg to client for k, v in self.kvmap.items(): send_single(k, v, route) # Now send END message with sequence number logging.info(f"I: Sending state shapshot={self.sequence:d}") self.snapshot.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = b"KTHXBAI" kvmsg.body = subtree kvmsg.send(self.snapshot) def handle_collect(self, msg): """Collect updates from clients""" kvmsg = KVMsg.from_msg(msg) self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = float(kvmsg.get(b'ttl', 0)) if ttl: kvmsg[b'ttl'] = b'%f' % (time.time() + ttl) kvmsg.store(self.kvmap) logging.info(f"I: publishing update={self.sequence:d}") def flush_ttl(self): """Purge ephemeral values that have expired""" # used list() to exhaust the iterator before deleting from the dict for key, kvmsg in list(self.kvmap.items()): self.flush_single(kvmsg) def flush_single(self, kvmsg): """If key-value pair has expired, delete it and publish the fact to listening clients.""" ttl = float(kvmsg.get(b'ttl', 0)) if ttl and ttl <= time.time(): kvmsg.body = b"" self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) del self.kvmap[kvmsg.key] logging.info(f"I: publishing delete={self.sequence:d}")
class MNWorker(MN_object): """Class for the MN worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. :param context: the context to use for socket creation. :type context: zmq.Context :param endpoint: endpoint to connect to. :type endpoint: str :param service: the name of the service we support. :type service: byte-string """ _proto_version = b'MNPW01' # worker protocol version def __init__(self, context, endpoint, service, worker_type, address, protocols): """Initialize the MNWorker. """ self.context = context self.endpoint = endpoint self.service = service self.type = worker_type self.address = address self.protocols = protocols self.envelope = None self.HB_RETRIES = HB_RETRIES self.HB_INTERVAL = HB_INTERVAL self._data = {} self.stream = None self._tmo = None self.timed_out = False self.need_handshake = True self.connected = False self.ticker = None self._delayed_cb = None self._create_stream() _LOG.info("Worker initialized and can be found at '%s'" % endpoint) return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ _LOG.debug("Informing broker I am ready") ready_msg = [ b'', WORKER_PROTO, MSG_READY, self.service, self.type, self.address, self.protocols ] if self.stream.closed(): self.shutdown() self.stream.send_multipart(ready_msg) self.curr_retries = self.HB_RETRIES return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_retries -= 1 self.send_hb() if self.curr_retries >= 0: return # connection seems to be dead self.shutdown() # try to recreate it # self._delayed_cb = IOLoop.call_later(self._create_stream, 5000) # self._delayed_cb = IOLoop.add_timeout(self._create_stream, 5000) self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ _LOG.debug("Sending heartbeat") msg = [b'', WORKER_PROTO, MSG_HEARTBEAT] if self.stream.closed(): self.shutdown() self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. :param msg: full message to send. :type msg: can either be a byte-string or a list of byte-strings """ if self.need_handshake: raise ConnectionNotReadyError() to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) if self.stream.closed(): self.shutdown() self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. :param msg: a list w/ the message parts :type msg: a list of byte-strings """ _LOG.debug("Received: %s." % msg) # 1st part is empty msg.pop(0) # 2nd part is protocol version proto = msg.pop(0) if proto != WORKER_PROTO: # ignore message from not supported protocol pass # 3rd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the retries counter self.need_handshake = False self.curr_retries = self.HB_RETRIES if msg_type == MSG_DISCONNECT: # disconnect _LOG.info("Broker wants us to disconnect.") self.curr_retries = 0 # reconnect will be triggered by hb timer elif msg_type == MSG_QUERY: # request # remaining parts are the user message _LOG.debug("Received new request: %s." % msg) envelope, msg = split_address(msg) envelope.append(b'') envelope = [b'', WORKER_PROTO, MSG_REPLY] + envelope # reply self.envelope = envelope self.on_request(msg) else: # invalid message # ignored _LOG.debug('ignoring message with invalid id') pass return def on_request(self, msg): """Public method called when a request arrived. :param msg: a list w/ the message parts :type msg: a list of byte-strings Must be overloaded to provide support for various services! """ pass
class CryptoTransportLayer(TransportLayer): def __init__(self, my_ip, my_port, market_id, db, bm_user=None, bm_pass=None, bm_port=None, seed_mode=0, dev_mode=False): self.log = logging.getLogger('[%s] %s' % (market_id, self.__class__.__name__)) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Connect to database self.db = db self.bitmessage_api = None if (bm_user, bm_pass, bm_port) != (None, None, None): if not self._connect_to_bitmessage(bm_user, bm_pass, bm_port): self.log.info('Bitmessage not installed or started') try: socket.inet_pton(socket.AF_INET6, my_ip) my_uri = "tcp://[%s]:%s" % (my_ip, my_port) except socket.error: my_uri = "tcp://%s:%s" % (my_ip, my_port) self.market_id = market_id self.nick_mapping = {} self.uri = my_uri self.ip = my_ip self.nickname = "" self._dev_mode = dev_mode # Set up self._setup_settings() self.dht = DHT(self, self.market_id, self.settings, self.db) # self._myself = ec.ECC(pubkey=self.pubkey.decode('hex'), # privkey=self.secret.decode('hex'), # curve='secp256k1') TransportLayer.__init__(self, market_id, my_ip, my_port, self.guid, self.nickname) self.setup_callbacks() self.listen(self.pubkey) if seed_mode == 0 and not dev_mode: self.start_ip_address_checker() def setup_callbacks(self): self.add_callbacks([('hello', self._ping), ('findNode', self._find_node), ('findNodeResponse', self._find_node_response), ('store', self._store_value)]) def start_ip_address_checker(self): '''Checks for possible public IP change''' self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() def _ip_updater_periodic_callback(self): try: r = requests.get('https://icanhazip.com') if r and hasattr(r, 'text'): ip = r.text ip = ip.strip(' \t\n\r') if ip != self.ip: self.ip = ip try: socket.inet_pton(socket.AF_INET6, self.ip) my_uri = 'tcp://[%s]:%s' % (self.ip, self.port) except socket.error: my_uri = 'tcp://%s:%s' % (self.ip, self.port) self.uri = my_uri self.stream.close() self.listen(self.pubkey) self.dht._iterativeFind(self.guid, [], 'findNode') else: self.log.error('Could not get IP') except Exception as e: self.log.error('[Requests] error: %s' % e) def save_peer_to_db(self, peer_tuple): uri = peer_tuple[0] pubkey = peer_tuple[1] guid = peer_tuple[2] nickname = peer_tuple[3] # Update query self.db.deleteEntries("peers", {"uri": uri, "guid": guid}, "OR") # if len(results) > 0: # self.db.updateEntries("peers", {"id": results[0]['id']}, {"market_id": self.market_id, "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname}) # else: if guid is not None: self.db.insertEntry( "peers", { "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname, "market_id": self.market_id }) def _connect_to_bitmessage(self, bm_user, bm_pass, bm_port): # Get bitmessage going # First, try to find a local instance result = False try: self.log.info( '[_connect_to_bitmessage] Connecting to Bitmessage on port %s' % bm_port) self.bitmessage_api = xmlrpclib.ServerProxy( "http://{}:{}@localhost:{}/".format(bm_user, bm_pass, bm_port), verbose=0) result = self.bitmessage_api.add(2, 3) self.log.info( "[_connect_to_bitmessage] Bitmessage API is live".format( result)) # If we failed, fall back to starting our own except Exception as e: self.log.info( "Failed to connect to bitmessage instance: {}".format(e)) self.bitmessage_api = None # self._log.info("Spawning internal bitmessage instance") # # Add bitmessage submodule path # sys.path.insert(0, os.path.join( # os.path.dirname(__file__), '..', 'pybitmessage', 'src')) # import bitmessagemain as bitmessage # bitmessage.logger.setLevel(logging.WARNING) # bitmessage_instance = bitmessage.Main() # bitmessage_instance.start(daemon=True) # bminfo = bitmessage_instance.getApiAddress() # if bminfo is not None: # self._log.info("Started bitmessage daemon at %s:%s".format( # bminfo['address'], bminfo['port'])) # bitmessage_api = xmlrpclib.ServerProxy("http://{}:{}@{}:{}/".format( # bm_user, bm_pass, bminfo['address'], bminfo['port'])) # else: # self._log.info("Failed to start bitmessage dameon") # self._bitmessage_api = None return result def _checkok(self, msg): self.log.info('Check ok') def get_guid(self): return self.guid def get_dht(self): return self.dht def get_bitmessage_api(self): return self.bitmessage_api def get_market_id(self): return self.market_id # def get_myself(self): # return self._myself def _ping(self, msg): self.log.info('Pinged %s ' % json.dumps(msg, ensure_ascii=False)) # # pinger = CryptoPeerConnection(self, msg['uri'], msg['pubkey'], msg['senderGUID']) # pinger.send_raw(json.dumps( # {"type": "hello_response", # "senderGUID": self.guid, # "uri": self.uri, # "senderNick": self.nickname, # "pubkey": self.pubkey, # })) def _store_value(self, msg): self.dht._on_storeValue(msg) def _find_node(self, msg): self.dht.on_find_node(msg) def _find_node_response(self, msg): self.dht.on_findNodeResponse(self, msg) def _setup_settings(self): try: self.settings = self.db.selectEntries( "settings", {"market_id": self.market_id}) except (OperationalError, DatabaseError) as e: print e raise SystemExit( "database file %s corrupt or empty - cannot continue" % self.db.db_path) if len(self.settings) == 0: self.settings = {"market_id": self.market_id, "welcome": "enable"} self.db.insertEntry("settings", self.settings) else: self.settings = self.settings[0] # Generate PGP key during initial setup or if previous PGP gen failed if not ('PGPPubKey' in self.settings and self.settings["PGPPubKey"]): try: self.log.info( 'Generating PGP keypair. This may take several minutes...') print 'Generating PGP keypair. This may take several minutes...' gpg = gnupg.GPG() input_data = gpg.gen_key_input( key_type="RSA", key_length=2048, name_email='*****@*****.**', name_comment="Autogenerated by Open Bazaar", passphrase="P@ssw0rd") assert input_data is not None key = gpg.gen_key(input_data) assert key is not None pubkey_text = gpg.export_keys(key.fingerprint) newsettings = { "PGPPubKey": pubkey_text, "PGPPubkeyFingerprint": key.fingerprint } self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) self.log.info('PGP keypair generated.') except Exception as e: self.log.error("Encountered a problem with GPG: %s" % e) raise SystemExit("Encountered a problem with GPG: %s" % e) if not ('pubkey' in self.settings and self.settings['pubkey']): # Generate Bitcoin keypair self._generate_new_keypair() if not ('bitmessage' in self.settings and self.settings['bitmessage']): # Generate Bitmessage address if self.bitmessage_api is not None: self._generate_new_bitmessage_address() if not ('nickname' in self.settings and self.settings['nickname']): newsettings = {'nickname': 'Default'} self.db.updateEntries('settings', {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) self.nickname = self.settings[ 'nickname'] if 'nickname' in self.settings else "" self.secret = self.settings[ 'secret'] if 'secret' in self.settings else "" self.pubkey = self.settings[ 'pubkey'] if 'pubkey' in self.settings else "" self.privkey = self.settings.get('privkey') self.btc_pubkey = privkey_to_pubkey(self.privkey) self.guid = self.settings['guid'] if 'guid' in self.settings else "" self.sin = self.settings['sin'] if 'sin' in self.settings else "" self.bitmessage = self.settings[ 'bitmessage'] if 'bitmessage' in self.settings else "" self._myself = ec.ECC(pubkey=pubkey_to_pyelliptic( self.pubkey).decode('hex'), raw_privkey=self.secret.decode('hex'), curve='secp256k1') self.log.debug('Retrieved Settings: \n%s', pformat(self.settings)) def _generate_new_keypair(self): secret = str(random.randrange(2**256)) self.secret = hashlib.sha256(secret).hexdigest() self.pubkey = privtopub(self.secret) self.privkey = random_key() print 'PRIVATE KEY: ', self.privkey self.btc_pubkey = privtopub(self.privkey) print 'PUBLIC KEY: ', self.btc_pubkey # Generate SIN sha_hash = hashlib.sha256() sha_hash.update(self.pubkey) ripe_hash = hashlib.new('ripemd160') ripe_hash.update(sha_hash.digest()) self.guid = ripe_hash.digest().encode('hex') self.sin = obelisk.EncodeBase58Check('\x0F\x02%s' + ripe_hash.digest()) newsettings = { "secret": self.secret, "pubkey": self.pubkey, "privkey": self.privkey, "guid": self.guid, "sin": self.sin } self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) def _generate_new_bitmessage_address(self): # Use the guid generated previously as the key self.bitmessage = self.bitmessage_api.createRandomAddress( self.guid.encode('base64'), False, 1.05, 1.1111) newsettings = {"bitmessage": self.bitmessage} self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) def join_network(self, seed_peers=[], callback=lambda msg: None): self.log.info('Joining network') known_peers = [] # Connect up through seed servers for idx, seed in enumerate(seed_peers): try: socket.inet_pton(socket.AF_INET6, seed) seed_peers[idx] = "tcp://[%s]:12345" % seed except socket.error: seed_peers[idx] = "tcp://%s:12345" % seed # Connect to persisted peers db_peers = self.get_past_peers() known_peers = list(set(seed_peers)) + list(set(db_peers)) print 'known_peers', known_peers self.connect_to_peers(known_peers) # Populate routing table by searching for self if len(known_peers) > 0: self.search_for_my_node() if callback is not None: callback('Joined') def get_past_peers(self): peers = [] result = self.db.selectEntries("peers", {"market_id": self.market_id}) for peer in result: peers.append(peer['uri']) return peers def search_for_my_node(self): print 'Searching for myself' self.dht._iterativeFind(self.guid, self.dht.knownNodes, 'findNode') def connect_to_peers(self, known_peers): for known_peer in known_peers: t = Thread(target=self.dht.add_peer, args=( self, known_peer, )) t.start() def get_crypto_peer(self, guid=None, uri=None, pubkey=None, nickname=None, callback=None): if guid == self.guid: self.log.error('Cannot get CryptoPeerConnection for your own node') return self.log.debug('Getting CryptoPeerConnection' + '\nGUID:%s\nURI:%s\nPubkey:%s\nNickname:%s' % (guid, uri, pubkey, nickname)) return connection.CryptoPeerConnection(self, uri, pubkey, guid=guid, nickname=nickname, callback=callback) def addCryptoPeer(self, peer_to_add): foundOutdatedPeer = False for idx, peer in enumerate(self.dht.activePeers): if (peer.address, peer.guid, peer.pub) == \ (peer_to_add.address, peer_to_add.guid, peer_to_add.pub): self.log.info('Found existing peer, not adding.') return if peer.guid == peer_to_add.guid or \ peer.pub == peer_to_add.pub or \ peer.address == peer_to_add.address: foundOutdatedPeer = True self.log.info('Found an outdated peer') # Update existing peer self.activePeers[idx] = peer_to_add self.dht.add_peer(self, peer_to_add.address, peer_to_add.pub, peer_to_add.guid, peer_to_add.nickname) if not foundOutdatedPeer and peer_to_add.guid != self.guid: self.log.info('Adding crypto peer at %s' % peer_to_add.nickname) self.dht.add_peer(self, peer_to_add.address, peer_to_add.pub, peer_to_add.guid, peer_to_add.nickname) def get_profile(self): peers = {} self.settings = self.db.selectEntries("settings", {"market_id": self.market_id})[0] for uri, peer in self.peers.iteritems(): if peer.pub: peers[uri] = peer.pub.encode('hex') return { 'uri': self.uri, 'pub': self._myself.get_pubkey().encode('hex'), 'nickname': self.nickname, 'peers': peers } def respond_pubkey_if_mine(self, nickname, ident_pubkey): if ident_pubkey != self.pubkey: self.log.info("Public key does not match your identity") return # Return signed pubkey pubkey = self._myself.pubkey ec_key = obelisk.EllipticCurveKey() ec_key.set_secret(self.secret) digest = obelisk.Hash(pubkey) signature = ec_key.sign(digest) # Send array of nickname, pubkey, signature to transport layer self.send(proto_response_pubkey(nickname, pubkey, signature)) def pubkey_exists(self, pub): for uri, peer in self.peers.iteritems(): self.log.info('PEER: %s Pub: %s' % (peer.pub.encode('hex'), pub.encode('hex'))) if peer.pub.encode('hex') == pub.encode('hex'): return True return False def create_peer(self, uri, pub, node_guid): if pub: pub = pub.decode('hex') # Create the peer if public key is not already in the peer list # if not self.pubkey_exists(pub): self.peers[uri] = connection.CryptoPeerConnection( self, uri, pub, node_guid) # Call 'peer' callbacks on listeners self.trigger_callbacks('peer', self.peers[uri]) # else: # print 'Pub Key is already in peer list' def send(self, data, send_to=None, callback=lambda msg: None): self.log.debug("Outgoing Data: %s %s" % (data, send_to)) # Directed message if send_to is not None: peer = self.dht.routingTable.getContact(send_to) if not peer: for activePeer in self.dht.activePeers: if activePeer.guid == send_to: peer = activePeer break # peer = CryptoPeerConnection(msg['uri']) if peer: self.log.debug('Directed Data (%s): %s' % (send_to, data)) try: peer.send(data, callback=callback) except Exception as e: self.log.error('Not sending message directly to peer %s' % e) else: self.log.error('No peer found') else: # FindKey and then send for peer in self.dht.activePeers: try: peer = self.dht.routingTable.getContact(peer.guid) data['senderGUID'] = self.guid data['pubkey'] = self.pubkey def cb(msg): self.log.debug('Message Back: \n%s' % pformat(msg)) peer.send(data, cb) except: self.log.info("Error sending over peer!") traceback.print_exc() def send_enc(self, uri, msg): peer = self.peers[uri] pub = peer.pub # Now send a hello message to the peer if pub: self.log.info("Sending encrypted [%s] message to %s" % (msg['type'], uri)) peer.send(msg) else: # Will send clear profile on initial if no pub self.log.info("Sending unencrypted [%s] message to %s" % (msg['type'], uri)) self.peers[uri].send_raw(json.dumps(msg)) def _init_peer(self, msg): uri = msg['uri'] pub = msg.get('pub') nickname = msg.get('nickname') msg_type = msg.get('type') guid = msg['guid'] if not self.valid_peer_uri(uri): self.log.error("Invalid Peer: %s " % uri) return if uri not in self.peers: # Unknown peer self.log.info('Add New Peer: %s' % uri) self.create_peer(uri, pub, guid) if not msg_type: self.send_enc(uri, hello_request(self.get_profile())) elif msg_type == 'hello_request': self.send_enc(uri, hello_response(self.get_profile())) else: # Known peer if pub: # test if we have to update the pubkey if not self.peers[uri].pub: self.log.info("Setting public key for seed node") self.peers[uri].pub = pub.decode('hex') self.trigger_callbacks('peer', self.peers[uri]) if self.peers[uri].pub != pub.decode('hex'): self.log.info("Updating public key for node") self.peers[uri].nickname = nickname self.peers[uri].pub = pub.decode('hex') self.trigger_callbacks('peer', self.peers[uri]) if msg_type == 'hello_request': # reply only if necessary self.send_enc(uri, hello_response(self.get_profile())) def _on_message(self, msg): # here goes the application callbacks # we get a "clean" msg which is a dict holding whatever # self.log.info("[On Message] Data received: %s" % msg) pubkey = msg.get('pubkey') uri = msg.get('uri') ip = urlparse(uri).hostname port = urlparse(uri).port guid = msg.get('senderGUID') nickname = msg.get('senderNick')[:120] self.dht.add_known_node((ip, port, guid, nickname)) self.log.info('ON MESSAGE %s' % json.dumps(msg, ensure_ascii=False)) self.dht.add_peer(self, uri, pubkey, guid, nickname) self.log.debug('Callbacks %s' % self.callbacks) t = Thread(target=self.trigger_callbacks, args=( msg['type'], msg, )) t.start() def _on_raw_message(self, serialized): try: # Decompress message serialized = zlib.decompress(serialized) msg = json.loads(serialized) self.log.info("Message Received [%s]" % msg.get('type', 'unknown')) if msg.get('type') is None: data = msg.get('data').decode('hex') sig = msg.get('sig').decode('hex') try: cryptor = makePrivCryptor(self.secret) try: data = cryptor.decrypt(data) except Exception as e: self.log.info('Exception: %s' % e) self.log.debug('Signature: %s' % sig.encode('hex')) self.log.debug('Signed Data: %s' % data) # Check signature data_json = json.loads(data) sigCryptor = makePubCryptor(data_json['pubkey']) if sigCryptor.verify(sig, data): self.log.info('Verified') else: self.log.error( 'Message signature could not be verified %s' % msg) # return msg = json.loads(data) self.log.debug('Message Data %s ' % msg) except Exception as e: self.log.error('Could not decrypt message properly %s' % e) except ValueError: try: # Encrypted? try: msg = self._myself.decrypt(serialized) msg = json.loads(msg) self.log.info("Decrypted Message [%s]" % msg.get('type', 'unknown')) except: self.log.error("Could not decrypt message: %s" % msg) return except: self.log.error('Message probably sent using incorrect pubkey') return if msg.get('type') is not None: self._on_message(msg) else: self.log.error('Received a message with no type') def shutdown(self): print "CryptoTransportLayer.shutdown()!" try: TransportLayer.shutdown(self) print "CryptoTransportLayer.shutdown(): ZMQ sockets destroyed." except Exception as e: self.log.error("Transport shutdown error: " + e.message) print "Notice: explicit DHT Shutdown not implemented." try: self.bitmessage_api.close() except Exception as e: # might not even be open, not much more we can do on our way out if exception thrown here. self.log.error( "Could not shutdown bitmessage_api's ServerProxy. " + e.message)
class MDPWorker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = b'MDPW01' # TODO: integrate that into API HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, endpoint, service): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ self.context = context self.endpoint = endpoint self.service = service self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ ready_msg = [b'', self._proto_version, b'\x01', self.service] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return ## print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [b'', self._proto_version, b'\x04'] self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ ## if self.need_handshake: ## raise ConnectionNotReadyError() # prepare full message to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ # 1st part is empty msg.pop(0) # 2nd part is protocol version # TODO: version check proto = msg.pop(0) # 3rd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if msg_type == b'\x05': # disconnect self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == b'\x02': # request # remaining parts are the user message envelope, msg = split_address(msg) envelope.append(b'') envelope = [b'', self._proto_version, b'\x03'] + envelope # REPLY self.envelope = envelope self.on_request(msg) else: # invalid message # ignored pass return def on_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ pass
class CloneServer(object): # Our server is defined by these properties ctx = None # Context wrapper kvmap = None # Key-value store bstar = None # Binary Star sequence = 0 # How many updates so far port = None # Main port we're working on peer = None # Main port of our peer publisher = None # Publish updates and hugz collector = None # Collect updates from clients subscriber = None # Get updates from peer pending = None # Pending updates from client primary = False # True if we're primary master = False # True if we're master slave = False # True if we're slave def __init__(self, primary=True, ports=(5556,5566)): self.primary = primary if primary: self.port, self.peer = ports frontend = "tcp://*:5003" backend = "tcp://localhost:5004" self.kvmap = {} else: self.peer, self.port = ports frontend = "tcp://*:5004" backend = "tcp://localhost:5003" self.ctx = zmq.Context.instance() self.pending = [] self.bstar = BinaryStar(primary, frontend, backend) self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER, self.handle_snapshot) # Set up our clone server sockets self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.SUB) self.collector.setsockopt(zmq.SUBSCRIBE, b'') self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Set up our own clone client interface to peer self.subscriber = self.ctx.socket(zmq.SUB) self.subscriber.setsockopt(zmq.SUBSCRIBE, b'') self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1)) # Register state change handlers self.bstar.master_callback = self.become_master self.bstar.slave_callback = self.become_slave # Wrap sockets in ZMQStreams for IOLoop handlers self.publisher = ZMQStream(self.publisher) self.subscriber = ZMQStream(self.subscriber) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) self.hugz_callback = PeriodicCallback(self.send_hugz, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) def start(self): # start periodic callbacks self.flush_callback.start() self.hugz_callback.start() # Run bstar reactor until process interrupted try: self.bstar.start() except KeyboardInterrupt: pass def handle_snapshot(self, socket, msg): """snapshot requests""" if msg[1] != "ICANHAZ?" or len(msg) != 3: logging.error("E: bad request, aborting") dump(msg) self.bstar.loop.stop() return identity, request = msg[:2] if len(msg) >= 3: subtree = msg[2] # Send state snapshot to client route = Route(socket, identity, subtree) # For each entry in kvmap, send kvmsg to client for k,v in self.kvmap.items(): send_single(k,v,route) # Now send END message with sequence number logging.info("I: Sending state shapshot=%d" % self.sequence) socket.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree kvmsg.send(socket) def handle_collect(self, msg): """Collect updates from clients If we're master, we apply these to the kvmap If we're slave, or unsure, we queue them on our pending list """ kvmsg = KVMsg.from_msg(msg) if self.master: self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + ttl kvmsg.store(self.kvmap) logging.info("I: publishing update=%d", self.sequence) else: # If we already got message from master, drop it, else # hold on pending list if not self.was_pending(kvmsg): self.pending.append(kvmsg) def was_pending(self, kvmsg): """If message was already on pending list, remove and return True. Else return False. """ found = False for idx, held in enumerate(self.pending): if held.uuid == kvmsg.uuid: found = True break if found: self.pending.pop(idx) return found def flush_ttl(self): """Purge ephemeral values that have expired""" if self.kvmap: for key,kvmsg in self.kvmap.items(): self.flush_single(kvmsg) def flush_single(self, kvmsg): """If key-value pair has expired, delete it and publish the fact to listening clients.""" if kvmsg.get('ttl', 0) <= time.time(): kvmsg.body = "" self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) del self.kvmap[kvmsg.key] logging.info("I: publishing delete=%d", self.sequence) def send_hugz(self): """Send hugz to anyone listening on the publisher socket""" kvmsg = KVMsg(self.sequence) kvmsg.key = "HUGZ" kvmsg.body = "" kvmsg.send(self.publisher) # --------------------------------------------------------------------- # State change handlers def become_master(self): """We're becoming master The backup server applies its pending list to its own hash table, and then starts to process state snapshot requests. """ self.master = True self.slave = False # stop receiving subscriber updates while we are master self.subscriber.stop_on_recv() # Apply pending list to own kvmap while self.pending: kvmsg = self.pending.pop(0) self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.store(self.kvmap) logging.info ("I: publishing pending=%d", self.sequence) def become_slave(self): """We're becoming slave""" # clear kvmap self.kvmap = None self.master = False self.slave = True self.subscriber.on_recv(self.handle_subscriber) def handle_subscriber(self, msg): """Collect updates from peer (master) We're always slave when we get these updates """ if self.master: logging.warn("received subscriber message, but we are master %s", msg) return # Get state snapshot if necessary if self.kvmap is None: self.kvmap = {} snapshot = self.ctx.socket(zmq.DEALER) snapshot.linger = 0 snapshot.connect("tcp://localhost:%i" % self.peer) logging.info ("I: asking for snapshot from: tcp://localhost:%d", self.peer) snapshot.send_multipart(["ICANHAZ?", '']) while True: try: kvmsg = KVMsg.recv(snapshot) except KeyboardInterrupt: # Interrupted self.bstar.loop.stop() return if kvmsg.key == "KTHXBAI": self.sequence = kvmsg.sequence break # Done kvmsg.store(self.kvmap) logging.info ("I: received snapshot=%d", self.sequence) # Find and remove update off pending list kvmsg = KVMsg.from_msg(msg) # update integer ttl -> timestamp ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + ttl if kvmsg.key != "HUGZ": if not self.was_pending(kvmsg): # If master update came before client update, flip it # around, store master update (with sequence) on pending # list and use to clear client update when it comes later self.pending.append(kvmsg) # If update is more recent than our kvmap, apply it if (kvmsg.sequence > self.sequence): self.sequence = kvmsg.sequence kvmsg.store(self.kvmap) logging.info ("I: received update=%d", self.sequence)
class Master(object): def __init__(self, frontier, data_in_sock='ipc:///tmp/robot-data-w2m.sock', data_out_sock='ipc:///tmp/robot-data-m2w.sock', msg_in_sock='ipc:///tmp/robot-msg-w2m.sock', msg_out_sock='ipc:///tmp/robot-msg-m2w.sock', io_loop=None): self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid()) context = zmq.Context() self._io_loop = io_loop or IOLoop.instance() self._in_socket = context.socket(zmq.SUB) self._in_socket.setsockopt(zmq.SUBSCRIBE, '') self._in_socket.bind(data_in_sock) self._in_stream = ZMQStream(self._in_socket, io_loop) self._out_socket = context.socket(zmq.PUSH) self._out_socket.bind(data_out_sock) self._out_stream = ZMQStream(self._out_socket, io_loop) self._online_workers = set() self._running = False self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop) self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop) self.frontier = frontier self.messenger = ServerMessenger(msg_in_sock, msg_out_sock, context, io_loop) def start(self): logging.info('[%s] starting', self.identity) self.messenger.add_callback(CTRL_MSG_WORKER, self._on_worker_msg) self.messenger.start() self._in_stream.on_recv(self._on_receive_processed) self._updater.start() self._reloader.start() self._running = True def stop(self): self._running = False self._reloader.stop() self._updater.stop() self.messenger.stop() # self.messenger.publish(CTRL_MSG_WORKER, self.identity, # CTRL_MSG_WORKER_QUIT) def close(self): self._in_stream.close() self._in_socket.close() self._out_stream.close() self._out_socket.close() self.messenger.close() def reload(self): pass def _on_worker_msg(self, msg): if msg.data == CTRL_MSG_WORKER_ONLINE: self._online_workers.add(msg.identity) logging.info('[%s] append [%s]', self.identity, msg.identity) self._send_next() # if msg.data == CTRL_MSG_WORKER_QUIT_ACK: # if msg.identity in self._online_workers: # self._online_workers.remove(msg.identity) def _send_next(self): if not self._running: return worker_num = len(self._online_workers) if self._running and worker_num > 0: while self._out_stream._send_queue.qsize() < worker_num * 4: request = self.frontier.get_next_request() if not request: break msg = RequestMessage(self.identity, request) self._out_stream.send_multipart(msg.serialize()) logging.debug('[%s] send request(%s)', self.identity, request.url) self.frontier.reload_request(request) def _on_receive_processed(self, zmq_msg): msg = ResponseMessage.deserialize(zmq_msg) request = msg.response.request logging.debug('[%s] receive response(%s)', self.identity, request.url) self._send_next()
class MDPBroker(object): """The MDP broker class. The broker routes messages from clients to appropriate workers based on the requested service. This base class defines the overall functionality and the API. Subclasses are ment to implement additional features (like logging). The broker uses ØMQ XREQ sockets to deal witch clients and workers. These sockets are wrapped in pyzmq streams to fit well into IOLoop. .. note:: The workers will *always* be served by the `main_ep` endpoint. In a two-endpoint setup clients will be handled via the `opt_ep` endpoint. :param context: the context to use for socket creation. :type context: zmq.Context :param main_ep: the primary endpoint for workers and clients. :type main_ep: str :param opt_ep: is an optional 2nd endpoint. :type opt_ep: str :param worker_q: the class to be used for the worker-queue. :type worker_q: class """ CLIENT_PROTO = b'MDPC01' #: Client protocol identifier WORKER_PROTO = b'MDPW01' #: Worker protocol identifier def __init__(self, context, main_ep, opt_ep=None, worker_q=None): """Init MDPBroker instance. """ socket = context.socket(zmq.XREP) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.XREP) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self._workers = {} # services contain the worker queue and the request queue self._services = {} self._worker_cmds = { '\x01': self.on_ready, '\x03': self.on_reply, '\x04': self.on_heartbeat, '\x05': self.on_disconnect, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() return def register_worker(self, wid, service): """Register the worker id and add it to the given service. Does nothing if worker is already known. :param wid: the worker id. :type wid: str :param service: the service name. :type service: str :rtype: None """ if wid in self._workers: return self._workers[wid] = WorkerRep(self.WORKER_PROTO, wid, service, self.main_stream) if service in self._services: wq, wr = self._services[service] wq.put(wid) else: q = ServiceQueue() q.put(wid) self._services[service] = (q, []) return def unregister_worker(self, wid): """Unregister the worker with the given id. If the worker id is not registered, nothing happens. Will stop all timers for the worker. :param wid: the worker id. :type wid: str :rtype: None """ try: wrep = self._workers[wid] except KeyError: # not registered, ignore return wrep.shutdown() service = wrep.service if service in self._services: wq, wr = self._services[service] wq.remove(wid) del self._workers[wid] return def disconnect(self, wid): """Send disconnect command and unregister worker. If the worker id is not registered, nothing happens. :param wid: the worker id. :type wid: str :rtype: None """ try: wrep = self._workers[wid] except KeyError: # not registered, ignore return to_send = [ wid, self.WORKER_PROTO, b'\x05' ] self.main_stream.send_multipart(to_send) self.unregister_worker(wid) return def client_response(self, rp, service, msg): """Package and send reply to client. :param rp: return address stack :type rp: list of str :param service: name of service :type service: str :param msg: message parts :type msg: list of str :rtype: None """ to_send = rp[:] to_send.extend([b'', self.CLIENT_PROTO, service]) to_send.extend(msg) self.client_stream.send_multipart(to_send) return def shutdown(self): """Shutdown broker. Will unregister all workers, stop all timers and ignore all further messages. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ if self.client_stream == self.main_stream: self.client_stream = None self.main_stream.on_recv(None) self.main_stream.socket.setsockopt(zmq.LINGER, 0) self.main_stream.socket.close() self.main_stream.close() self.main_stream = None if self.client_stream: self.client_stream.on_recv(None) self.client_stream.socket.setsockopt(zmq.LINGER, 0) self.client_stream.socket.close() self.client_stream.close() self.client_stream = None self._workers = {} self._services = {} return def on_timer(self): """Method called on timer expiry. Checks which workers are dead and unregisters them. :rtype: None """ for wrep in self._workers.values(): if not wrep.is_alive(): self.unregister_worker(wrep.id) return def on_ready(self, rp, msg): """Process worker READY command. Registers the worker for a service. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] self.register_worker(ret_id, msg[0]) return def on_reply(self, rp, msg): """Process worker REPLY command. Route the `msg` to the client given by the address(es) in front of `msg`. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] wrep = self._workers[ret_id] service = wrep.service # make worker available again try: wq, wr = self._services[service] cp, msg = split_address(msg) self.client_response(cp, service, msg) wq.put(wrep.id) if wr: proto, rp, msg = wr.pop(0) self.on_client(proto, rp, msg) except KeyError: # unknown service self.disconnect(ret_id) return def on_heartbeat(self, rp, msg): """Process worker HEARTBEAT command. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ret_id = rp[0] try: worker = self._workers[ret_id] if worker.is_alive(): worker.on_heartbeat() except KeyError: # ignore HB for unknown worker pass return def on_disconnect(self, rp, msg): """Process worker DISCONNECT command. Unregisters the worker who sent this message. :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ wid = rp[0] self.unregister_worker(wid) return def on_mmi(self, rp, service, msg): """Process MMI request. For now only mmi.service is handled. :param rp: return address stack :type rp: list of str :param service: the protocol id sent :type service: str :param msg: message parts :type msg: list of str :rtype: None """ if service == b'mmi.service': s = msg[0] ret = b'404' for wr in self._workers.values(): if s == wr.service: ret = b'200' break self.client_response(rp, service, [ret]) else: self.client_response(rp, service, [b'501']) return def on_client(self, proto, rp, msg): """Method called on client message. Frame 0 of msg is the requested service. The remaining frames are the request to forward to the worker. .. note:: If the service is unknown to the broker the message is ignored. .. note:: If currently no worker is available for a known service, the message is queued for later delivery. If a worker is available for the requested service, the message is repackaged and sent to the worker. The worker in question is removed from the pool of available workers. If the service name starts with `mmi.`, the message is passed to the internal MMI_ handler. .. _MMI: http://rfc.zeromq.org/spec:8 :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ ## print 'client message:' ## pprint(msg) service = msg.pop(0) if service.startswith(b'mmi.'): self.on_mmi(rp, service, msg) return try: wq, wr = self._services[service] wid = wq.get() if not wid: # no worker ready # queue message msg.insert(0, service) wr.append((proto, rp, msg)) return wrep = self._workers[wid] to_send = [ wrep.id, b'', self.WORKER_PROTO, b'\x02'] to_send.extend(rp) to_send.append(b'') to_send.extend(msg) self.main_stream.send_multipart(to_send) except KeyError: # unknwon service # ignore request print 'broker has no service "%s"' % service return def on_worker(self, proto, rp, msg): """Method called on worker message. Frame 0 of msg is the command id. The remaining frames depend on the command. This method determines the command sent by the worker and calls the appropriate method. If the command is unknown the message is ignored and a DISCONNECT is sent. :param proto: the protocol id sent :type proto: str :param rp: return address stack :type rp: list of str :param msg: message parts :type msg: list of str :rtype: None """ cmd = msg.pop(0) if cmd in self._worker_cmds: fnc = self._worker_cmds[cmd] fnc(rp, msg) else: # ignore unknown command # DISCONNECT worker self.disconnect(rp[0]) return def on_message(self, msg): """Processes given message. Decides what kind of message it is -- client or worker -- and calls the appropriate method. If unknown, the message is ignored. :param msg: message parts :type msg: list of str :rtype: None """ rp, msg = split_address(msg) # dispatch on first frame after path t = msg.pop(0) if t.startswith(b'MDPW'): self.on_worker(t, rp, msg) elif t.startswith(b'MDPC'): self.on_client(t, rp, msg) else: print 'Broker unknown Protocol: "%s"' % t return
class GlinApp: """Main Class for Management""" def __init__(self, led_count, hw_backend, port=6606): self.ctx = zmq.Context() self.led_count = led_count self.port = port self.loop = IOLoop.instance() self.caller = PeriodicCallback(self._on_next_frame, 1000/30) self.hw_communication = hw_backend self.hw_communication.connect() self.zmq_collector = GlinAppZmqCollector(self, self.ctx) self.zmq_publisher = GlinAppZmqPublisher(self, self.ctx) # server side configuration self.config = SimpleNamespace() self.config.max_fps = 60 # current state (somehow client side configuration) self.state = SimpleNamespace() self.state.animationClasses = [] self.state.activeSceneId = None self.state.activeAnimation = None self.state.scenes = {} self.state.brightness = 1.0 self.state.sceneIdCtr = 0 self.state.mainswitch = True self.state.target_fps = 0 self.state.lastFrameSent = None def set_brightness(self, brightness): """set general brightness in range 0...1""" brightness = min([1.0, max([brightness, 0.0])]) # enforces range 0 ... 1 self.state.brightness = brightness self._repeat_last_frame() sequence_number = self.zmq_publisher.publish_brightness(brightness) logging.debug("Set brightness to {brightPercent:05.1f}%".format(brightPercent=brightness*100)) return (True, sequence_number, "OK") def register_animation(self, animation_class): """Add a new animation""" self.state.animationClasses.append(animation_class) return len(self.state.animationClasses) - 1 def add_scene(self, animation_id, name, color, velocity, config): """Add a new scene, returns Scene ID""" # check arguments if animation_id < 0 or animation_id >= len(self.state.animationClasses): err_msg = "Requested to register scene with invalid Animation ID. Out of range." logging.info(err_msg) return(False, 0, err_msg) if self.state.animationClasses[animation_id].check_config(config) is False: err_msg = "Requested to register scene with invalid configuration." logging.info(err_msg) return(False, 0, err_msg) self.state.sceneIdCtr += 1 self.state.scenes[self.state.sceneIdCtr] = Scene(animation_id, name, color, velocity, config) sequence_number = self.zmq_publisher.publish_scene_add(self.state.sceneIdCtr, animation_id, name, color, velocity, config) logging.debug("Registered new scene.") # set this scene as active scene if none is configured yet if self.state.activeSceneId is None: self.set_scene_active(self.state.sceneIdCtr) return (True, sequence_number, "OK") def remove_scene(self, scene_id): """remove a scene by Scene ID""" if self.state.activeSceneId == scene_id: err_msg = "Requested to delete scene {sceneNum}, which is currently active. Cannot delete active scene.".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) try: del self.state.scenes[scene_id] logging.debug("Deleted scene {sceneNum}".format(sceneNum=scene_id)) except KeyError: err_msg = "Requested to delete scene {sceneNum}, which does not exist".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) # if we are here, we deleted a scene, so publish it sequence_number = self.zmq_publisher.publish_scene_remove(scene_id) logging.debug("Removed scene {sceneNum}".format(sceneNum=scene_id)) return (True, sequence_number, "OK") def set_scene_name(self, scene_id, name): """rename a scene by scene ID""" if not scene_id in self.state.scenes: # does that scene_id exist? err_msg = "Requested to rename scene {sceneNum}, which does not exist".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) self.state.scenes[scene_id] = self.state.scenes[scene_id]._replace(name=name) # TODO: is there a better solution? sequence_number = self.zmq_publisher.publish_scene_name(scene_id, name) logging.debug("Renamed scene {sceneNum}".format(sceneNum=scene_id)) return (True, sequence_number, "OK") def set_scene_config(self, scene_id, config): """reconfigure a scene by scene ID""" if not scene_id in self.state.scenes: # does that scene_id exist? err_msg = "Requested to reconfigure scene {sceneNum}, which does not exist".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) if scene_id == self.state.activeSceneId: pass # TODO: maybe calculate next frame, esp. if static scene self.state.scenes[scene_id] = self.state.scenes[scene_id]._replace(config=config) sequence_number = self.zmq_publisher.publish_scene_config(scene_id, config) logging.debug("Reconfigured scene {sceneNum}".format(sceneNum=scene_id)) return (True, sequence_number, "OK") def set_scene_color(self, scene_id, color): """reconfigure a scene by scene ID""" if not scene_id in self.state.scenes: # does that scene_id exist? err_msg = "Requested to recolor scene {sceneNum}, which does not exist".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) self.state.scenes[scene_id] = self.state.scenes[scene_id]._replace(color=color) sequence_number = self.zmq_publisher.publish_scene_color(scene_id, color) logging.debug("Recolored scene {sceneNum}".format(sceneNum=scene_id)) if scene_id == self.state.activeSceneId: self.state.activeAnimation.set_color(color) self._do_next_frame() # TODO: make it more sensible, e.g. call only if static scene return (True, sequence_number, "OK") def set_scene_velocity(self, scene_id, velocity): """reconfigure a scene by scene ID""" if not scene_id in self.state.scenes: # does that scene_id exist? err_msg = "Requested to set velocity on scene {sceneNum}, which does not exist".format(sceneNum=scene_id) logging.info(err_msg) return(False, 0, err_msg) self.state.scenes[scene_id] = self.state.scenes[scene_id]._replace(velocity=velocity) sequence_number = self.zmq_publisher.publish_scene_velocity(scene_id, velocity) logging.debug("set velocity on scene {sceneNum}".format(sceneNum=scene_id)) if scene_id == self.state.activeSceneId: self.state.activeAnimation.set_velocity(velocity) self._do_next_frame() # TODO: make it more sensible, e.g. call only if static scene return (True, sequence_number, "OK") def set_scene_active(self, scene_id): """sets the active scene by scene ID""" if self.state.activeSceneId != scene_id: # do nothing if scene has not changed self._deactivate_scene() sequence_number = self.zmq_publisher.publish_active_scene(scene_id) self.state.activeSceneId = scene_id if self.state.mainswitch is True: # activate scene only if we are switched on self._activate_scene() logging.debug("Set scene {sceneNum} as active scene".format(sceneNum=scene_id)) return (True, sequence_number, "OK") else: logging.debug("Scene {sceneNum} already is active scene".format(sceneNum=scene_id)) return (False, 0, "This already is the activated scene.") def set_mainswitch_state(self, state): """Turns output on or off. Also turns hardware on ir off""" if self.state.mainswitch == state: err_msg = "MainSwitch unchanged, already is {sState}".format(sState="On" if state else "Off") # fo obar lorem ipsum logging.debug(err_msg) # fo obar lorem ipsum return (False, 0, err_msg) # because nothing changed self.state.mainswitch = state sequence_number = self.zmq_publisher.publish_mainswitch_state(state) logging.debug("MainSwitch toggled, new state is {sState}".format(sState="On" if state else "Off")) # fo obar lorem ipsum if state is True: self.hw_communication.switch_on() self._activate_scene() # reinit scene else: self._deactivate_scene() self.hw_communication.switch_off() return (True, sequence_number, "OK") def toggle_mainswitch_state(self): """Toggles the mainswitch state""" return self.set_mainswitch_state(not self.state.mainswitch) def _activate_scene(self): if self.state.activeSceneId in self.state.scenes: # is scene_id valid? if not, assume there is no scene configured animation_class = self.state.animationClasses[self.state.scenes[self.state.activeSceneId].animation_id] self.state.activeAnimation = animation_class() target_fps = min(self.config.max_fps, self.state.activeAnimation.get_max_fps(), self.hw_communication.get_max_fps()) if target_fps < 0: target_fps = 0 self.state.target_fps = target_fps logging.debug("Running with {fps} FPS".format(fps=target_fps)) self.state.activeAnimation.prepare(self.led_count, target_fps) self.state.activeAnimation.set_color(self.state.scenes[self.state.activeSceneId].color) self.state.activeAnimation.set_velocity(self.state.scenes[self.state.activeSceneId].velocity) self.state.activeAnimation.set_config(self.state.scenes[self.state.activeSceneId].config) if target_fps > 0: # 0 FPS means one-shot -> no periodic callback required self.caller.callback_time = 1000/target_fps self.caller.start() self.loop.add_callback_from_signal(self._do_next_frame) # execute once to not have to wait for periodic callback (self.caller), esp. if 0 or low FPS else: self.state.activeAnimation = None # don't do anything. stuck with last frame. def _deactivate_scene(self): if not self.state.activeAnimation is None: self.caller.stop() # stop rendering new frames self.state.activeAnimation.finish() self.state.activeAnimation = None def _on_next_frame(self): logging.debug("generating next frame") self._do_next_frame() def _do_next_frame(self): if self.state.activeAnimation: buf = np.zeros((self.led_count, 3)) self.state.activeAnimation.render_next_frame(buf) self.state.last_buf = np.copy(buf) self._send_frame(buf) else: logging.debug("app: No Active Animation") def _repeat_last_frame(self): # only do something, if there is an active animation, else output is considered to be turned off if hasattr(self.state, 'last_buf') and self.state.last_buf is not None and self.state.activeAnimation is not None: if self.state.target_fps < self.config.max_fps / 4: # to not overload hwbackend, only resend, if active animation is very slow self._send_frame(np.copy(self.state.last_buf)) def _send_frame(self, buf): np.clip(buf, 0.0, 1.0, out=buf) self.state.lastFrameSent = datetime.datetime.now() buf *= self.state.brightness self.hw_communication.send(buf) def execute(self): """Execute Main Loop""" try: logging.debug("Entering IOLoop") self.loop.start() logging.debug("Leaving IOLoop") except KeyboardInterrupt: logging.debug("Leaving IOLoop by KeyboardInterrupt") finally: self.hw_communication.disconnect()
class DkvServer(object): """ Dkv Server object """ ctx = None # Context wrapper kvmap = None # Key-value store sequence = 0 # How many updates so far #bstar = None #router = None # Router socket used for DKV publisher = None # Publish updates and hugz collector = None # Collect updates from clients pending = None # Pending updates from client primary = False # True if we're primary master = False # True if we're master slave = False # True if we're slave _debug = False def add_peer(self, peer): log.info('DkvServer: Adding Peer %s.', peer.uuid) uuid = peer.uuid self.peers[uuid] = peer def remove_peer(self, peer): log.info('DkvServer: Removing Peer %s.', peer.uuid) uuid = peer.uuid del self.peers[uuid] @property def router_endpoint(self): return 'tcp://%s:%d' % (self.service_addr, self.port) def __init__(self, service_addr='*', port=conf.ports.dkv): self.service_addr = service_addr self.port = port if not self.ctx: self.ctx = zmq.Context.instance() self.loop = IOLoop.instance() # Base init self.peers = {} self.pending = [] """TODO Automatic voting mechanism to pick primary""" if conf.hostname == 'san0': remote_host = 'san1' primary = True elif conf.hostname == 'san1': remote_host = 'san0' primary = False self.remote_host = remote_host self.primary = primary if primary: self.kvmap = {} bstar_local_ep = 'tcp://*:%d' % conf.ports.bstar_primary bstar_remote_ep = 'tcp://%s:%d' % (remote_host, conf.ports.bstar_secondary) else: bstar_local_ep = 'tcp://*:%d' % conf.ports.bstar_secondary bstar_remote_ep = 'tcp://%s:%d' % (remote_host, conf.ports.bstar_primary) # Setup router socket #self.router = self.ctx.socket(zmq.ROUTER) #self.router.bind(self.router_endpoint) #self.router = ZMQStream(self.router) #self.router.on_recv_stream(self.handle_snapshot) self.bstar = BinaryStar(primary, bstar_local_ep, bstar_remote_ep) self.bstar.register_voter(self.router_endpoint, zmq.ROUTER, self.handle_snapshot) # Register state change handlers self.bstar.master_callback = self.become_master self.bstar.slave_callback = self.become_slave # Set up our dkv server sockets self.publisher = self.ctx.socket(zmq.PUB) address = self.publisher_endpoint log.debug('Binding publisher on %s', address) self.publisher.bind(address) #self.publisher = ZMQStream(self.publisher) self.collector = self.ctx.socket(zmq.SUB) self.collector.setsockopt(zmq.SUBSCRIBE, b'') #self.collector.setsockopt(zmq.SUBSCRIBE, '/clients/updates') address = self.collector_endpoint log.debug('Binding collector on %s', address) self.collector.bind(address) self.collector = ZMQStream(self.collector) self.collector.on_recv(self.handle_collect) # Register our handlers with reactor self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) self.hugz_callback = PeriodicCallback(self.send_hugz, 1000) @property def collector_port(self): return conf.ports.dkv_collector @property def publisher_port(self): return conf.ports.dkv_publisher @property def collector_endpoint(self): return 'tcp://%s:%d' % (self.service_addr, conf.ports.dkv_collector) @property def publisher_endpoint(self): return 'tcp://%s:%d' % (self.service_addr, conf.ports.dkv_publisher) def start(self, loop=True): log.debug('DkvServer starting') # start periodic callbacks self.flush_callback.start() self.hugz_callback.start() # Start bstar reactor until process interrupted self.bstar.start(loop=loop) #if loop: # self.loop.start() def handle_snapshot(self, socket, msg): """snapshot requests""" #log.debug('handle_snapshot: Got msg=%s', msg[1:]) if msg[1] != "ICANHAZ?" or len(msg) != 3: log.error("bad request, aborting") dump(msg) #self.bstar.loop.stop() return identity, request = msg[:2] if len(msg) >= 3: subtree = msg[2] # Send state snapshot to client route = Route(socket, identity, subtree) # For each entry in kvmap, send kvmsg to client for k, v in self.kvmap.items(): send_single(k, v, route) # Now send END message with sequence number log.info("Sending state shapshot=%d", self.sequence) socket.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree kvmsg.send(socket) def handle_collect(self, msg): """Collect updates from clients If we're master, we apply these to the kvmap If we're slave, or unsure, we queue them on our pending list """ if self._debug: log.debug('msg=%s', msg) #if len(msg) != 5: # log.info('handle_collect: Got bad message %s.', msg) # return kvmsg = KVMsg.from_msg(msg) if self.master: self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = kvmsg.get('ttl') if ttl: kvmsg['ttl'] = time.time() + ttl kvmsg.store(self.kvmap) log.info("publishing update=%d", self.sequence) else: # If we already got message from master, drop it, else # hold on pending list if not self.was_pending(kvmsg): self.pending.append(kvmsg) def was_pending(self, kvmsg): """If message was already on pending list, remove and return True. Else return False. """ found = False for idx, held in enumerate(self.pending): if held.uuid == kvmsg.uuid: found = True break if found: self.pending.pop(idx) return found def flush_ttl(self): """Purge ephemeral values that have expired""" if self.kvmap: for key, kvmsg in self.kvmap.items(): self.flush_single(kvmsg) def flush_single(self, kvmsg): """If key-value pair has expired, delete it and publish the fact to listening clients.""" ttl = kvmsg.get('ttl') if ttl and ttl <= time.time(): kvmsg.body = "" self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) del self.kvmap[kvmsg.key] log.debug("publishing delete=%d", self.sequence) def send_hugz(self): """Send hugz to anyone listening on the publisher socket""" if self._debug: log.debug('Sending HUGZ to publisher') kvmsg = KVMsg(self.sequence) kvmsg.key = "HUGZ" kvmsg.body = "" kvmsg.send(self.publisher) """ State change handlers """ def become_master(self): """We're becoming master The backup server applies its pending list to its own hash table, and then starts to process state snapshot requests. """ log.info('Becoming master') self.master = True self.slave = False # stop receiving subscriber updates while we are master for peer in self.peers.values(): peer.subscriber.stop_on_recv() # Apply pending list to own kvmap while self.pending: kvmsg = self.pending.pop(0) self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.store(self.kvmap) log.info("publishing pending=%d", self.sequence) def become_slave(self): """We're becoming slave""" log.info('Becoming slave') self.kvmap = None # clear kvmap self.master = False self.slave = True # start receiving subscriber updates for peer in self.peers.values(): peer.subscriber.on_recv(peer._on_subscriber_recv) def handle_subscriber(self, peer, msg): """Collect updates from peer (master) We're always slave when we get these updates """ if self.master: if msg[0] != 'HUGZ': log.warn( "Received subscriber message, but we are master msg=%s from_peer=%s", msg, peer.uuid) return # Get state snapshot if necessary if self.kvmap is None: self.kvmap = {} snapshot = peer.get_snapshot() while True: try: kvmsg = KVMsg.recv(snapshot) except KeyboardInterrupt: #self.bstar.loop.stop() self.loop.stop() return if kvmsg.key == "KTHXBAI": self.sequence = kvmsg.sequence break # Done kvmsg.store(self.kvmap) log.info("received snapshot=%d", self.sequence) ##if not isinstance(msg, (tuple, list)) or len(msg) < 5: #if not len(msg) == 5: # log.debug('bad msg=%s', msg) # return # Find and remove update off pending list kvmsg = KVMsg.from_msg(msg) # update integer ttl -> timestamp ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + ttl if kvmsg.key != "HUGZ": if not self.was_pending(kvmsg): """ If master update came before client update, flip it around, store master update (with sequence) on pending list and use to clear client update when it comes later. """ self.pending.append(kvmsg) # If update is more recent than our kvmap, apply it if (kvmsg.sequence > self.sequence): self.sequence = kvmsg.sequence kvmsg.store(self.kvmap) log.info("received update=%d", self.sequence)
class ProxyHandler (BaseProxyHandler): """ Simply proxies further """ log = skytools.getLogger ('h:ProxyHandler') ping_tick = 1 def __init__ (self, hname, hcf, ccscript): super(ProxyHandler, self).__init__(hname, hcf, ccscript) self.ping_remote = self.cf.getbool ("ping", False) if self.ping_remote: self.echo_stats = EchoState (self.remote_url) self.echo_timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop) self.echo_timer.start() self.log.debug ("will ping %s", self.remote_url) def on_recv (self, zmsg): """ Got message from remote CC, process it. """ try: # pongs to our pings should come back w/o any routing info if self.ping_remote and zmsg[0] == '': self.log.trace ("%r", zmsg) cmsg = CCMessage (zmsg) req = cmsg.get_dest() if req == "echo.response": self._recv_pong (cmsg) else: self.log.warn ("unknown msg: %s", req) except: self.log.exception ("crashed") finally: super(ProxyHandler, self).on_recv(zmsg) def _recv_pong (self, cmsg): """ Pong received, evaluate it. """ msg = cmsg.get_payload (self.xtx) if not msg: return if msg.orig_target != self.remote_url: self.log.warn ("unknown pong: %s", msg.orig_target) return echo = self.echo_stats echo.update_pong (msg) rtt = echo.time_pong - msg.orig_time if msg.orig_time == echo.time_ping: self.log.trace ("echo time: %f s (%s)", rtt, self.remote_url) elif rtt <= 5 * self.ping_tick: self.log.debug ("late pong: %f s (%s)", rtt, self.remote_url) else: self.log.info ("too late pong: %f s (%s)", rtt, self.remote_url) def _send_ping (self): """ Send ping to remote CC. """ msg = EchoRequestMessage( target = self.remote_url) cmsg = self.xtx.create_cmsg (msg) self.stream.send_cmsg (cmsg) self.echo_stats.update_ping (msg) self.log.trace ("%r", msg) def ping (self): """ Echo requesting and monitoring. """ self.log.trace ("") miss = self.echo_stats.time_ping - self.echo_stats.time_pong if miss > 5 * self.ping_tick: self.log.warn ("no pong from %s for %f s", self.remote_url, miss) self._send_ping () def stop (self): super(ProxyHandler, self).stop() self.log.info ("stopping") if hasattr (self, "echo_timer"): self.echo_timer.stop()
class MDPWorker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.XREQ socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = b'MDPW01' # TODO: integrate that into API HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, endpoint, service): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ self.context = context self.endpoint = endpoint self.service = service self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.XREQ) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ ready_msg = [ b'', self._proto_version, chr(1), self.service ] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [ b'', self._proto_version, chr(4) ] self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ ## if self.need_handshake: ## raise ConnectionNotReadyError() # prepare full message to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ # 1st part is empty msg.pop(0) # 2nd part is protocol version # TODO: version check proto = msg.pop(0) # 3nd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if msg_type == '\x05': # disconnect print ' DISC' self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == '\x02': # request # remaining parts are the user message envelope, msg = split_address(msg) envelope.append(b'') envelope = [ b'', self._proto_version, '\x03'] + envelope # REPLY self.envelope = envelope self.on_request(msg) else: # invalid message # ignored pass return def on_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ pass
class CryptoTransportLayer(TransportLayer): def __init__(self, my_ip, my_port, market_id, bm_user=None, bm_pass=None, bm_port=None, seed_mode=0, dev_mode=False): self._log = logging.getLogger('[%s] %s' % (market_id, self.__class__.__name__)) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Connect to database MONGODB_URI = 'mongodb://*****:*****@localhost:{}/".format(bm_user, bm_pass, bm_port), verbose=0) result = self._bitmessage_api.add(2,3) self._log.info("Bitmessage test result: {}, API is live".format(result)) # If we failed, fall back to starting our own except Exception as e: self._log.info("Failed to connect to bitmessage instance: {}".format(e)) self._bitmessage_api = None # self._log.info("Spawning internal bitmessage instance") # # Add bitmessage submodule path # sys.path.insert(0, os.path.join( # os.path.dirname(__file__), '..', 'pybitmessage', 'src')) # import bitmessagemain as bitmessage # bitmessage.logger.setLevel(logging.WARNING) # bitmessage_instance = bitmessage.Main() # bitmessage_instance.start(daemon=True) # bminfo = bitmessage_instance.getApiAddress() # if bminfo is not None: # self._log.info("Started bitmessage daemon at %s:%s".format( # bminfo['address'], bminfo['port'])) # bitmessage_api = xmlrpclib.ServerProxy("http://{}:{}@{}:{}/".format( # bm_user, bm_pass, bminfo['address'], bminfo['port'])) # else: # self._log.info("Failed to start bitmessage dameon") # self._bitmessage_api = None return result def _checkok(self, msg): self._log.info('Check ok') def get_guid(self): return self._guid def getDHT(self): return self._dht def getBitmessageAPI(self): return self._bitmessage_api def getMarketID(self): return self._market_id def getMyself(self): return self._myself def _ping(self, msg): self._log.info('Pinged %s ' % msg) pinger = CryptoPeerConnection(self,msg['uri'], msg['pubkey'], msg['senderGUID']) pinger.send_raw(json.dumps( {"type": "hello_response", "senderGUID": self.guid, "uri": self._uri, "senderNick": self._nickname, "pubkey": self.pubkey, })) def _storeValue(self, msg): self._dht._on_storeValue(msg) def _findNode(self, msg): self._dht.on_find_node(msg) def _findNodeResponse(self, msg): self._dht.on_findNodeResponse(self, msg) def _setup_settings(self): self.settings = self._db.settings.find_one({'id':"%s" % self._market_id}) if self.settings: self._nickname = self.settings['nickname'] if self.settings.has_key("nickname") else "" self.secret = self.settings['secret'] if self.settings.has_key("secret") else "" self.pubkey = self.settings['pubkey'] if self.settings.has_key("pubkey") else "" self.guid = self.settings['guid'] if self.settings.has_key("guid") else "" self.sin = self.settings['sin'] if self.settings.has_key("sin") else "" self.bitmessage = self.settings['bitmessage'] if self.settings.has_key('bitmessage') else "" else: self._nickname = 'Default' # Generate Bitcoin keypair self._generate_new_keypair() # Generate Bitmessage address if self._bitmessage_api is not None: self._generate_new_bitmessage_address() # Generate PGP key gpg = gnupg.GPG() input_data = gpg.gen_key_input(key_type="RSA", key_length=2048, name_comment="Autogenerated by Open Bazaar", passphrase="P@ssw0rd") key = gpg.gen_key(input_data) pubkey_text = gpg.export_keys(key.fingerprint) self._db.settings.update({"id":'%s' % self._market_id}, {"$set": {"PGPPubKey":pubkey_text, "PGPPubkeyFingerprint":key.fingerprint}}, True) self.settings = self._db.settings.find_one({'id':"%s" % self._market_id}) self._log.debug('Retrieved Settings: %s', self.settings) def _generate_new_keypair(self): # Generate new keypair key = ec.ECC(curve='secp256k1') self.secret = key.get_privkey().encode('hex') pubkey = key.get_pubkey() signedPubkey = key.sign(pubkey) self.pubkey = pubkey.encode('hex') self._myself = key # Generate SIN sha_hash = hashlib.sha256() sha_hash.update(pubkey) ripe_hash = hashlib.new('ripemd160') ripe_hash.update(sha_hash.digest()) self.guid = ripe_hash.digest().encode('hex') self.sin = obelisk.EncodeBase58Check('\x0F\x02%s' + ripe_hash.digest()) self._db.settings.update({"id":'%s' % self._market_id}, {"$set": {"secret":self.secret, "pubkey":self.pubkey, "guid":self.guid, "sin":self.sin}}, True) def _generate_new_bitmessage_address(self): # Use the guid generated previously as the key self.bitmessage = self._bitmessage_api.createRandomAddress(self.guid.encode('base64'), False, 1.05, 1.1111) self._db.settings.update({"id":'%s' % self._market_id}, {"$set": {"bitmessage":self.bitmessage}}, True) def join_network(self, dev_mode=0, callback=lambda msg: None): if dev_mode: self._log.info('DEV MODE') seed_peers = {'127.0.0.1'} else: seed_peers = ('seed.openbazaar.org', 'seed2.openbazaar.org') for seed in seed_peers: self._log.info('Initializing Seed Peer(s): [%s]' % seed) def cb(msg): self._dht._iterativeFind(self._guid, self._dht._knownNodes, 'findNode') callback(msg) self.connect('tcp://%s:12345' % seed, callback=cb) # Try to connect to known peers known_peers = self._db.peers.find() for known_peer in known_peers: def cb(msg): #self._dht._iterativeFind(self._guid, self._dht._knownNodes, 'findNode') callback(msg) self.connect(known_peer['uri'], callback=cb) # self.listen(self.pubkey) # Turn on zmq socket # # if seed_uri: # self._log.info('Initializing Seed Peer(s): [%s]' % seed_uri) # seed_peer = CryptoPeerConnection(self, seed_uri) # self._dht.start(seed_peer) def connect(self, uri, callback): def cb(msg): ip = urlparse(uri).hostname port = urlparse(uri).port self._dht.add_known_node((ip, port, peer._guid)) # Turning off peers #self._init_peer({'uri': seed_uri, 'guid':seed_guid}) # Add to routing table #self.addCryptoPeer(peer) callback(msg) peer = CryptoPeerConnection(self, uri, callback=cb) return peer def get_crypto_peer(self, guid, uri, pubkey=None, nickname=None): if guid == self.guid: self._log.info('Trying to get crypto peer for yourself') return self._log.info('%s %s %s %s' % (guid, uri, pubkey, nickname)) peer = CryptoPeerConnection(self, uri, pubkey, guid=guid, nickname=nickname) return peer def addCryptoPeer(self, peer_to_add): foundOutdatedPeer = False for idx, peer in enumerate(self._dht._activePeers): if (peer._address, peer._guid, peer._pub) == (peer_to_add._address, peer_to_add._guid, peer_to_add._pub): self._log.info('Found existing peer, not adding.') return if peer._guid == peer_to_add._guid or peer._pub == peer_to_add._pub or peer._address == peer_to_add._address: foundOutdatedPeer = True self._log.info('Found an outdated peer') # Update existing peer self._activePeers[idx] = peer_to_add if not foundOutdatedPeer and peer_to_add._guid != self._guid: self._log.info('Adding crypto peer at %s' % peer_to_add._nickname) self._dht.add_active_peer(self, (peer_to_add._pub, peer_to_add._address, peer_to_add._guid, peer_to_add._nickname)) # Return data array with details from the crypto file # TODO: This needs to be protected better; potentially encrypted file or DB @staticmethod def load_crypto_details(store_file): with open(store_file) as f: data = json.loads(f.read()) assert "nickname" in data assert "secret" in data assert "pubkey" in data assert len(data["secret"]) == 2 * 32 assert len(data["pubkey"]) == 2 * 33 return data["nickname"], data["secret"].decode("hex"), \ data["pubkey"].decode("hex") def get_profile(self): peers = {} self.settings = self._db.settings.find_one({'id':"%s" % self._market_id}) for uri, peer in self._peers.iteritems(): if peer._pub: peers[uri] = peer._pub.encode('hex') return {'uri': self._uri, 'pub': self._myself.get_pubkey().encode('hex'),'nickname': self._nickname, 'peers': peers} def respond_pubkey_if_mine(self, nickname, ident_pubkey): if ident_pubkey != self.pubkey: self._log.info("Public key does not match your identity") return # Return signed pubkey pubkey = self._myself.pubkey ec_key = obelisk.EllipticCurveKey() ec_key.set_secret(self.secret) digest = obelisk.Hash(pubkey) signature = ec_key.sign(digest) # Send array of nickname, pubkey, signature to transport layer self.send(proto_response_pubkey(nickname, pubkey, signature)) def pubkey_exists(self, pub): for uri, peer in self._peers.iteritems(): self._log.info('PEER: %s Pub: %s' % (peer._pub.encode('hex'), pub.encode('hex'))) if peer._pub.encode('hex') == pub.encode('hex'): return True return False def create_peer(self, uri, pub, node_guid): if pub: pub = pub.decode('hex') # Create the peer if public key is not already in the peer list # if not self.pubkey_exists(pub): self._peers[uri] = CryptoPeerConnection(self, uri, pub, node_guid) # Call 'peer' callbacks on listeners self.trigger_callbacks('peer', self._peers[uri]) # else: # print 'Pub Key is already in peer list' def send(self, data, send_to=None, callback=lambda msg: None): self._log.debug("Outgoing Data: %s %s" % (data, send_to)) # Directed message if send_to is not None: peer = self._dht._routingTable.getContact(send_to) #peer = CryptoPeerConnection(msg['uri']) if peer: self._log.debug('Directed Data (%s): %s' % (send_to, data)) try: peer.send(data, callback=callback) except: self._log.error('Not sending messing directly to peer') else: self._log.error('No peer found') else: # FindKey and then send for peer in self._dht._activePeers: try: peer = self._dht._routingTable.getContact(peer._guid) data['senderGUID'] = self._guid data['pubkey'] = self.pubkey print data #if peer._pub: # peer.send(data, callback) #else: def cb(msg): print 'msg %s' % msg peer.send(data, cb) except: self._log.info("Error sending over peer!") traceback.print_exc() def send_enc(self, uri, msg): peer = self._peers[uri] pub = peer._pub # Now send a hello message to the peer if pub: self._log.info("Sending encrypted [%s] message to %s" % (msg['type'], uri)) peer.send(msg) else: # Will send clear profile on initial if no pub self._log.info("Sending unencrypted [%s] message to %s" % (msg['type'], uri)) self._peers[uri].send_raw(json.dumps(msg)) def _init_peer(self, msg): uri = msg['uri'] pub = msg.get('pub') nickname = msg.get('nickname') msg_type = msg.get('type') guid = msg['guid'] if not self.valid_peer_uri(uri): self._log.error("Invalid Peer: %s " % uri) return if uri not in self._peers: # Unknown peer self._log.info('Add New Peer: %s' % uri) self.create_peer(uri, pub, guid) if not msg_type: self.send_enc(uri, hello_request(self.get_profile())) elif msg_type == 'hello_request': self.send_enc(uri, hello_response(self.get_profile())) else: # Known peer if pub: # test if we have to update the pubkey if not self._peers[uri]._pub: self._log.info("Setting public key for seed node") self._peers[uri]._pub = pub.decode('hex') self.trigger_callbacks('peer', self._peers[uri]) if self._peers[uri]._pub != pub.decode('hex'): self._log.info("Updating public key for node") self._peers[uri]._nickname = nickname self._peers[uri]._pub = pub.decode('hex') self.trigger_callbacks('peer', self._peers[uri]) if msg_type == 'hello_request': # reply only if necessary self.send_enc(uri, hello_response(self.get_profile())) def _on_message(self, msg): # here goes the application callbacks # we get a "clean" msg which is a dict holding whatever #self._log.info("[On Message] Data received: %s" % msg) pubkey = msg.get('pubkey') uri = msg.get('uri') ip = urlparse(uri).hostname port = urlparse(uri).port guid = msg.get('senderGUID') nickname = msg.get('senderNick') self._log.info('on message %s' % nickname) self._dht.add_known_node((ip, port, guid, nickname)) self._dht.add_active_peer(self, (pubkey, uri, guid, nickname)) self.trigger_callbacks(msg['type'], msg) def _on_raw_message(self, serialized): try: # Try to de-serialize clear text message msg = json.loads(serialized) self._log.info("Message Received [%s]" % msg.get('type', 'unknown')) if msg.get('type') is None: data = msg.get('data').decode('hex') sig = msg.get('sig').decode('hex') try: data = self._myself.decrypt(data) self._log.debug('Signature: %s' % sig.encode('hex')) self._log.debug('Signed Data: %s' % data) guid = json.loads(data).get('guid') peer = self._dht._routingTable.getContact(guid) ecc = ec.ECC(curve='secp256k1',pubkey=json.loads(data).get('pubkey').decode('hex')) # Check signature if ecc.verify(sig, data): self._log.info('Verified') else: self._log.error('Message signature could not be verified %s' % msg) return msg = json.loads(data) self._log.debug('Message Data %s ' % msg) except: self._log.error('Could not decrypt message properly') except ValueError: try: # Encrypted? try: msg = self._myself.decrypt(serialized) msg = json.loads(msg) self._log.info("Decrypted Message [%s]" % msg.get('type', 'unknown')) except: self._log.error("Could not decrypt message: %s" % msg) return except: self._log.error('Message probably sent using incorrect pubkey') return if msg.get('type') is not None: msg_type = msg.get('type') msg_uri = msg.get('uri') msg_guid = msg.get('guid') self._on_message(msg) else: self._log.error('Received a message with no type')
class TailWriter_Worker (threading.Thread): """ Simply appends to files """ log = skytools.getLogger ('h:TailWriter_Worker') def __init__ (self, name, xtx, zctx, ioloop, dealer_url, router_url, params = {}): super(TailWriter_Worker, self).__init__(name=name) self.log = skytools.getLogger ('h:TailWriter_Worker' + name[name.rfind('-'):]) #self.log = skytools.getLogger (self.log.name + name[name.rfind('-'):]) #self.log = skytools.getLogger (name) self.xtx = xtx self.zctx = zctx self.ioloop = ioloop self.shared_url = dealer_url self.direct_url = router_url for k, v in params.items(): self.log.trace ("setattr: %s -> %r", k, v) setattr (self, k, v) self.files = {} self.looping = True def startup (self): # announce channel (for new files) self.sconn = self.zctx.socket (zmq.XREP) self.sconn.connect (self.shared_url) # direct channel (for grabbed files) self.dconn = self.zctx.socket (zmq.XREQ) self.dconn.connect (self.direct_url) # polling interface self.poller = zmq.Poller() self.poller.register (self.sconn, zmq.POLLIN) self.poller.register (self.dconn, zmq.POLLIN) # schedule regular maintenance self.timer_maint = PeriodicCallback (self.do_maint, self.maint_period * 1000, self.ioloop) self.timer_maint.start() def run (self): self.log.info ("%s running", self.name) self.startup() while self.looping: try: self.work() except: self.log.exception ("worker crashed, dropping msg") self.shutdown() def work (self): socks = dict (self.poller.poll (1000)) if self.dconn in socks and socks[self.dconn] == zmq.POLLIN: zmsg = self.dconn.recv_multipart() elif self.sconn in socks and socks[self.sconn] == zmq.POLLIN: zmsg = self.sconn.recv_multipart() else: # timeout return try: cmsg = CCMessage (zmsg) except: self.log.exception ("invalid CC message") else: self.handle_msg (cmsg) def handle_msg (self, cmsg): """ Got message from master, process it. """ data = cmsg.get_payload (self.xtx) if not data: return mode = data['mode'] host = data['hostname'] fn = os.path.basename (data['filename']) op_mode = data.get('op_mode') st_dev = data.get('st_dev') st_ino = data.get('st_ino') # let master know asap :-) self._send_ack (host, st_dev, st_ino, data['filename'], data.get('fpos')) # sanitize host = host.replace ('/', '_') if mode not in ['', 'b']: self.log.warning ("unsupported fopen mode (%r), ignoring it", mode) mode = 'b' if op_mode not in [None, '', 'classic', 'rotated']: self.log.warning ("unsupported operation mode (%r), ignoring it", op_mode) op_mode = None # add file ext if needed if self.write_compressed == 'keep': if data['comp'] not in [None, '', 'none']: fn += comp_ext[data['comp']] elif self.write_compressed == 'yes': fn += comp_ext[self.compression] # Cache open files fi = (host, st_dev, st_ino, fn) if fi in self.files: fd = self.files[fi] if mode != fd['mode']: self.log.error ("fopen mode mismatch (%s -> %s)", mode, fd['mode']) return if op_mode != fd['op_mode']: self.log.error ("operation mode mismatch (%s -> %s)", op_mode, fd['op_mode']) return else: # decide destination file if self.host_subdirs: subdir = os.path.join (self.dstdir, host) dstfn = os.path.join (subdir, fn) if not os.path.isdir (subdir): os.mkdir (subdir) else: dstfn = os.path.join (self.dstdir, '%s--%s' % (host, fn)) if op_mode == 'rotated': dt = datetime.datetime.today() dstfn += dt.strftime (DATETIME_SUFFIX) fobj = open (dstfn, 'a' + mode) self.log.info ('opened %s', dstfn) now = time.time() fd = { 'obj': fobj, 'mode': mode, 'path': dstfn, 'wtime': now, 'ftime': now, 'buf': [], 'bufsize': 0, 'offset': 0, 'op_mode': op_mode } self.files[fi] = fd raw = cmsg.get_part3() # blob if not raw: raw = data['data'].decode('base64') if self.write_compressed in [None, '', 'no']: if data['comp'] not in (None, '', 'none'): body = cc.util.decompress (raw, data['comp']) self.log.debug ("decompressed from %i to %i", len(raw), len(body)) else: body = raw elif self.write_compressed == 'keep': body = raw elif self.write_compressed == 'yes': if (data['comp'] != self.compression): deco = cc.util.decompress (raw, data['comp']) fd['buf'].append(deco) fd['bufsize'] += len(deco) if fd['bufsize'] < self.buf_maxbytes: return body = self._process_buffer(fd) else: body = raw if hasattr (data, 'fpos') and (self.write_compressed in [None, '', 'no'] or (self.write_compressed == 'keep' and data['comp'] in [None, '', 'none'])): fpos = fd['obj'].tell() if data['fpos'] != fpos + fd['offset']: self.log.warning ("sync lost: %i -> %i", fpos, data['fpos']) fd['offset'] = data['fpos'] - fpos # append to file self.log.debug ('appending %i bytes to %s', len(body), fd['path']) fd['obj'].write (body) fd['wtime'] = time.time() self.stat_inc ('appended_bytes', len(body)) def _process_buffer (self, fd): """ Compress and reset write buffer """ buf = ''.join(fd['buf']) out = cc.util.compress (buf, self.compression, {'level': self.compression_level}) self.log.debug ("compressed from %i to %i", fd['bufsize'], len(out)) fd['buf'] = [] fd['bufsize'] = 0 return out def _send_ack (self, hostname, st_dev, st_ino, filename, fpos): """ Send ack to master """ rep = ReplyMessage( worker = self.name, d_hostname = hostname, d_st_dev = st_dev, d_st_ino = st_ino, d_filename = filename, d_fpos = fpos) rcm = self.xtx.create_cmsg (rep) rcm.send_to (self.dconn) def do_maint (self): """ Close long-open files; flush inactive files. """ self.log.trace ('cleanup') now = time.time() zombies = [] for k, fd in self.files.iteritems(): if now - fd['wtime'] > CLOSE_DELAY: if fd['buf']: body = self._process_buffer(fd) fd['obj'].write(body) fd['obj'].close() self.log.info ('closed %s', fd['path']) zombies.append(k) elif (fd['wtime'] > fd['ftime']) and (now - fd['wtime'] > FLUSH_DELAY): # note: think about small writes within flush period fd['obj'].flush() self.log.debug ('flushed %s', fd['path']) fd['ftime'] = now for k in zombies: self.files.pop(k) def stop (self): self.looping = False def shutdown (self): """ Close all open files """ self.log.info ('%s stopping', self.name) for fd in self.files.itervalues(): if fd['buf']: body = self._process_buffer(fd) fd['obj'].write(body) fd['obj'].close() self.log.info ('closed %s', fd['path'])
class Echo(CCHandler): """ Echo handler / sender / monitor """ CC_ROLES = ['local', 'remote'] log = skytools.getLogger('h:Echo') ping_tick = 1 zmq_hwm = 1 zmq_linger = 0 def __init__(self, hname, hcf, ccscript): super(Echo, self).__init__(hname, hcf, ccscript) self.echoes = {} # echo stats for monitored peers self.stream = {} # connections to monitored peers for url in self.cf.getlist("ping-remotes", ""): sock = self._make_socket(url) self.stream[url] = CCStream(sock, ccscript.ioloop, qmaxsize=self.zmq_hwm) self.stream[url].on_recv(self.on_recv) self.echoes[url] = EchoState(url) self.log.debug("will ping %s", url) self.timer = PeriodicCallback(self.ping, self.ping_tick * 1000, self.ioloop) self.timer.start() def _make_socket(self, url): """ Create socket for pinging remote CC. """ sock = self.zctx.socket(zmq.XREQ) try: sock.setsockopt(zmq.HWM, self.zmq_hwm) except AttributeError: sock.set_hwm(self.zmq_hwm) sock.setsockopt(zmq.LINGER, self.zmq_linger) sock.connect(url) return sock def on_recv(self, zmsg): """ Got reply from a remote CC, process it. """ try: self.log.trace("%r", zmsg) cmsg = CCMessage(zmsg) req = cmsg.get_dest() if req == "echo.response": self.process_response(cmsg) else: self.log.warn("unknown msg: %s", req) except: self.log.exception("crashed, dropping msg") def handle_msg(self, cmsg): """ Got a message, process it. """ self.log.trace("%r", cmsg) req = cmsg.get_dest() if req == "echo.request": self.process_request(cmsg) else: self.log.warn("unknown msg: %s", req) def process_request(self, cmsg): """ Ping received, respond with pong. """ msg = cmsg.get_payload(self.xtx) if not msg: return rep = EchoResponseMessage(orig_hostname=msg['hostname'], orig_target=msg['target'], orig_time=msg['time']) rcm = self.xtx.create_cmsg(rep) rcm.take_route(cmsg) rcm.send_to(self.cclocal) def process_response(self, cmsg): """ Pong received, evaluate it. """ msg = cmsg.get_payload(self.xtx) if not msg: return url = msg.orig_target if url not in self.echoes: self.log.warn("unknown pong: %s", url) return echo = self.echoes[url] echo.update_pong(msg) rtt = echo.time_pong - msg.orig_time if msg.orig_time == echo.time_ping: self.log.trace("echo time: %f s (%s)", rtt, url) elif rtt <= 5 * self.ping_tick: self.log.debug("late pong: %f s (%s)", rtt, url) else: self.log.info("too late pong: %f s (%s)", rtt, url) def send_request(self, url): """ Send ping to remote CC. """ msg = EchoRequestMessage(target=url) cmsg = self.xtx.create_cmsg(msg) self.stream[url].send_cmsg(cmsg) self.echoes[url].update_ping(msg) self.log.trace("%r", msg) def ping(self): """ Echo requesting and monitoring. """ self.log.trace("") for url in self.stream: echo = self.echoes[url] if echo.time_ping - echo.time_pong > 5 * self.ping_tick: self.log.warn("no pong from %s for %f s", url, echo.time_ping - echo.time_pong) self.send_request(url) def stop(self): super(Echo, self).stop() self.log.info("stopping") self.timer.stop()
class TailWriter (CCHandler): """ Simply appends to files (with help from workers) """ CC_ROLES = ['remote'] log = skytools.getLogger ('h:TailWriter') def __init__ (self, hname, hcf, ccscript): super(TailWriter, self).__init__(hname, hcf, ccscript) self.files = {} self.workers = [] self.wparams = {} # passed to workers self.wparams['dstdir'] = self.cf.getfile ('dstdir') self.wparams['host_subdirs'] = self.cf.getbool ('host-subdirs', 0) self.wparams['maint_period'] = self.cf.getint ('maint-period', 3) self.wparams['write_compressed'] = self.cf.get ('write-compressed', '') assert self.wparams['write_compressed'] in [None, '', 'no', 'keep', 'yes'] if self.wparams['write_compressed'] in ('keep', 'yes'): self.log.info ("position checking not supported for compressed files") if self.wparams['write_compressed'] == 'yes': self.wparams['compression'] = self.cf.get ('compression', '') if self.wparams['compression'] not in ('gzip', 'bzip2'): self.log.error ("unsupported compression: %s", self.wparams['compression']) self.wparams['compression_level'] = self.cf.getint ('compression-level', '') self.wparams['buf_maxbytes'] = cc.util.hsize_to_bytes (self.cf.get ('buffer-bytes', '1 MB')) if self.wparams['buf_maxbytes'] < BUF_MINBYTES: self.log.info ("buffer-bytes too low, adjusting: %i -> %i", self.wparams['buf_maxbytes'], BUF_MINBYTES) self.wparams['buf_maxbytes'] = BUF_MINBYTES # initialise sockets for communication with workers self.dealer_stream, self.dealer_url = self.init_comm (zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv) self.router_stream, self.router_url = self.init_comm (zmq.XREP, 'inproc://workers-router', self.router_on_recv) self.launch_workers() self.timer_maint = PeriodicCallback (self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop) self.timer_maint.start() def init_comm (self, stype, url, cb): """ Create socket, stream, etc for communication with workers. """ sock = self.zctx.socket (stype) port = sock.bind_to_random_port (url) curl = "%s:%d" % (url, port) stream = CCStream (sock, self.ioloop) stream.on_recv (cb) return (stream, curl) def launch_workers (self): """ Create and start worker threads. """ nw = self.cf.getint ('worker-threads', 10) for i in range (nw): wname = "%s.worker-%i" % (self.hname, i) self.log.info ("starting %s", wname) w = TailWriter_Worker( wname, self.xtx, self.zctx, self.ioloop, self.dealer_url, self.router_url, self.wparams) w.stat_inc = self.stat_inc # XXX self.workers.append (w) w.start() def handle_msg (self, cmsg): """ Got message from client, process it. """ data = cmsg.get_payload (self.xtx) if not data: return host = data['hostname'] fn = data['filename'] st_dev = data.get('st_dev') st_ino = data.get('st_ino') fi = (host, st_dev, st_ino, fn) if fi in self.files: fd = self.files[fi] if fd.waddr: # already accepted ? self.log.trace ("passing %r to %s", fn, fd.wname) fd.queue.append (cmsg) fd.send_to (self.router_stream) else: self.log.trace ("queueing %r", fn) fd.queue.append (cmsg) else: fd = FileState (fi, 1) self.files[fi] = fd self.log.trace ("offering %r", fn) self.dealer_stream.send_cmsg (cmsg) def dealer_on_recv (self, zmsg): """ Got reply from worker via "dealer" connection """ self.log.warning ("reply via dealer: %s", zmsg) def router_on_recv (self, zmsg): """ Got reply from worker via "router" connection """ cmsg = CCMessage (zmsg) data = cmsg.get_payload (self.xtx) fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'], data['d_filename']) fd = self.files[fi] if fd.waddr is None: fd.waddr = zmsg[0] fd.wname = data['worker'] else: assert fd.waddr == zmsg[0] and fd.wname == data['worker'] fd.atime = time.time() fd.count -= 1 assert fd.count >= 0 def do_maint (self): """ Check & flush queues; drop inactive files. """ self.log.trace ('cleanup') now = time.time() zombies = [] for k, fd in self.files.iteritems(): if fd.queue and fd.waddr: self.log.trace ("passing %r to %s", fd.ident, fd.wname) fd.send_to (self.router_stream) if (fd.count == 0) and (now - fd.atime > 2 * CLOSE_DELAY): # you'd better use msg for this self.log.debug ("forgetting %r", fd.ident) zombies.append(k) for k in zombies: self.files.pop(k) def stop (self): """ Signal workers to shut down. """ super(TailWriter, self).stop() self.log.info ('stopping') self.timer_maint.stop() for w in self.workers: self.log.info ("signalling %s", w.name) w.stop()
class CryptoTransportLayer(TransportLayer): def __init__(self, my_ip, my_port, market_id, db, bm_user=None, bm_pass=None, bm_port=None, seed_mode=0, dev_mode=False, disable_ip_update=False): self.log = logging.getLogger( '[%s] %s' % (market_id, self.__class__.__name__) ) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Connect to database self.db = db self.bitmessage_api = None if (bm_user, bm_pass, bm_port) != (None, None, None): if not self._connect_to_bitmessage(bm_user, bm_pass, bm_port): self.log.info('Bitmessage not installed or started') try: socket.inet_pton(socket.AF_INET6, my_ip) my_uri = "tcp://[%s]:%s" % (my_ip, my_port) except (socket.error, ValueError): my_uri = "tcp://%s:%s" % (my_ip, my_port) self.market_id = market_id self.nick_mapping = {} self.uri = my_uri self.ip = my_ip self.nickname = "" self._dev_mode = dev_mode # Set up self._setup_settings() self.dht = DHT(self, self.market_id, self.settings, self.db) # self._myself = ec.ECC(pubkey=self.pubkey.decode('hex'), # privkey=self.secret.decode('hex'), # curve='secp256k1') TransportLayer.__init__(self, market_id, my_ip, my_port, self.guid, self.nickname) self.setup_callbacks() self.listen(self.pubkey) if seed_mode == 0 and not dev_mode and not disable_ip_update: self.start_ip_address_checker() def setup_callbacks(self): self.add_callbacks([('hello', self._ping), ('findNode', self._find_node), ('findNodeResponse', self._find_node_response), ('store', self._store_value)]) def start_ip_address_checker(self): '''Checks for possible public IP change''' self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() def _ip_updater_periodic_callback(self): try: r = requests.get('http://ipv4.icanhazip.com') if r and hasattr(r, 'text'): ip = r.text ip = ip.strip(' \t\n\r') if ip != self.ip: self.ip = ip try: socket.inet_pton(socket.AF_INET6, self.ip) my_uri = 'tcp://[%s]:%s' % (self.ip, self.port) except (socket.error, ValueError): my_uri = 'tcp://%s:%s' % (self.ip, self.port) self.uri = my_uri self.stream.close() self.listen(self.pubkey) self.dht._iterativeFind(self.guid, [], 'findNode') else: self.log.error('Could not get IP') except Exception as e: self.log.error('[Requests] error: %s' % e) def save_peer_to_db(self, peer_tuple): uri = peer_tuple[0] pubkey = peer_tuple[1] guid = peer_tuple[2] nickname = peer_tuple[3] # Update query self.db.deleteEntries("peers", {"uri": uri, "guid": guid}, "OR") # if len(results) > 0: # self.db.updateEntries("peers", {"id": results[0]['id']}, {"market_id": self.market_id, "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname}) # else: if guid is not None: self.db.insertEntry("peers", { "uri": uri, "pubkey": pubkey, "guid": guid, "nickname": nickname, "market_id": self.market_id }) def _connect_to_bitmessage(self, bm_user, bm_pass, bm_port): # Get bitmessage going # First, try to find a local instance result = False try: self.log.info('[_connect_to_bitmessage] Connecting to Bitmessage on port %s' % bm_port) self.bitmessage_api = xmlrpclib.ServerProxy("http://{}:{}@localhost:{}/".format(bm_user, bm_pass, bm_port), verbose=0) result = self.bitmessage_api.add(2, 3) self.log.info("[_connect_to_bitmessage] Bitmessage API is live: %s", result) # If we failed, fall back to starting our own except Exception as e: self.log.info("Failed to connect to bitmessage instance: {}".format(e)) self.bitmessage_api = None # self._log.info("Spawning internal bitmessage instance") # # Add bitmessage submodule path # sys.path.insert(0, os.path.join( # os.path.dirname(__file__), '..', 'pybitmessage', 'src')) # import bitmessagemain as bitmessage # bitmessage.logger.setLevel(logging.WARNING) # bitmessage_instance = bitmessage.Main() # bitmessage_instance.start(daemon=True) # bminfo = bitmessage_instance.getApiAddress() # if bminfo is not None: # self._log.info("Started bitmessage daemon at %s:%s".format( # bminfo['address'], bminfo['port'])) # bitmessage_api = xmlrpclib.ServerProxy("http://{}:{}@{}:{}/".format( # bm_user, bm_pass, bminfo['address'], bminfo['port'])) # else: # self._log.info("Failed to start bitmessage dameon") # self._bitmessage_api = None return result def _checkok(self, msg): self.log.info('Check ok') def get_guid(self): return self.guid def get_dht(self): return self.dht def get_bitmessage_api(self): return self.bitmessage_api def get_market_id(self): return self.market_id # def get_myself(self): # return self._myself def _ping(self, msg): self.log.info('Pinged %s ' % json.dumps(msg, ensure_ascii=False)) # # pinger = CryptoPeerConnection(self, msg['uri'], msg['pubkey'], msg['senderGUID']) # pinger.send_raw(json.dumps( # {"type": "hello_response", # "senderGUID": self.guid, # "uri": self.uri, # "senderNick": self.nickname, # "pubkey": self.pubkey, # })) def _store_value(self, msg): self.dht._on_storeValue(msg) def _find_node(self, msg): self.dht.on_find_node(msg) def _find_node_response(self, msg): self.dht.on_findNodeResponse(self, msg) def _setup_settings(self): try: self.settings = self.db.selectEntries("settings", {"market_id": self.market_id}) except (OperationalError, DatabaseError) as e: print e raise SystemExit("database file %s corrupt or empty - cannot continue" % self.db.db_path) if len(self.settings) == 0: self.settings = {"market_id": self.market_id, "welcome": "enable"} self.db.insertEntry("settings", self.settings) else: self.settings = self.settings[0] # Generate PGP key during initial setup or if previous PGP gen failed if not ('PGPPubKey' in self.settings and self.settings["PGPPubKey"]): try: self.log.info('Generating PGP keypair. This may take several minutes...') print 'Generating PGP keypair. This may take several minutes...' gpg = gnupg.GPG() input_data = gpg.gen_key_input(key_type="RSA", key_length=2048, name_email='*****@*****.**', name_comment="Autogenerated by Open Bazaar", passphrase="P@ssw0rd") assert input_data is not None key = gpg.gen_key(input_data) assert key is not None pubkey_text = gpg.export_keys(key.fingerprint) newsettings = {"PGPPubKey": pubkey_text, "PGPPubkeyFingerprint": key.fingerprint} self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) self.log.info('PGP keypair generated.') except Exception as e: self.log.error("Encountered a problem with GPG: %s" % e) raise SystemExit("Encountered a problem with GPG: %s" % e) if not ('pubkey' in self.settings and self.settings['pubkey']): # Generate Bitcoin keypair self._generate_new_keypair() if not ('nickname' in self.settings and self.settings['nickname']): newsettings = {'nickname': 'Default'} self.db.updateEntries('settings', {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) self.nickname = self.settings['nickname'] if 'nickname' in self.settings else "" self.secret = self.settings['secret'] if 'secret' in self.settings else "" self.pubkey = self.settings['pubkey'] if 'pubkey' in self.settings else "" self.privkey = self.settings.get('privkey') self.btc_pubkey = privkey_to_pubkey(self.privkey) self.guid = self.settings['guid'] if 'guid' in self.settings else "" self.sin = self.settings['sin'] if 'sin' in self.settings else "" self.bitmessage = self.settings['bitmessage'] if 'bitmessage' in self.settings else "" if not ('bitmessage' in self.settings and self.settings['bitmessage']): # Generate Bitmessage address if self.bitmessage_api is not None: self._generate_new_bitmessage_address() self._myself = ec.ECC( pubkey=pubkey_to_pyelliptic(self.pubkey).decode('hex'), raw_privkey=self.secret.decode('hex'), curve='secp256k1' ) self.log.debug('Retrieved Settings: \n%s', pformat(self.settings)) def _generate_new_keypair(self): secret = str(random.randrange(2 ** 256)) self.secret = hashlib.sha256(secret).hexdigest() self.pubkey = privtopub(self.secret) self.privkey = random_key() print 'PRIVATE KEY: ', self.privkey self.btc_pubkey = privtopub(self.privkey) print 'PUBLIC KEY: ', self.btc_pubkey # Generate SIN sha_hash = hashlib.sha256() sha_hash.update(self.pubkey) ripe_hash = hashlib.new('ripemd160') ripe_hash.update(sha_hash.digest()) self.guid = ripe_hash.digest().encode('hex') self.sin = obelisk.EncodeBase58Check('\x0F\x02%s' + ripe_hash.digest()) newsettings = { "secret": self.secret, "pubkey": self.pubkey, "privkey": self.privkey, "guid": self.guid, "sin": self.sin } self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) def _generate_new_bitmessage_address(self): # Use the guid generated previously as the key self.bitmessage = self.bitmessage_api.createRandomAddress( self.guid.encode('base64'), False, 1.05, 1.1111 ) newsettings = {"bitmessage": self.bitmessage} self.db.updateEntries("settings", {"market_id": self.market_id}, newsettings) self.settings.update(newsettings) def join_network(self, seed_peers=None, callback=lambda msg: None): if seed_peers is None: seed_peers = [] self.log.info('Joining network') known_peers = [] # Connect up through seed servers for idx, seed in enumerate(seed_peers): try: socket.inet_pton(socket.AF_INET6, seed) seed_peers[idx] = "tcp://[%s]:12345" % seed except (socket.error, ValueError): seed_peers[idx] = "tcp://%s:12345" % seed # Connect to persisted peers db_peers = self.get_past_peers() known_peers = list(set(seed_peers)) + list(set(db_peers)) self.connect_to_peers(known_peers) # TODO: This needs rethinking. Normally we can search for ourselves # but because we are not connected to them quick enough this # will always fail. Need @gubatron to review # Populate routing table by searching for self # if len(known_peers) > 0: # self.search_for_my_node() if callback is not None: callback('Joined') def get_past_peers(self): peers = [] result = self.db.selectEntries("peers", {"market_id": self.market_id}) for peer in result: peers.append(peer['uri']) return peers def search_for_my_node(self): print 'Searching for myself' self.dht._iterativeFind(self.guid, self.dht.knownNodes, 'findNode') def connect_to_peers(self, known_peers): for known_peer in known_peers: t = Thread(target=self.dht.add_peer, args=(self, known_peer,)) t.start() def get_crypto_peer(self, guid=None, uri=None, pubkey=None, nickname=None, callback=None): if guid == self.guid: self.log.error('Cannot get CryptoPeerConnection for your own node') return self.log.debug('Getting CryptoPeerConnection' + '\nGUID:%s\nURI:%s\nPubkey:%s\nNickname:%s' % (guid, uri, pubkey, nickname)) return connection.CryptoPeerConnection(self, uri, pubkey, guid=guid, nickname=nickname) def addCryptoPeer(self, peer_to_add): foundOutdatedPeer = False for idx, peer in enumerate(self.dht.activePeers): if (peer.address, peer.guid, peer.pub) == \ (peer_to_add.address, peer_to_add.guid, peer_to_add.pub): self.log.info('Found existing peer, not adding.') return if peer.guid == peer_to_add.guid or \ peer.pub == peer_to_add.pub or \ peer.address == peer_to_add.address: foundOutdatedPeer = True self.log.info('Found an outdated peer') # Update existing peer self.activePeers[idx] = peer_to_add self.dht.add_peer(self, peer_to_add.address, peer_to_add.pub, peer_to_add.guid, peer_to_add.nickname) if not foundOutdatedPeer and peer_to_add.guid != self.guid: self.log.info('Adding crypto peer at %s' % peer_to_add.nickname) self.dht.add_peer(self, peer_to_add.address, peer_to_add.pub, peer_to_add.guid, peer_to_add.nickname) def get_profile(self): peers = {} self.settings = self.db.selectEntries("settings", {"market_id": self.market_id})[0] for uri, peer in self.peers.iteritems(): if peer.pub: peers[uri] = peer.pub.encode('hex') return {'uri': self.uri, 'pub': self._myself.get_pubkey().encode('hex'), 'nickname': self.nickname, 'peers': peers} def respond_pubkey_if_mine(self, nickname, ident_pubkey): if ident_pubkey != self.pubkey: self.log.info("Public key does not match your identity") return # Return signed pubkey pubkey = self._myself.pubkey ec_key = obelisk.EllipticCurveKey() ec_key.set_secret(self.secret) digest = obelisk.Hash(pubkey) signature = ec_key.sign(digest) # Send array of nickname, pubkey, signature to transport layer self.send(proto_response_pubkey(nickname, pubkey, signature)) def pubkey_exists(self, pub): for peer in self.peers.itervalues(): self.log.info( 'PEER: %s Pub: %s' % ( peer.pub.encode('hex'), pub.encode('hex') ) ) if peer.pub.encode('hex') == pub.encode('hex'): return True return False def create_peer(self, uri, pub, node_guid): if pub: pub = pub.decode('hex') # Create the peer if public key is not already in the peer list # if not self.pubkey_exists(pub): self.peers[uri] = connection.CryptoPeerConnection(self, uri, pub, node_guid) # Call 'peer' callbacks on listeners self.trigger_callbacks('peer', self.peers[uri]) # else: # print 'Pub Key is already in peer list' def send(self, data, send_to=None, callback=lambda msg: None): self.log.debug("Outgoing Data: %s %s" % (data, send_to)) # Directed message if send_to is not None: peer = self.dht.routingTable.getContact(send_to) if not peer: for activePeer in self.dht.activePeers: if activePeer.guid == send_to: peer = activePeer break # peer = CryptoPeerConnection(msg['uri']) if peer: self.log.debug('Directed Data (%s): %s' % (send_to, data)) try: peer.send(data, callback=callback) except Exception as e: self.log.error('Not sending message directly to peer %s' % e) else: self.log.error('No peer found') else: # FindKey and then send for peer in self.dht.activePeers: try: peer = self.dht.routingTable.getContact(peer.guid) data['senderGUID'] = self.guid data['pubkey'] = self.pubkey def cb(msg): self.log.debug('Message Back: \n%s' % pformat(msg)) peer.send(data, cb) except: self.log.info("Error sending over peer!") traceback.print_exc() def send_enc(self, uri, msg): peer = self.peers[uri] pub = peer.pub # Now send a hello message to the peer if pub: self.log.info( "Sending encrypted [%s] message to %s" % ( msg['type'], uri ) ) peer.send(msg) else: # Will send clear profile on initial if no pub self.log.info( "Sending unencrypted [%s] message to %s" % ( msg['type'], uri ) ) self.peers[uri].send_raw(json.dumps(msg)) def _init_peer(self, msg): uri = msg['uri'] pub = msg.get('pub') nickname = msg.get('nickname') msg_type = msg.get('type') guid = msg['guid'] if not self.valid_peer_uri(uri): self.log.error("Invalid Peer: %s " % uri) return if uri not in self.peers: # Unknown peer self.log.info('Add New Peer: %s' % uri) self.create_peer(uri, pub, guid) if not msg_type: self.send_enc(uri, hello_request(self.get_profile())) elif msg_type == 'hello_request': self.send_enc(uri, hello_response(self.get_profile())) else: # Known peer if pub: # test if we have to update the pubkey if not self.peers[uri].pub: self.log.info("Setting public key for seed node") self.peers[uri].pub = pub.decode('hex') self.trigger_callbacks('peer', self.peers[uri]) if self.peers[uri].pub != pub.decode('hex'): self.log.info("Updating public key for node") self.peers[uri].nickname = nickname self.peers[uri].pub = pub.decode('hex') self.trigger_callbacks('peer', self.peers[uri]) if msg_type == 'hello_request': # reply only if necessary self.send_enc(uri, hello_response(self.get_profile())) def _on_message(self, msg): # here goes the application callbacks # we get a "clean" msg which is a dict holding whatever # self.log.info("[On Message] Data received: %s" % msg) pubkey = msg.get('pubkey') uri = msg.get('uri') ip = urlparse(uri).hostname port = urlparse(uri).port guid = msg.get('senderGUID') nickname = msg.get('senderNick')[:120] self.dht.add_known_node((ip, port, guid, nickname)) self.log.info('On Message: %s' % json.dumps(msg, ensure_ascii=False)) self.dht.add_peer(self, uri, pubkey, guid, nickname) t = Thread(target=self.trigger_callbacks, args=(msg['type'], msg,)) t.start() def _on_raw_message(self, serialized): try: msg = json.loads(serialized) self.log.info("Message Received [%s]" % msg.get('type', 'unknown')) if msg.get('type') is None: data = msg.get('data').decode('hex') sig = msg.get('sig').decode('hex') try: cryptor = makePrivCryptor(self.secret) try: data = cryptor.decrypt(data) except Exception as e: self.log.info('Exception: %s' % e) self.log.debug('Signature: %s' % sig.encode('hex')) self.log.debug('Signed Data: %s' % data) # Check signature data_json = json.loads(data) sigCryptor = makePubCryptor(data_json['pubkey']) if sigCryptor.verify(sig, data): self.log.info('Verified') else: self.log.error('Message signature could not be verified %s' % msg) # return msg = json.loads(data) self.log.debug('Message Data %s ' % msg) except Exception as e: self.log.error('Could not decrypt message properly %s' % e) except ValueError: try: # Encrypted? try: msg = self._myself.decrypt(serialized) msg = json.loads(msg) self.log.info( "Decrypted Message [%s]" % msg.get('type', 'unknown') ) except: self.log.error("Could not decrypt message: %s" % msg) return except: self.log.error('Message probably sent using incorrect pubkey') return if msg.get('type') is not None: self._on_message(msg) else: self.log.error('Received a message with no type') def shutdown(self): print "CryptoTransportLayer.shutdown()!" try: TransportLayer.shutdown(self) print "CryptoTransportLayer.shutdown(): ZMQ sockets destroyed." except Exception as e: self.log.error("Transport shutdown error: " + e.message) print "Notice: explicit DHT Shutdown not implemented." try: self.bitmessage_api.close() except Exception as e: # might not even be open, not much more we can do on our way out if exception thrown here. self.log.error("Could not shutdown bitmessage_api's ServerProxy. " + e.message)
class TailWriter(CCHandler): """ Simply appends to files (with help from workers) """ CC_ROLES = ['remote'] log = skytools.getLogger('h:TailWriter') def __init__(self, hname, hcf, ccscript): super(TailWriter, self).__init__(hname, hcf, ccscript) self.files = {} self.workers = [] self.wparams = {} # passed to workers self.wparams['dstdir'] = self.cf.getfile('dstdir') self.wparams['host_subdirs'] = self.cf.getbool('host-subdirs', 0) self.wparams['maint_period'] = self.cf.getint('maint-period', 3) self.wparams['write_compressed'] = self.cf.get('write-compressed', '') assert self.wparams['write_compressed'] in [ None, '', 'no', 'keep', 'yes' ] if self.wparams['write_compressed'] in ('keep', 'yes'): self.log.info( "position checking not supported for compressed files") if self.wparams['write_compressed'] == 'yes': self.wparams['compression'] = self.cf.get('compression', '') if self.wparams['compression'] not in ('gzip', 'bzip2'): self.log.error("unsupported compression: %s", self.wparams['compression']) self.wparams['compression_level'] = self.cf.getint( 'compression-level', '') self.wparams['buf_maxbytes'] = cc.util.hsize_to_bytes( self.cf.get('buffer-bytes', '1 MB')) if self.wparams['buf_maxbytes'] < BUF_MINBYTES: self.log.info("buffer-bytes too low, adjusting: %i -> %i", self.wparams['buf_maxbytes'], BUF_MINBYTES) self.wparams['buf_maxbytes'] = BUF_MINBYTES # initialise sockets for communication with workers self.dealer_stream, self.dealer_url = self.init_comm( zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv) self.router_stream, self.router_url = self.init_comm( zmq.XREP, 'inproc://workers-router', self.router_on_recv) self.launch_workers() self.timer_maint = PeriodicCallback( self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop) self.timer_maint.start() def init_comm(self, stype, url, cb): """ Create socket, stream, etc for communication with workers. """ sock = self.zctx.socket(stype) port = sock.bind_to_random_port(url) curl = "%s:%d" % (url, port) stream = CCStream(sock, self.ioloop) stream.on_recv(cb) return (stream, curl) def launch_workers(self): """ Create and start worker threads. """ nw = self.cf.getint('worker-threads', 10) for i in range(nw): wname = "%s.worker-%i" % (self.hname, i) self.log.info("starting %s", wname) w = TailWriter_Worker(wname, self.xtx, self.zctx, self.ioloop, self.dealer_url, self.router_url, self.wparams) w.stat_inc = self.stat_inc # XXX self.workers.append(w) w.start() def handle_msg(self, cmsg): """ Got message from client, process it. """ data = cmsg.get_payload(self.xtx) if not data: return host = data['hostname'] fn = data['filename'] st_dev = data.get('st_dev') st_ino = data.get('st_ino') fi = (host, st_dev, st_ino, fn) if fi in self.files: fd = self.files[fi] if fd.waddr: # already accepted ? self.log.trace("passing %r to %s", fn, fd.wname) fd.queue.append(cmsg) fd.send_to(self.router_stream) else: self.log.trace("queueing %r", fn) fd.queue.append(cmsg) else: fd = FileState(fi, 1) self.files[fi] = fd self.log.trace("offering %r", fn) self.dealer_stream.send_cmsg(cmsg) def dealer_on_recv(self, zmsg): """ Got reply from worker via "dealer" connection """ self.log.warning("reply via dealer: %s", zmsg) def router_on_recv(self, zmsg): """ Got reply from worker via "router" connection """ cmsg = CCMessage(zmsg) data = cmsg.get_payload(self.xtx) fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'], data['d_filename']) fd = self.files[fi] if fd.waddr is None: fd.waddr = zmsg[0] fd.wname = data['worker'] else: assert fd.waddr == zmsg[0] and fd.wname == data['worker'] fd.atime = time.time() fd.count -= 1 assert fd.count >= 0 def do_maint(self): """ Check & flush queues; drop inactive files. """ self.log.trace('cleanup') now = time.time() zombies = [] for k, fd in self.files.iteritems(): if fd.queue and fd.waddr: self.log.trace("passing %r to %s", fd.ident, fd.wname) fd.send_to(self.router_stream) if (fd.count == 0) and (now - fd.atime > 2 * CLOSE_DELAY ): # you'd better use msg for this self.log.debug("forgetting %r", fd.ident) zombies.append(k) for k in zombies: self.files.pop(k) def stop(self): """ Signal workers to shut down. """ super(TailWriter, self).stop() self.log.info('stopping') self.timer_maint.stop() for w in self.workers: self.log.info("signalling %s", w.name) w.stop()