def run(self):
_period = 1000 * self.configreader["services"][self.name]["checkmail_interval"]
beat = PeriodicCallback(self.cmd_processmails,
_period,
io_loop=self.ioloop)
beat.start()
Service.run(self)
def __init__(self, primary, local, remote):
# initialize the Binary Star
self.ctx = zmq.Context() # Our private context
self.loop = IOLoop.instance() # Reactor loop
self.state = STATE_PRIMARY if primary else STATE_BACKUP
self.event = None # Current event
self.peer_expiry = 0 # When peer is considered 'dead'
self.voter_callback = None # Voting socket handler
self.master_callback = None # Call when become master
self.slave_callback = None # Call when become slave
# Create publisher for state going to peer
self.statepub = self.ctx.socket(zmq.PUB)
self.statepub.bind(local)
# Create subscriber for state coming from peer
self.statesub = self.ctx.socket(zmq.SUB)
self.statesub.setsockopt_string(zmq.SUBSCRIBE, u'')
self.statesub.connect(remote)
# wrap statesub in ZMQStream for event triggers
self.statesub = ZMQStream(self.statesub, self.loop)
# setup basic reactor events
self.heartbeat = PeriodicCallback(self.send_state,
HEARTBEAT, self.loop)
self.statesub.on_recv(self.recv_state)
def __init__(self):
self.stop = False
self.conx = Context.instance()
self.socket = self.conx.socket(zmq.ROUTER)
self.socket.bind('tcp://*:5555')
self.periodic = PeriodicCallback(self.timer, 4000)
self.periodic.start()
def __init__(self, context, main_ep, opt_ep=None):
"""Init MDPBroker instance.
"""
l = logger.Logger('mq_broker')
self.log = l.get_logger()
self.log.info("MDP broker startup...")
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
self.log.debug("Socket created...")
self._workers = {}
# services contain the worker queue and the request queue
self._services = {}
self._worker_cmds = {
b'\x01': self.on_ready,
b'\x03': self.on_reply,
b'\x04': self.on_heartbeat,
b'\x05': self.on_disconnect,
}
self.log.debug("Launch the timer...")
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
self.log.info("MDP broker started")
return
def __init__(self,
frontier,
data_in_sock='ipc:///tmp/robot-data-w2m.sock',
data_out_sock='ipc:///tmp/robot-data-m2w.sock',
msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
io_loop=None):
self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())
context = zmq.Context()
self._io_loop = io_loop or IOLoop.instance()
self._in_socket = context.socket(zmq.SUB)
self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
self._in_socket.bind(data_in_sock)
self._in_stream = ZMQStream(self._in_socket, io_loop)
self._out_socket = context.socket(zmq.PUSH)
self._out_socket.bind(data_out_sock)
self._out_stream = ZMQStream(self._out_socket, io_loop)
self._online_workers = set()
self._running = False
self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)
self.frontier = frontier
self.messenger = ServerMessenger(msg_in_sock, msg_out_sock, context,
io_loop)
class WorkerRep(object):
"""Helper class to represent a worker in the broker.
Instances of this class are used to track the state of the attached worker
and carry the timers for incomming and outgoing heartbeats.
:type wid: str
:param wid: the worker id.
:param service: service this worker serves
:type service: str
:param stream: the ZMQStream used to send messages
:type stream: ZMQStream
"""
def __init__(self, wid, service, stream):
self.id = wid
self.service = service
self.multicasts = []
self.curr_liveness = HB_LIVENESS
self.stream = stream
self.last_hb = 0
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
def send_hb(self):
"""Called on every HB_INTERVAL.
Decrements the current liveness by one.
Sends heartbeat to worker.
"""
self.curr_liveness -= 1
msg = [ self.id, b'', MDP_WORKER_VERSION, b'\x05' ]
self.stream.send_multipart(msg)
return
def on_heartbeat(self):
"""Called when a heartbeat message from the worker was received.
Sets current liveness to HB_LIVENESS.
"""
self.curr_liveness = HB_LIVENESS
return
def is_alive(self):
"""Returns True when the worker is considered alive.
"""
return self.curr_liveness > 0
def shutdown(self):
"""Cleanup worker.
Stops timer.
"""
self.hb_out_timer.stop()
self.hb_out_timer = None
self.stream = None
return
class AppServer(object):
# listen = "127.0.0.1"
listen = "*"
port = 5555
def __init__(self):
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.client_identities = {}
self.server = self.ctx.socket(zmq.ROUTER)
bind_addr = "tcp://%s:%s" % (self.listen, self.port)
print("bind_addr=", bind_addr)
self.server.bind(bind_addr)
print("Server listening for new client connections at", bind_addr)
self.server = ZMQStream(self.server)
self.server.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
def start(self):
self.periodic.start()
try:
self.loop.start()
except KeyboardInterrupt:
pass
def periodictask(self):
stale_clients = []
for client_id, last_seen in self.client_identities.items():
if last_seen + timedelta(seconds=10) < datetime.utcnow():
stale_clients.append(client_id)
else:
msg = HelloMessage()
msg.send(self.server, client_id)
for client_id in stale_clients:
print(
"Haven't received a HELO from cliient %s recently. Dropping from list of connected clients."
% client_id)
del self.client_identities[client_id]
sys.stdout.write(".")
sys.stdout.flush()
def on_recv(self, msg):
identity = msg[0]
self.client_identities[identity] = datetime.utcnow()
msg_type = msg[1]
print("Received message of type %s from client ID %s!" %
(msg_type, identity))
class WorkerRep(object):
"""Helper class to represent a worker in the broker.
Instances of this class are used to track the state of the attached worker
and carry the timers for incomming and outgoing heartbeats.
:param proto: the worker protocol id.
:type wid: str
:param wid: the worker id.
:type wid: str
:param service: service this worker serves
:type service: str
:param stream: the ZMQStream used to send messages
:type stream: ZMQStream
"""
def __init__(self, proto, wid, service, stream):
self.proto = proto
self.id = wid
self.service = service
self.curr_liveness = HB_LIVENESS
self.stream = stream
self.last_hb = 0
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
def send_hb(self):
"""Called on every HB_INTERVAL.
Decrements the current liveness by one.
Sends heartbeat to worker.
"""
self.curr_liveness -= 1
msg = [self.id, b'', self.proto, b'\x04']
self.stream.send_multipart(msg)
return
def on_heartbeat(self):
"""Called when a heartbeat message from the worker was received.
Sets current liveness to HB_LIVENESS.
"""
self.curr_liveness = HB_LIVENESS
return
def is_alive(self):
"""Returns True when the worker is considered alive.
"""
return self.curr_liveness > 0
def shutdown(self):
"""Cleanup worker.
Stops timer.
"""
self.hb_out_timer.stop()
self.hb_out_timer = None
self.stream = None
return
def main(pat):
fname = find_connection_file(pat)
with open(fname) as f:
cfg = json.load(f)
url = "%s://%s:%s" % (cfg.get('transport', 'tcp'), cfg['ip'], cfg['iopub_port'])
session = Session(key=cfg['key'])
ctx = zmq.Context.instance()
sub = ctx.socket(zmq.SUB)
sub.subscribe = b''
sub.connect(url)
# import IPython
# IPython.embed()
# return
stream = ZMQStream(sub)
stream.on_recv(lambda msg_list: log_msg(session, msg_list))
pc = PeriodicCallback(print_time, 5 * 60 * 1000)
pc.start()
IOLoop.instance().start()
def __init__(self, port=5556):
self.port = port
self.ctx = zmq.Context()
self.kvmap = {}
self.loop = IOLoop.instance()
# Set up our clone server sockets
self.snapshot = self.ctx.socket(zmq.ROUTER)
self.publisher = self.ctx.socket(zmq.PUB)
self.collector = self.ctx.socket(zmq.PULL)
self.snapshot.bind(f"tcp://*:{self.port:d}")
self.publisher.bind(f"tcp://*:{self.port + 1:d}")
self.collector.bind(f"tcp://*:{self.port + 2:d}")
# Wrap sockets in ZMQStreams for IOLoop handlers
self.snapshot = ZMQStream(self.snapshot)
self.publisher = ZMQStream(self.publisher)
self.collector = ZMQStream(self.collector)
# Register our handlers with reactor
self.snapshot.on_recv(self.handle_snapshot)
self.collector.on_recv(self.handle_collect)
self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)
# basic log formatting:
logging.basicConfig(format="%(asctime)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
class BufferModule(base.ZmqProcess):
def __init__(self, recv_addr, send_addr, recv_title, send_title, state_handler, period=1000):
super().__init__()
self.sub_stream = None
self.timer = None
self.recv_addr = recv_addr
self.recv_title = recv_title
self.state_handler = state_handler
self.period = period
self.callback = Callback(Publisher(send_addr, send_title), self.state_handler)
def setup(self):
super().setup()
self.sub_stream, _ = self.stream(zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8'))
self.sub_stream.on_recv(SubStreamHandler(self.sub_stream, self.stop, self.state_handler))
self.timer = PeriodicCallback(self.callback, self.period, self.loop)
def run(self):
self.setup()
print('Start loop!')
self.timer.start()
self.loop.start()
def stop(self):
self.loop.stop()
class DeviceRep(object):
"""
Helper class to represent a device to a worker
"""
def __init__(self, device_id, state='unknown'):
self.id = device_id
self.state = state
self.curr_liveness = CLIENT_HB_LIVENESS
self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL)
self.hb_timer.start()
return
def heartbeat(self):
if self.curr_liveness > 0:
self.curr_liveness -= 1
if self.curr_liveness == 0:
self.state = 'dead'
return
def on_message_received(self):
self.curr_liveness = CLIENT_HB_LIVENESS
return
def is_alive(self):
return self.curr_liveness > 0
def get_state(self):
return self.state
def shutdown(self):
self.hb_timer.stop()
self.hb_timer = None
def __init__(self, context, main_ep, opt_ep=None, service_q=None):
"""Init MDPBroker instance.
"""
if service_q is None:
self.service_q = ServiceQueue
else:
self.service_q = service_q
socket = context.socket(zmq.ROUTER)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = context.socket(zmq.ROUTER)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
self._workers = {}
# services contain the service queue and the request queue
self._services = {}
self._worker_cmds = {
b'\x01': self.on_ready,
b'\x03': self.on_reply,
b'\x04': self.on_heartbeat,
b'\x05': self.on_disconnect,
}
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
return
def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier,
log_handler, log_level, io_loop):
"""
Initialize the master.
"""
LoggingMixin.__init__(self, log_handler, log_level)
self._identity = identity
self._io_loop = io_loop or IOLoop.instance()
self._in_stream = ZMQStream(insocket, io_loop)
self._out_stream = ZMQStream(outsocket, io_loop)
self._mgmt = mgmt
self._frontier = frontier
self._running = False
self._available_workers = []
# periodically check if there are pending URIs to crawl
self._periodic_update = PeriodicCallback(self._send_next_uri,
settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop)
# start this periodic callback when you are waiting for the workers to
# finish
self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500,
io_loop=io_loop)
self._shutdown_counter = 0
self._logger.debug("zmqmaster::initialized")
def __init__(self, frontier,
data_in_sock='ipc:///tmp/robot-data-w2m.sock',
data_out_sock='ipc:///tmp/robot-data-m2w.sock',
msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
io_loop=None):
self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())
context = zmq.Context()
self._io_loop = io_loop or IOLoop.instance()
self._in_socket = context.socket(zmq.SUB)
self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
self._in_socket.bind(data_in_sock)
self._in_stream = ZMQStream(self._in_socket, io_loop)
self._out_socket = context.socket(zmq.PUSH)
self._out_socket.bind(data_out_sock)
self._out_stream = ZMQStream(self._out_socket, io_loop)
self._online_workers = set()
self._running = False
self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)
self.frontier = frontier
self.messenger = ServerMessenger(msg_in_sock, msg_out_sock,
context, io_loop)
def __init__(self, led_count, hw_backend, port=6606):
self.ctx = zmq.Context()
self.led_count = led_count
self.port = port
self.loop = IOLoop.instance()
self.caller = PeriodicCallback(self._on_next_frame, 1000/30)
self.hw_communication = hw_backend
self.hw_communication.connect()
self.zmq_collector = GlinAppZmqCollector(self, self.ctx)
self.zmq_publisher = GlinAppZmqPublisher(self, self.ctx)
# server side configuration
self.config = SimpleNamespace()
self.config.max_fps = 60
# current state (somehow client side configuration)
self.state = SimpleNamespace()
self.state.animationClasses = []
self.state.activeSceneId = None
self.state.activeAnimation = None
self.state.scenes = {}
self.state.brightness = 1.0
self.state.sceneIdCtr = 0
self.state.mainswitch = True
self.state.target_fps = 0
self.state.lastFrameSent = None
def start_ip_address_checker(self):
'''Checks for possible public IP change'''
if self.ob_ctx.enable_ip_checker:
self.caller = PeriodicCallback(self._ip_updater_periodic_callback,
5000, ioloop.IOLoop.instance())
self.caller.start()
self.log.info(
"IP_CHECKER_ENABLED: Periodic IP Address Checker started.")
def startup(self):
"""Setup sockets and handlers."""
super(CCServer, self).startup()
self.log.info("C&C server version %s starting up..", self.__version__)
self.xtx = CryptoContext(self.cf)
self.zctx = zmq.Context(self.zmq_nthreads)
self.ioloop = IOLoop.instance()
self.local_url = self.cf.get('cc-socket')
self.cur_role = self.cf.get('cc-role', 'insecure')
if self.cur_role == 'insecure':
self.log.warning(
'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config'
)
self.stat_level = self.cf.getint('cc-stats', 1)
if self.stat_level < 1:
self.log.warning('CC statistics level too low: %d',
self.stat_level)
# initialize local listen socket
s = self.zctx.socket(zmq.XREP)
s.setsockopt(zmq.LINGER, self.zmq_linger)
s.setsockopt(zmq.HWM, self.zmq_hwm)
if self.zmq_rcvbuf > 0:
s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf)
if self.zmq_sndbuf > 0:
s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf)
if self.zmq_tcp_keepalive > 0:
if getattr(zmq, 'TCP_KEEPALIVE', -1) > 0:
s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive)
s.setsockopt(zmq.TCP_KEEPALIVE_INTVL,
self.zmq_tcp_keepalive_intvl)
s.setsockopt(zmq.TCP_KEEPALIVE_IDLE,
self.zmq_tcp_keepalive_idle)
s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt)
else:
self.log.info("TCP_KEEPALIVE not available")
s.bind(self.local_url)
self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm)
self.local.on_recv(self.handle_cc_recv)
self.handlers = {}
self.routes = {}
rcf = skytools.Config('routes', self.cf.filename, ignore_defs=True)
for r, hnames in rcf.cf.items('routes'):
self.log.info('New route: %s = %s', r, hnames)
for hname in [hn.strip() for hn in hnames.split(',')]:
h = self.get_handler(hname)
self.add_handler(r, h)
self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop)
self.stimer.start()
class TaskState (object):
""" Tracks task state (with help of watchdog) """
log = skytools.getLogger ('d:TaskState')
def __init__ (self, uid, name, info, ioloop, cc, xtx):
self.uid = uid
self.name = name
self.info = info
self.pidfile = info['config']['pidfile']
self.ioloop = ioloop
self.cc = cc
self.xtx = xtx
self.timer = None
self.timer_tick = 1
self.heartbeat = False
self.start_time = None
self.dead_since = None
def start (self):
self.start_time = time.time()
self.timer = PeriodicCallback (self.watchdog, self.timer_tick * 1000, self.ioloop)
self.timer.start()
def stop (self):
try:
self.log.info ('Signalling %s', self.name)
skytools.signal_pidfile (self.pidfile, signal.SIGINT)
except:
self.log.exception ('signal_pidfile failed: %s', self.pidfile)
def watchdog (self):
live = skytools.signal_pidfile (self.pidfile, 0)
if live:
self.log.debug ('%s is alive', self.name)
if self.heartbeat:
self.send_reply ('running')
else:
self.log.info ('%s is over', self.name)
self.dead_since = time.time()
self.timer.stop()
self.timer = None
self.send_reply ('stopped')
def ccpublish (self, msg):
assert isinstance (msg, TaskReplyMessage)
cmsg = self.xtx.create_cmsg (msg)
cmsg.send_to (self.cc)
def send_reply (self, status, feedback = {}):
msg = TaskReplyMessage(
req = 'task.reply.%s' % self.uid,
handler = self.info['task']['task_handler'],
task_id = self.info['task']['task_id'],
status = status,
feedback = feedback)
self.ccpublish (msg)
def __init__ (self, hname, hcf, ccscript):
super(ProxyHandler, self).__init__(hname, hcf, ccscript)
self.ping_remote = self.cf.getbool ("ping", False)
if self.ping_remote:
self.echo_stats = EchoState (self.remote_url)
self.echo_timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop)
self.echo_timer.start()
self.log.debug ("will ping %s", self.remote_url)
class TaskState(object):
""" Tracks task state (with help of watchdog) """
log = skytools.getLogger('d:TaskState')
def __init__(self, uid, name, info, ioloop, cc, xtx):
self.uid = uid
self.name = name
self.info = info
self.pidfile = info['config']['pidfile']
self.ioloop = ioloop
self.cc = cc
self.xtx = xtx
self.timer = None
self.timer_tick = 1
self.heartbeat = False
self.start_time = None
self.dead_since = None
def start(self):
self.start_time = time.time()
self.timer = PeriodicCallback(self.watchdog, self.timer_tick * 1000,
self.ioloop)
self.timer.start()
def stop(self):
try:
self.log.info('Signalling %s', self.name)
skytools.signal_pidfile(self.pidfile, signal.SIGINT)
except:
self.log.exception('signal_pidfile failed: %s', self.pidfile)
def watchdog(self):
live = skytools.signal_pidfile(self.pidfile, 0)
if live:
self.log.debug('%s is alive', self.name)
if self.heartbeat:
self.send_reply('running')
else:
self.log.info('%s is over', self.name)
self.dead_since = time.time()
self.timer.stop()
self.timer = None
self.send_reply('stopped')
def ccpublish(self, msg):
assert isinstance(msg, TaskReplyMessage)
cmsg = self.xtx.create_cmsg(msg)
cmsg.send_to(self.cc)
def send_reply(self, status, feedback={}):
msg = TaskReplyMessage(req='task.reply.%s' % self.uid,
handler=self.info['task']['task_handler'],
task_id=self.info['task']['task_id'],
status=status,
feedback=feedback)
self.ccpublish(msg)
class AppClient(object):
# where to connect to - point to another IP or hostname:5556 if running server on another machine
endpoint = "tcp://127.0.0.1:5556"
# crypto = True means 'use CurveZMQ'. False means don't. Must match server.
crypto = True
def __init__(self):
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.client = self.ctx.socket(zmq.DEALER)
if self.crypto:
self.keymonkey = KeyMonkey("client")
self.client = self.keymonkey.setupClient(self.client,
self.endpoint, "server")
self.client.connect(self.endpoint)
print("Connecting to", self.endpoint)
self.client = ZMQStream(self.client)
self.client.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
self.last_recv = None
def periodictask(self):
if not self.last_recv or self.last_recv + timedelta(
seconds=5) < datetime.utcnow():
print(
"Hmmm... haven't heard from the server in 5 seconds... Server unresponsive."
)
print("Sending HELLO to server")
msg = HelloMessage()
msg.send(self.client)
def start(self):
self.periodic.start()
try:
self.loop.start()
except KeyboardInterrupt:
pass
def on_recv(self, msg):
self.last_recv = datetime.utcnow()
print("Received a message of type %s from server!" % msg[0])
def setup(self):
super().setup()
self.sub_stream, _ = self.stream(
zmq.SUB,
self.recv_addr,
bind=False,
subscribe=self.recv_title.encode('utf-8'))
self.sub_stream.on_recv(
SubStreamHandler(self.sub_stream, self.stop, self.state_handler))
self.timer = PeriodicCallback(self.callback, self.period, self.loop)
def __init__(self, proto, wid, service, stream):
self.proto = proto
self.id = wid
self.service = service
self.curr_liveness = HB_LIVENESS
self.stream = stream
self.last_hb = 0
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
class AppClient(object):
def __init__(self):
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.endpoint = "tcp://127.0.0.1:5556"
self.client = self.ctx.socket(zmq.DEALER)
self.client.setsockopt(
zmq.LINGER, 0
) # Without linger and timeouts you might have problems when closing context
self.client.setsockopt(zmq.RCVTIMEO, 5000) # 5s
self.client.setsockopt(zmq.SNDTIMEO, 5000)
print("Connecting to", self.endpoint)
self.client.connect(self.endpoint)
self.client = ZMQStream(self.client)
self.client.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
self.last_recv = None
def disconnect(self):
if self.ctx is not None:
try:
self.periodic.stop()
print("Closing socket and context")
self.client.close()
self.ctx.term()
except Exception as e:
print(e)
def periodictask(self):
if self.client is None:
return
if not self.last_recv or self.last_recv + timedelta(
seconds=5) < datetime.utcnow():
print("No data from remote (5s)... [ping]")
print("Sending HELLO to server")
msg = HelloMessage()
msg.send(self.client)
def start(self):
try:
self.periodic.start()
self.loop.start()
msg = HelloMessage()
msg.send(self.client)
except KeyboardInterrupt:
print("\n\nCtrl+C detected\n")
except Exception as E:
print("Error detected")
print(str(E))
finally:
self.disconnect()
def on_recv(self, msg):
self.last_recv = datetime.utcnow()
print("Received a message of type %s from server!" % msg[0])
def __init__ (self, hname, hcf, ccscript):
super(Delay, self).__init__(hname, hcf, ccscript)
self.fwd_hname = self.cf.get ('forward-to')
self.delay = self.cf.getint ('delay', 0)
self.fwd_handler = ccscript.get_handler (self.fwd_hname)
self.queue = collections.deque()
self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop)
self.timer.start()
def __init__(self, *args):
super(TaskRouter, self).__init__(*args)
self.route_map = {}
self.reply_map = {}
# 1 hr? XXX
self.route_lifetime = self.cf.getint ('route-lifetime', 1 * 60 * 60)
self.reply_timeout = self.cf.getint ('reply-timeout', 5 * 60)
self.maint_period = self.cf.getint ('maint-period', 1 * 60)
self.timer = PeriodicCallback(self.do_maint, self.maint_period*1000, self.ioloop)
self.timer.start()
class AppServer(object):
def __init__(self):
self.listen = "127.0.0.1"
self.port = 5556
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.client_identities = {}
self.server = self.ctx.socket(zmq.ROUTER)
self.server.setsockopt(
zmq.LINGER, 0
) # Without linger and timeouts you might have problems when closing context
self.server.setsockopt(zmq.RCVTIMEO, 5000) # 5s
self.server.setsockopt(zmq.SNDTIMEO, 5000)
bind_addr = "tcp://%s:%s" % (self.listen, self.port)
self.server.bind(bind_addr)
print("Server listening for new client connections at", bind_addr)
self.server = ZMQStream(self.server)
self.server.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
def start(self):
self.periodic.start()
try:
self.loop.start()
except KeyboardInterrupt:
self.periodic.stop()
print("\nClosing socket and context\n")
self.server.close()
self.ctx.term()
def periodictask(self):
stale_clients = []
for client_id, last_seen in self.client_identities.items():
if last_seen + timedelta(seconds=10) < datetime.utcnow():
stale_clients.append(client_id)
else:
msg = HelloMessage()
msg.send(self.server, client_id)
for client_id in stale_clients:
print(
"\nHaven't received a HELO from client %s recently. Dropping from list of connected clients."
% client_id)
del self.client_identities[client_id]
sys.stdout.write(".")
sys.stdout.flush()
def on_recv(self, msg):
identity = msg[
0] # This contains client id (socket handle), use to reply it back
self.client_identities[identity] = datetime.utcnow()
msg_type = msg[1]
print("Received message of type %s from client ID %s!" %
(msg_type, identity))
def __init__(self,
context,
main_ep,
opt_ep=None,
service_q=None,
data_q=None):
"""Init MNBroker instance.
"""
if service_q is None:
self.service_q = ServiceQueue
else:
self.service_q = service_q
if data_q is None:
self.data_q = ServiceQueue
else:
self.data_q = data_q
socket = context.socket(zmq.ROUTER)
socket.bind(main_ep)
socket.setsockopt(zmq.IDENTITY, b'BROKER')
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = context.socket(zmq.ROUTER)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
# TODO: merge worker_tracker and info
self._workers = {}
self._workers_info = {}
self._services = {
} # TODO: each worker must have his own request queue
self._worker_cmds = {
MSG_READY: self.on_ready,
MSG_REPLY: self.on_reply,
MSG_HEARTBEAT: self.on_heartbeat,
MSG_DISCONNECT: self.on_disconnect,
}
self._local_cmds = {
MSG_WINFO: self.get_workers_info,
}
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
self.hb_get_winfo = PeriodicCallback(self.collect_workers_info,
HB_INTERVAL)
self.hb_get_winfo.start()
self.register_worker_info(self.main_stream.getsockopt(
zmq.IDENTITY)) # register this instance
_LOG.info("Broker initialized and can be found at '%s'" % main_ep)
return
def __init__(self, proto, wid, service, stream):
self.proto = proto
self.id = wid
self.service = service
self.curr_retries = HB_RETRIES
self.stream = stream
self.last_hb = 0
self.free_space = 0
self.data = []
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
def __init__(self, history_filepath=None):
self._logger = logging.getLogger(__name__)
if history_filepath is None:
self._logger.info(' getting default history filepath.')
history_filepath = _get_default_history_filepath()
self._logger.info(' history filepath %s', history_filepath)
self.history = FileHistory(history_filepath)
self.messaging = _Messaging('shell', run_control_loop=True)
# TODO: Cleanup this API access
self.messaging._heartbeat_reciever.identity_callback = self._identity_callback
self._thread = _Thread(target=self.messaging.start, daemon=True)
self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state,
1000)
self.shebangs = [
'!',
]
self.command_observer = CommandObserver(self.messaging, prompt=self)
commands = self.command_observer._get_commands()
self._word_completer = WordCompleter(commands, WORD=_WORD)
self._services_completer = ServiceCompleter(self._word_completer)
self.author_interface = AuthorInterface(self._word_completer,
self.messaging)
self.service_interface = ServiceInterface(self._word_completer,
self.messaging)
self.entity_interface = EntityInterface(self.author_interface,
self.service_interface)
self._bot_status = ''
super().__init__(message='vexbot: ',
history=self.history,
completer=self._services_completer,
enable_system_prompt=True,
enable_suspend=True,
enable_open_in_editor=True,
complete_while_typing=False)
self.print_observer = PrintObserver(self.app)
self._print_subscription = self.messaging.chatter.subscribe(
self.print_observer)
self.messaging.chatter.subscribe(LogObserver())
self.messaging.command.subscribe(self.command_observer)
self.messaging.command.subscribe(
ServicesObserver(self._identity_setter,
self._set_service_completion))
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def loop():
self.__ioloop = IOLoop()
queue_socket = self.__context.socket(zmq.PULL)
queue_socket.connect(self.message_address)
queue_stream = ZMQStream(queue_socket, self.__ioloop)
worker_socket = self.__context.socket(zmq.DEALER)
for address in self.active_connections:
worker_socket.connect(address)
worker_stream = ZMQStream(worker_socket, self.__ioloop)
def receive_response(message, response_override=None):
self.__queued_messages.pop(message[1], None)
self.__message_timeouts.pop(message[1], None)
callback = self.__callbacks.pop(message[1], None)
if callback:
try:
callback(response_override or self.loads(self.decompress(message[2])))
except Exception as e:
self.log_error(e)
callback({'error': e})
worker_stream.on_recv(receive_response)
def queue_message(message):
if message[0]:
if message[0] == WORKER_SOCKET_CONNECT and message[2] not in self.active_connections:
self.active_connections.add(message[2])
worker_stream.socket.connect(message[2])
elif message[0] == WORKER_SOCKET_DISCONNECT and message[2] in self.active_connections:
self.active_connections.remove(message[2])
worker_stream.socket.disconnect(message[2])
return
self.__queued_messages[message[1]] = (time(), message)
try:
worker_stream.send_multipart(message)
except Exception as e:
self.log_error(e)
queue_stream.on_recv(queue_message)
def timeout_message():
now = time()
for message, retry in [(item[1], self.__message_auto_retry.get(item[1][1], self.__auto_retry)) for item, t in ((i, self.__message_timeouts.get(i[1][1], self.__timeout)) for i in self.__queued_messages.itervalues()) if t >= 0 and (item[0] + t < now)]:
if retry:
logging.info('Worker timeout, requeuing ' + message[1])
queue_message(message)
else:
receive_response(('', message[1]), {'error': 'timeout'})
timeout_callback = PeriodicCallback(timeout_message, int(abs(self.__timeout * 1000.0)), io_loop = self.__ioloop)
timeout_callback.start()
self.__ioloop.start()
self.__thread = None
def __init__(self):
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.client = self.ctx.socket(zmq.DEALER)
if self.crypto:
self.keymonkey = KeyMonkey("client")
self.client = self.keymonkey.setupClient(self.client,
self.endpoint, "server")
self.client.connect(self.endpoint)
print("Connecting to", self.endpoint)
self.client = ZMQStream(self.client)
self.client.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
self.last_recv = None
def __init__(self, context, main_ep, client_ep, hb_ep, service_q=None):
"""Init MDPBroker instance.
"""
if service_q is None:
self.service_q = ServiceQueue
else:
self.service_q = service_q
#
# Setup the zmq sockets.
#
socket = context.socket(zmq.ROUTER)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
socket = context.socket(zmq.ROUTER)
socket.bind(client_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
socket = context.socket(zmq.ROUTER)
socket.bind(hb_ep)
self.hb_stream = ZMQStream(socket)
self.hb_stream.on_recv(self.on_message)
self._workers = {}
#
# services contain the service queue and the request queue
#
self._services = {}
#
# Mapping of worker commands and callbacks.
#
self._worker_cmds = {
W_READY: self.on_ready,
W_REPLY: self.on_reply,
W_HEARTBEAT: self.on_heartbeat,
W_DISCONNECT: self.on_disconnect,
}
#
# 'Cleanup' timer for workers without heartbeat.
#
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
class DeviceApp(object):
# endpoint = "tcp://127.0.0.1:5556"
# endpoint = "tcp://localhost:5556"
endpoint = "tcp://165.227.24.226:5556"
def __init__(self):
self.ctx = zmq.Context()
self.loop = IOLoop.instance()
self.client = self.ctx.socket(zmq.DEALER)
self.client.connect(self.endpoint)
print("Connecting to", self.endpoint)
self.client = ZMQStream(self.client)
self.client.on_recv(self.on_recv)
self.periodic = PeriodicCallback(self.periodictask, 1000)
self.last_recv = None
def periodictask(self):
if not (self.last_recv) or (self.last_recv +
timedelta(seconds=5)) < datetime.utcnow():
print(
"Hmmm... haven't heard from the server in 5 seconds... Server unresponsive."
)
print("Sending HELLO to server")
msg = HelloMessage()
print("msg=", msg)
msg.send(self.client)
def start(self):
self.periodic.start()
try:
self.loop.start()
except KeyboardInterrupt:
pass
def on_recv(self, msg):
self.last_recv = datetime.utcnow()
print("Received a message of type %s from server!" % msg[0])
def __init__(self, primary=True, ports=(5556, 5566)):
self.primary = primary
if primary:
self.port, self.peer = ports
frontend = "tcp://*:5003"
backend = "tcp://localhost:5004"
self.kvmap = {}
else:
self.peer, self.port = ports
frontend = "tcp://*:5004"
backend = "tcp://localhost:5003"
self.ctx = zmq.Context.instance()
self.pending = []
self.bstar = BinaryStar(primary, frontend, backend)
self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER,
self.handle_snapshot)
# Set up our clone server sockets
self.publisher = self.ctx.socket(zmq.PUB)
self.collector = self.ctx.socket(zmq.SUB)
self.collector.setsockopt(zmq.SUBSCRIBE, b'')
self.publisher.bind("tcp://*:%d" % (self.port + 1))
self.collector.bind("tcp://*:%d" % (self.port + 2))
# Set up our own clone client interface to peer
self.subscriber = self.ctx.socket(zmq.SUB)
self.subscriber.setsockopt(zmq.SUBSCRIBE, b'')
self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1))
# Register state change handlers
self.bstar.master_callback = self.become_master
self.bstar.slave_callback = self.become_slave
# Wrap sockets in ZMQStreams for IOLoop handlers
self.publisher = ZMQStream(self.publisher)
self.subscriber = ZMQStream(self.subscriber)
self.collector = ZMQStream(self.collector)
# Register our handlers with reactor
self.collector.on_recv(self.handle_collect)
self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)
self.hugz_callback = PeriodicCallback(self.send_hugz, 1000)
# basic log formatting:
logging.basicConfig(format="%(asctime)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
def start(self):
loop = self.loop
loop.add_handler(self.udp.handle.fileno(), self.handle_beacon, loop.READ)
stream = ZMQStream(self.pipe, loop)
stream.on_recv(self.control_message)
pc = PeriodicCallback(self.send_ping, PING_INTERVAL * 1000, loop)
pc.start()
pc = PeriodicCallback(self.reap_peers, PING_INTERVAL * 1000, loop)
pc.start()
pc = PeriodicCallback(self.send_ports, PING_INTERVAL * 1000, loop)
pc.start()
loop.start()
def __init__(self, port=5556):
self.port = port
self.ctx = zmq.Context()
self.kvmap = {}
self.loop = IOLoop.instance()
# Set up our clone server sockets
self.snapshot = self.ctx.socket(zmq.ROUTER)
self.publisher = self.ctx.socket(zmq.PUB)
self.collector = self.ctx.socket(zmq.PULL)
self.snapshot.bind("tcp://*:%d" % self.port)
self.publisher.bind("tcp://*:%d" % (self.port + 1))
self.collector.bind("tcp://*:%d" % (self.port + 2))
# Wrap sockets in ZMQStreams for IOLoop handlers
self.snapshot = ZMQStream(self.snapshot)
self.publisher = ZMQStream(self.publisher)
self.collector = ZMQStream(self.collector)
# Register our handlers with reactor
self.snapshot.on_recv(self.handle_snapshot)
self.collector.on_recv(self.handle_collect)
self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)
# basic log formatting:
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
def __init__(self, broker, service, io_loop=None):
"""Create and setup an MDP worker.
@param broker A string containing the broker's URL
@param service A string containing the service name
@param io_loop An existing I/O loop object. If None, the default will be used.
"""
self.service=service
self._broker = broker
self.ctx = zmq.Context()
sock = self.ctx.socket(zmq.DEALER)
ZMQStream.__init__(self, sock, io_loop)
# last watchdog timer tick
self.watchdog = 0
# connection callback one-shot
self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop)
# heartbeat callback..runs continuous when connected
self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop)
# number of connection attempts
self._conn_attempt = 0
# waiting to connect state
self._waiting_to_connect = True
# have we been disconnected? (flags a reconnect attempt)
self.disconnected = False
# connect the socket and send a READY when the io_loop starts
self.connect(self._broker)
self._conncb.start()
class UdpListener (CCDaemon):
""" UDP server to handle UDP stream. """
log = skytools.getLogger ('d:UdpListener')
def reload (self):
super(UdpListener, self).reload()
self.listen_host = self.cf.get ('listen-host')
self.listen_port = self.cf.getint ('listen-port')
self.stats_period = self.cf.getint ('stats-period', 30)
def startup (self):
super(UdpListener, self).startup()
# plugins should be ready before we start receiving udp stream
self.init_plugins()
self.listen_addr = (self.listen_host, self.listen_port)
self.sock = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setblocking (0)
try:
self.sock.bind (self.listen_addr)
except Exception, e:
self.log.exception ("failed to bind to %s - %s", self.listen_addr, e)
raise
self.ioloop = IOLoop.instance()
callback = functools.partial (self.handle_udp, self.sock)
self.ioloop.add_handler (self.sock.fileno(), callback, self.ioloop.READ)
self.timer_stats = PeriodicCallback (self.send_stats, self.stats_period * 1000, self.ioloop)
self.timer_stats.start()
def __init__ (self, hname, hcf, ccscript):
super(TailWriter, self).__init__(hname, hcf, ccscript)
self.files = {}
self.workers = []
self.wparams = {} # passed to workers
self.wparams['dstdir'] = self.cf.getfile ('dstdir')
self.wparams['host_subdirs'] = self.cf.getbool ('host-subdirs', 0)
self.wparams['maint_period'] = self.cf.getint ('maint-period', 3)
self.wparams['write_compressed'] = self.cf.get ('write-compressed', '')
assert self.wparams['write_compressed'] in [None, '', 'no', 'keep', 'yes']
if self.wparams['write_compressed'] in ('keep', 'yes'):
self.log.info ("position checking not supported for compressed files")
if self.wparams['write_compressed'] == 'yes':
self.wparams['compression'] = self.cf.get ('compression', '')
if self.wparams['compression'] not in ('gzip', 'bzip2'):
self.log.error ("unsupported compression: %s", self.wparams['compression'])
self.wparams['compression_level'] = self.cf.getint ('compression-level', '')
self.wparams['buf_maxbytes'] = cc.util.hsize_to_bytes (self.cf.get ('buffer-bytes', '1 MB'))
if self.wparams['buf_maxbytes'] < BUF_MINBYTES:
self.log.info ("buffer-bytes too low, adjusting: %i -> %i", self.wparams['buf_maxbytes'], BUF_MINBYTES)
self.wparams['buf_maxbytes'] = BUF_MINBYTES
# initialise sockets for communication with workers
self.dealer_stream, self.dealer_url = self.init_comm (zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv)
self.router_stream, self.router_url = self.init_comm (zmq.XREP, 'inproc://workers-router', self.router_on_recv)
self.launch_workers()
self.timer_maint = PeriodicCallback (self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop)
self.timer_maint.start()
def __init__(self, context, main_ep, opt_ep=None, worker_q=None):
"""Init MDPBroker instance.
"""
socket = context.socket(zmq.XREP)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = context.socket(zmq.XREP)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
self._workers = {}
# services contain the worker queue and the request queue
self._services = {}
self._worker_cmds = { '\x01': self.on_ready,
'\x03': self.on_reply,
'\x04': self.on_heartbeat,
'\x05': self.on_disconnect,
}
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
return
def __init__(self, device_id, state='unknown'):
self.id = device_id
self.state = state
self.curr_liveness = CLIENT_HB_LIVENESS
self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL)
self.hb_timer.start()
return
def __init__(self, context, main_ep, opt_ep=None):
"""Init MDPBroker instance.
"""
l = logger.Logger('mq_broker')
self.log = l.get_logger()
self.log.info("MDP broker startup...")
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
self.log.debug("Socket created...")
self._workers = {}
# services contain the worker queue and the request queue
self._services = {}
self._worker_cmds = { b'\x01': self.on_ready,
b'\x03': self.on_reply,
b'\x04': self.on_heartbeat,
b'\x05': self.on_disconnect,
}
self.log.debug("Launch the timer...")
self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL)
self.hb_check_timer.start()
self.log.info("MDP broker started")
return
def __init__(self, primary=True, ports=(5556,5566)):
self.primary = primary
if primary:
self.port, self.peer = ports
frontend = "tcp://*:5003"
backend = "tcp://localhost:5004"
self.kvmap = {}
else:
self.peer, self.port = ports
frontend = "tcp://*:5004"
backend = "tcp://localhost:5003"
self.ctx = zmq.Context.instance()
self.pending = []
self.bstar = BinaryStar(primary, frontend, backend)
self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER, self.handle_snapshot)
# Set up our clone server sockets
self.publisher = self.ctx.socket(zmq.PUB)
self.collector = self.ctx.socket(zmq.SUB)
self.collector.setsockopt(zmq.SUBSCRIBE, b'')
self.publisher.bind("tcp://*:%d" % (self.port + 1))
self.collector.bind("tcp://*:%d" % (self.port + 2))
# Set up our own clone client interface to peer
self.subscriber = self.ctx.socket(zmq.SUB)
self.subscriber.setsockopt(zmq.SUBSCRIBE, b'')
self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1))
# Register state change handlers
self.bstar.master_callback = self.become_master
self.bstar.slave_callback = self.become_slave
# Wrap sockets in ZMQStreams for IOLoop handlers
self.publisher = ZMQStream(self.publisher)
self.subscriber = ZMQStream(self.subscriber)
self.collector = ZMQStream(self.collector)
# Register our handlers with reactor
self.collector.on_recv(self.handle_collect)
self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)
self.hugz_callback = PeriodicCallback(self.send_hugz, 1000)
# basic log formatting:
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
def setup(self):
super(DemoApp, self).setup()
self.pub, self.pub_addr = self.stream(zmq.PUB, 'tcp://127.0.0.1:%(port)s', True)
self.sub, sub_addr = self.stream(zmq.SUB, self.pub_addr, False,
callback=DemoHandler())
self.heartbeat = PeriodicCallback(self.ping, 1000, self.loop)
def run(self):
self.received_count = 0
ctx, io_loop, self.socks, streams = get_run_context(self.sock_configs)
self.io_loop = io_loop
if self.request_delay > 0:
periodic_callback = PeriodicCallback(self.do_request,
self.request_delay * 1000., io_loop)
io_loop.add_timeout(timedelta(seconds=0.1), lambda: periodic_callback.start())
else:
for i in range(self.target_count):
io_loop.add_callback(self.do_request)
try:
io_loop.start()
except KeyboardInterrupt:
pass
def __init__(self, my_ip, my_port, market_id, bm_user=None, bm_pass=None, bm_port=None, seed_mode=0, dev_mode=False):
self._log = logging.getLogger('[%s] %s' % (market_id, self.__class__.__name__))
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Connect to database
MONGODB_URI = 'mongodb://localhost:27017'
_dbclient = MongoClient()
self._db = _dbclient.openbazaar
self._bitmessage_api = None
if (bm_user, bm_pass, bm_port) != (None, None, None):
if not self._connect_to_bitmessage(bm_user, bm_pass, bm_port):
self._log.info('Bitmessage not available')
self._market_id = market_id
self.nick_mapping = {}
self._uri = "tcp://%s:%s" % (my_ip, my_port)
self._ip = my_ip
self._nickname = ""
# Set up
self._setup_settings()
self._dht = DHT(self, market_id, self.settings)
self._myself = ec.ECC(pubkey=self.pubkey.decode('hex'), privkey=self.secret.decode('hex'), curve='secp256k1')
TransportLayer.__init__(self, market_id, my_ip, my_port, self.guid, self._nickname)
# Set up callbacks
self.add_callback('hello', self._ping)
self.add_callback('findNode', self._findNode)
self.add_callback('findNodeResponse', self._findNodeResponse)
self.add_callback('store', self._storeValue)
self.listen(self.pubkey)
def cb():
r = requests.get(r'http://icanhazip.com')
if r and hasattr(r,'text'):
ip = r.text
ip = ip.strip(' \t\n\r')
if ip != self._ip:
self._ip = ip
self._uri = 'tcp://%s:%s' % (self._ip, self._port)
self.stream.close()
self.listen(self.pubkey)
else:
self._log.error('Could not get ip')
if seed_mode == 0 and not dev_mode:
# Check IP periodically for changes
self.caller = PeriodicCallback(cb, 5000, ioloop.IOLoop.instance())
self.caller.start()
def _create_stream(self):
"""Overidden _create_stream for MDPWorker
Adds the device_timer to manage connected devices
"""
self.device_timer = PeriodicCallback(self.device_watcher,
CLIENT_HB_INTERVAL)
self.device_timer.start()
super(DeviceServiceManager, self)._create_stream()
return
def __init__ (self, hname, hcf, ccscript):
super(ProxyHandler, self).__init__(hname, hcf, ccscript)
self.ping_remote = self.cf.getbool ("ping", False)
if self.ping_remote:
self.echo_stats = EchoState (self.remote_url)
self.echo_timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop)
self.echo_timer.start()
self.log.debug ("will ping %s", self.remote_url)
class Delay (CCHandler):
""" Delays all received messages, then dispatches them to another handler. """
CC_ROLES = ['local', 'remote']
log = skytools.getLogger ('h:Delay')
tick = 250 # ms
def __init__ (self, hname, hcf, ccscript):
super(Delay, self).__init__(hname, hcf, ccscript)
self.fwd_hname = self.cf.get ('forward-to')
self.delay = self.cf.getint ('delay', 0)
self.fwd_handler = ccscript.get_handler (self.fwd_hname)
self.queue = collections.deque()
self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop)
self.timer.start()
def handle_msg (self, cmsg):
""" Got message from client -- queue it """
self.queue.append ((time.time() + self.delay, cmsg))
def process_queue (self):
now = time.time()
try:
while (self.queue[0][0] <= now):
at, cmsg = self.queue.popleft()
size = cmsg.get_size()
try:
self.fwd_handler.handle_msg (cmsg)
stat = 'ok'
except Exception:
self.log.exception ('crashed, dropping msg: %s', cmsg.get_dest())
stat = 'crashed'
self.stat_inc ('delay.count')
self.stat_inc ('delay.bytes', size)
self.stat_inc ('delay.count.%s' % stat)
self.stat_inc ('delay.bytes.%s' % stat, size)
except IndexError:
pass
def __init__(self, wid, service, stream):
self.id = wid
self.service = service
self.multicasts = []
self.curr_liveness = HB_LIVENESS
self.stream = stream
self.last_hb = 0
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
def start(self):
self.frontend.on_recv(self.handle_frontend)
self.backend.on_recv(self.handle_backend)
self.period = PeriodicCallback(self.workers.purge, self.heartbeat*1000)
self.period.start()
try:
self.frontend.send(PPP_READY)
#self.loop.start()
except KeyboardInterrupt:
times_str('ctrlc1')
def startup(self):
"""Setup sockets and handlers."""
super(CCServer, self).startup()
self.log.info("C&C server version %s starting up..", self.__version__)
self.xtx = CryptoContext(self.cf)
self.zctx = zmq.Context(self.zmq_nthreads)
self.ioloop = IOLoop.instance()
self.local_url = self.cf.get("cc-socket")
self.cur_role = self.cf.get("cc-role", "insecure")
if self.cur_role == "insecure":
self.log.warning(
'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config'
)
self.stat_level = self.cf.getint("cc-stats", 1)
if self.stat_level < 1:
self.log.warning("CC statistics level too low: %d", self.stat_level)
# initialize local listen socket
s = self.zctx.socket(zmq.XREP)
s.setsockopt(zmq.LINGER, self.zmq_linger)
s.setsockopt(zmq.HWM, self.zmq_hwm)
if self.zmq_rcvbuf > 0:
s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf)
if self.zmq_sndbuf > 0:
s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf)
if self.zmq_tcp_keepalive > 0:
if getattr(zmq, "TCP_KEEPALIVE", -1) > 0:
s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive)
s.setsockopt(zmq.TCP_KEEPALIVE_INTVL, self.zmq_tcp_keepalive_intvl)
s.setsockopt(zmq.TCP_KEEPALIVE_IDLE, self.zmq_tcp_keepalive_idle)
s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt)
else:
self.log.info("TCP_KEEPALIVE not available")
s.bind(self.local_url)
self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm)
self.local.on_recv(self.handle_cc_recv)
self.handlers = {}
self.routes = {}
rcf = skytools.Config("routes", self.cf.filename, ignore_defs=True)
for r, hnames in rcf.cf.items("routes"):
self.log.info("New route: %s = %s", r, hnames)
for hname in [hn.strip() for hn in hnames.split(",")]:
h = self.get_handler(hname)
self.add_handler(r, h)
self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop)
self.stimer.start()
def start(self):
"""Initialize and start the event loop. Listen for ZMQ control
messages."""
ctx = zmq.Context()
socket = ctx.socket(zmq.PAIR)
socket.bind("tcp://*:{}".format(settings.ZMQ_CONTROL_PORT))
logserver = LogServer()
logserver.listen(settings.TCP_LOGGING_PORT)
loop = self.loop
stream = ZMQStream(socket, loop)
stream.on_recv(self.handle_ctrl_msg)
# TODO: Stop loop and restart on CONFIG reread just to get possible new
# EVENT_POLL_INTERVAL setting?
pc = PeriodicCallback(self.chime, self.EVENT_POLL_INTERVAL * 1E3, loop)
pc.start()
loop.start()
def startup(self):
super(TaskRunner, self).startup()
self.ioloop = IOLoop.instance()
self.connect_cc()
self.ccs = CCStream (self.cc, self.ioloop, qmaxsize = self.zmq_hwm)
self.ccs.on_recv(self.handle_cc_recv)
self.local_id = self.cf.get('local-id', self.hostname)
self.reg_period = self.cf.getint('reg-period', 5 * 60)
self.maint_period = self.cf.getint('maint-period', 60)
self.grace_period = self.cf.getint('task-grace-period', 15 * 60)
self.task_heartbeat = self.cf.getboolean('task-heartbeat', False)
self.tasks = {}
self.periodic_reg()
self.timer_reg = PeriodicCallback (self.periodic_reg, self.reg_period * 1000, self.ioloop)
self.timer_reg.start()
self.timer_maint = PeriodicCallback (self.do_maint, self.maint_period * 1000, self.ioloop)
self.timer_maint.start()
