def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid",
errno=errors.INVALID_JSON)
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
if cmd_name is None:
error = "no cmd: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.MESSAGE_ERROR)
except OSError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb, cast=cast,
errno=errors.COMMAND_ERROR)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def handle_autodiscover_message(self, fd_no, type):
data, address = self.udp_socket.recvfrom(1024)
data = json.loads(data)
self.udp_socket.sendto(
json.dumps({
'endpoint': self.endpoint
}), address)
def update(x_key, y_key, init_sequence, subscriber, plot):
"""Add a data point to a given plot.
Parameters
----------
x_key : str
The key in the serialized JSON object that contains the x-axis
value.
y_key : str
See `x_key`.
init_sequence : int
The sequence number to start plotting with; entries with lower
sequence numbers will be ignored (they were probably already
plotted when receiving the snapshot).
subscriber : ZMQ socket
The ZMQ socket to receive the sequence number and JSON data over.
plot : Bokeh plot
The Bokeh plot whose data source will be updated.
"""
sequence = int(subscriber.recv())
entry = json.loads(subscriber.recv_string())
if sequence > init_sequence and x_key in entry and y_key in entry:
# Mutating data source in place doesn't work
x = plot.data_source.data['x'] + [entry[x_key]]
y = plot.data_source.data['y'] + [entry[y_key]]
plot.data_source.data['x'] = x
plot.data_source.data['y'] = y
class CircusClient(object):
def __init__(self,
context=None,
endpoint=DEFAULT_ENDPOINT_DEALER,
timeout=5.0,
ssh_server=None,
ssh_keyfile=None):
self.context = context or zmq.Context.instance()
self.endpoint = endpoint
self._id = uuid.uuid4().hex
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, self._id)
self.socket.setsockopt(zmq.LINGER, 0)
get_connection(self.socket, endpoint, ssh_server, ssh_keyfile)
self.poller = zmq.Poller()
self.poller.register(self.socket, zmq.POLLIN)
self._timeout = timeout
self.timeout = timeout * 1000
def stop(self):
self.socket.close()
def send_message(self, command, **props):
return self.call(make_message(command, **props))
def call(self, cmd):
if not isinstance(cmd, string_types):
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
try:
self.socket.send(cmd)
except zmq.ZMQError, e:
raise CallError(str(e))
while True:
try:
events = dict(self.poller.poll(self.timeout))
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise CallError(str(e))
else:
break
if len(events) == 0:
raise CallError("Timed out.")
for socket in events:
msg = socket.recv()
try:
return json.loads(msg)
except ValueError as e:
raise CallError(str(e))
def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid",
errno=errors.INVALID_JSON)
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.MESSAGE_ERROR)
except OSError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb, cast=cast,
errno=errors.COMMAND_ERROR)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg,
str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid")
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e))
except OSError as e:
return self.send_error(cid, msg, str(e))
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg,
str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error")
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def call(self, command, **props):
"""Sends to **circusd** the command.
Options:
- **command** -- the command to call
- **props** -- keywords argument to add to the call
Returns the JSON mapping sent back by **circusd**
"""
msg = make_message(command, **props)
self.client.send(json.dumps(msg))
msg = self.client.recv()
return json.loads(msg)
def call(self, command, **props):
"""Sends the command to **circusd**
Options:
- **command** -- the command to call
- **props** -- keyword arguments to add to the call
Returns the JSON mapping sent back by **circusd**
"""
msg = make_message(command, **props)
self.client.send(json.dumps(msg))
msg = self.client.recv()
return json.loads(msg)
class CircusClient(object):
def __init__(self, context=None, endpoint='tcp://127.0.0.1:5555',
timeout=5.0):
self.context = context or zmq.Context.instance()
self.endpoint = endpoint
self._id = uuid.uuid4().hex
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, self._id)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.connect(endpoint)
self.poller = zmq.Poller()
self.poller.register(self.socket, zmq.POLLIN)
self.timeout = timeout * 1000
def stop(self):
self.socket.close()
def call(self, cmd):
if not isinstance(cmd, string_types):
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
try:
self.socket.send(cmd)
except zmq.ZMQError, e:
raise CallError(str(e))
while True:
try:
events = dict(self.poller.poll(self.timeout))
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise CallError(str(e))
else:
break
if len(events) == 0:
raise CallError("Timed out")
for socket in events:
msg = socket.recv()
try:
return json.loads(msg)
except ValueError as e:
raise CallError(str(e))
def call(self, cmd, callback):
if not isinstance(cmd, string_types):
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
socket = self.context.socket(zmq.DEALER)
socket.setsockopt(zmq.IDENTITY, uuid.uuid4().hex)
socket.setsockopt(zmq.LINGER, 0)
get_connection(socket, self.endpoint, self.ssh_server,
self.ssh_keyfile)
if callback:
stream = ZMQStream(socket, self.loop)
def timeout_callback():
stream.stop_on_recv()
stream.close()
raise CallError('Call timeout for cmd', cmd)
timeout = self.loop.add_timeout(timedelta(seconds=5),
timeout_callback)
def recv_callback(msg):
self.loop.remove_timeout(timeout)
stream.stop_on_recv()
stream.close()
callback(json.loads(msg[0]))
stream.on_recv(recv_callback)
try:
socket.send(cmd)
except zmq.ZMQError as e:
raise CallError(str(e))
if not callback:
return json.loads(socket.recv())
def call(self, cmd, callback):
if not isinstance(cmd, string_types):
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
socket = self.context.socket(zmq.DEALER)
socket.setsockopt_string(zmq.IDENTITY, uuid.uuid4().hex)
socket.setsockopt(zmq.LINGER, 0)
get_connection(socket, self.endpoint, self.ssh_server,
self.ssh_keyfile)
if callback:
stream = ZMQStream(socket, self.loop)
def timeout_callback():
stream.stop_on_recv()
stream.close()
raise CallError('Call timeout for cmd', cmd)
timeout = self.loop.add_timeout(timedelta(seconds=5),
timeout_callback)
def recv_callback(msg):
self.loop.remove_timeout(timeout)
stream.stop_on_recv()
stream.close()
callback(json.loads(msg[0].decode('utf-8')))
stream.on_recv(recv_callback)
try:
socket.send_string(cmd)
except zmq.ZMQError as e:
raise CallError(str(e))
if not callback:
return json.loads(socket.recv().decode('utf-8'))
def call(self, command, **props):
msg = make_message(command, **props)
self.client.send(json.dumps(msg))
msg = self.client.recv()
return json.loads(msg)
import zmq
from zmq.utils.jsonapi import jsonmod as json
ctx = zmq.Context()
subscriber = ctx.socket(zmq.SUB)
subscriber.linger = 0
subscriber.setsockopt(zmq.SUBSCRIBE, b'')
subscriber.connect("tcp://localhost:5557")
while True:
sequence = int(subscriber.recv())
entry = json.loads(subscriber.recv_string())
print('{}: {}'.format(sequence, entry))
def connect(x_key, y_key, push_port=5557, router_port=5556, persistent=True):
"""Connect to a socket.
If connected to a persistent server, a snapshot of data will be
requested.
Parameters
----------
x_key : str
The key in the serialized JSON object that contains the x-axis
value.
y_key : str
See `x_key`.
push_port : int
The port over which entries are pushed by the server.
router_port : int
THe port over which requests for snapshots are sent and snapshots
received.
persistent : bool
If True, the server is assumed to have a snapshot of the data and
will be asked for it i.e. the `stream_maxlen` argument was
non-zero. If False, only new data will come in.
Returns
-------
subscriber : ZMQ socket
The socket over which log entries are streamed.
sequence : int
The sequence number of the last log entry that was received (always
0 in case `persistent` is set to `False`.
x : list
A list of the x values received as part of the snapshot. If
`persistent` is `False` this is an empty list.
y : list
See `x`.
"""
ctx = zmq.Context()
subscriber = ctx.socket(zmq.SUB)
subscriber.linger = 0
subscriber.setsockopt(zmq.SUBSCRIBE, b'')
subscriber.connect("tcp://localhost:{}".format(push_port))
sequence = 0
x, y = [], []
if persistent:
snapshot = ctx.socket(zmq.DEALER)
snapshot.linger = 0
snapshot.connect("tcp://localhost:{}".format(router_port))
snapshot.send(b'ICANHAZ?')
while True:
sequence = int(snapshot.recv())
entry = json.loads(snapshot.recv_string())
if sequence < 0:
break
if x_key in entry and y_key in entry:
x.append(entry[x_key])
y.append(entry[y_key])
return subscriber, sequence, x, y
def handle_autodiscover_message(self, fd_no, type):
data, address = self.udp_socket.recvfrom(1024)
data = json.loads(data)
self.udp_socket.sendto(json.dumps({'endpoint': self.endpoint}),
address)
def call(self, cmd):
self.client.send(json.dumps(cmd))
msg = self.client.recv()
return json.loads(msg)
def recv_callback(msg):
self.loop.remove_timeout(timeout)
stream.stop_on_recv()
stream.close()
callback(json.loads(msg[0].decode('utf-8')))
def dispatch(self, job, future=None):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(None,
cid,
msg,
"json invalid",
errno=errors.INVALID_JSON)
mid = json_msg.get('id')
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error_ = "unknown command: %r" % cmd_name
return self.send_error(mid,
cid,
msg,
error_,
cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
if isinstance(resp, Future):
if properties.get('waiting', False):
cb = functools.partial(self._dispatch_callback_future, msg,
cid, mid, cast, cmd_name, True)
resp.add_done_callback(cb)
else:
cb = functools.partial(self._dispatch_callback_future, msg,
cid, mid, cast, cmd_name, False)
resp.add_done_callback(cb)
self._dispatch_callback(msg, cid, mid, cast, cmd_name,
None)
else:
self._dispatch_callback(msg, cid, mid, cast, cmd_name, resp)
except MessageError as e:
return self.send_error(mid,
cid,
msg,
str(e),
cast=cast,
errno=errors.MESSAGE_ERROR)
except ConflictError as e:
if self._managing_watchers_future is not None:
logger.debug("the command conflicts with running "
"manage_watchers, re-executing it at "
"the end")
cb = functools.partial(self.dispatch, job)
self.loop.add_future(self._managing_watchers_future, cb)
return
# conflicts between two commands, sending error...
return self.send_error(mid,
cid,
msg,
str(e),
cast=cast,
errno=errors.COMMAND_ERROR)
except OSError as e:
return self.send_error(mid,
cid,
msg,
str(e),
cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(mid,
cid,
msg,
reason,
tb,
cast=cast,
errno=errors.COMMAND_ERROR)
ctx = zmq.Context()
snapshot = ctx.socket(zmq.DEALER)
snapshot.linger = 0
snapshot.connect("tcp://localhost:5556")
subscriber = ctx.socket(zmq.SUB)
subscriber.linger = 0
subscriber.setsockopt(zmq.SUBSCRIBE, b'')
subscriber.connect("tcp://localhost:5557")
store = {}
sequence = 0
snapshot.send(b'ICANHAZ?')
while True:
sequence = int(snapshot.recv())
entry = json.loads(snapshot.recv_string())
if sequence < 0:
break
store[sequence] = entry
print('{}: {}'.format(sequence, entry))
while True:
sequence = int(subscriber.recv())
entry = json.loads(subscriber.recv_string())
if sequence not in store:
store[sequence] = entry
print('{}: {}'.format(sequence, entry))
def call(self, command, **props):
msg = make_message(command, **props)
self.client.send(json.dumps(msg))
msg = self.client.recv()
return json.loads(msg)
def recv_callback(msg):
self.loop.remove_timeout(timeout)
stream.stop_on_recv()
stream.close()
callback(json.loads(msg[0]))
def dispatch(self, job, future=None):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(None, cid, msg, "json invalid",
errno=errors.INVALID_JSON)
mid = json_msg.get('id')
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error_ = "unknown command: %r" % cmd_name
return self.send_error(mid, cid, msg, error_, cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
if isinstance(resp, Future):
if properties.get('waiting', False):
cb = functools.partial(self._dispatch_callback_future, msg,
cid, mid, cast, cmd_name, True)
resp.add_done_callback(cb)
else:
cb = functools.partial(self._dispatch_callback_future, msg,
cid, mid, cast, cmd_name, False)
resp.add_done_callback(cb)
self._dispatch_callback(msg, cid, mid, cast,
cmd_name, None)
else:
self._dispatch_callback(msg, cid, mid, cast,
cmd_name, resp)
except MessageError as e:
return self.send_error(mid, cid, msg, str(e), cast=cast,
errno=errors.MESSAGE_ERROR)
except ConflictError as e:
if self._managing_watchers_future is not None:
logger.debug("the command conflicts with running "
"manage_watchers, re-executing it at "
"the end")
cb = functools.partial(self.dispatch, job)
self.loop.add_future(self._managing_watchers_future, cb)
return
# conflicts between two commands, sending error...
return self.send_error(mid, cid, msg, str(e), cast=cast,
errno=errors.COMMAND_ERROR)
except OSError as e:
return self.send_error(mid, cid, msg, str(e), cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(mid, cid, msg, reason, tb, cast=cast,
errno=errors.COMMAND_ERROR)
