def _on_actuator_result(self, topic, headers, message, match):
"""lock result"""
msg = jsonapi.loads(message[0])
print 'Actuator Results:', match, msg
if headers['requesterID'] == agent_id:
if self.actuator_handler is not None:
self.actuator_handler(match, jsonapi.loads(message[0]))
def load_config_from_json(self):
"""load config from existing json connector files."""
c = self.config
# load from engine config
with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
cfg = json.loads(f.read())
key = c.Session.key = asbytes(cfg['exec_key'])
xport,addr = cfg['url'].split('://')
c.HubFactory.engine_transport = xport
ip,ports = addr.split(':')
c.HubFactory.engine_ip = ip
c.HubFactory.regport = int(ports)
self.location = cfg['location']
if not self.engine_ssh_server:
self.engine_ssh_server = cfg['ssh']
# load client config
with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
cfg = json.loads(f.read())
assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
xport,addr = cfg['url'].split('://')
c.HubFactory.client_transport = xport
ip,ports = addr.split(':')
c.HubFactory.client_ip = ip
if not self.ssh_server:
self.ssh_server = cfg['ssh']
assert int(ports) == c.HubFactory.regport, "regport mismatch"
def on_message(self, message):
prefix, message = message.split(",", 1)
kernel, channel = prefix.split("/", 1)
if channel=="stdin":
# TODO: Support the stdin channel
# See http://ipython.org/ipython-doc/dev/development/messaging.html
return
try:
if kernel == "complete":
application = self.session.handler.application
message = jsonapi.loads(message)
if message["header"]["msg_type"] in ("complete_request", "object_info_request"):
application.completer.registerRequest(self, message)
elif kernel not in self.channels:
# handler may be None in certain circumstances (it seems to only be set
# in GET requests, not POST requests, so even using it here may
# only work with JSONP because of a race condition)
application = self.session.handler.application
kernel_info = application.km.kernel_info(kernel)
self.kernel_info = {'remote_ip': kernel_info['remote_ip'],
'referer': kernel_info['referer'],
'timeout': kernel_info['timeout']}
self.channels[kernel] = \
{"shell": ShellSockJSHandler(kernel, self.send, application),
"iopub": IOPubSockJSHandler(kernel, self.send, application)}
self.channels[kernel]["shell"].open(kernel)
self.channels[kernel]["iopub"].open(kernel)
if kernel != "complete":
self._log_stats(kernel, message)
self.channels[kernel][channel].on_message(message)
except KeyError:
jsonmessage=jsonapi.loads(message)
logger.info("%s message sent to deleted kernel: %s"%(jsonmessage["header"]["msg_type"], kernel))
pass # Ignore messages to nonexistant or killed kernels
def iter_messages(self):
""" Yields tuples of (watcher, subtopic, stat)"""
recv = self.pubsub_socket.recv_multipart
with self:
while True:
try:
events = dict(self.poller.poll(self.timeout * 1000))
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
raise
if len(events) == 0:
continue
try:
topic, stat = recv()
except zmq.core.error.ZMQError as e:
if e.errno != errno.EINTR:
raise
else:
try:
sys.exc_clear()
except Exception:
pass
continue
topic = s(topic).split('.')
if len(topic) == 3:
__, watcher, subtopic = topic
yield watcher, subtopic, json.loads(stat)
elif len(topic) == 2:
__, watcher = topic
yield watcher, None, json.loads(stat)
def unpack_legacy_message(headers, message):
"""Unpack legacy pubsub messages for VIP agents.
Loads JSON-formatted message parts and removes single-frame messages
from their containing list. Does not alter headers.
"""
if not isinstance(headers, Headers):
headers = Headers(headers)
try:
content_type = headers["Content-Type"]
except KeyError:
return headers, message
if isinstance(content_type, basestring):
if content_type.lower() == "application/json":
if isinstance(message, list) and len(message) == 1:
return jsonapi.loads(message[0])
if isinstance(message, basestring):
return jsonapi.loads(message)
if isinstance(message, list) and len(message) == 1:
return message[0]
if isinstance(content_type, list) and isinstance(message, list):
parts = [
(jsonapi.loads(msg) if str(ctype).lower() == "application/json" else msg)
for ctype, msg in zip(content_type, message)
]
parts.extend(message[len(parts) :])
if len(parts) == len(content_type) == 1:
return parts[0]
return parts
return message
def capture_data(self, peer, sender, bus, topic, headers, message, device):
timestamp_string = headers.get(headers_mod.DATE)
timestamp, my_tz = process_timestamp(timestamp_string)
try:
# 2.0 agents compatability layer makes sender == pubsub.compat so
# we can do the proper thing when it is here
if sender == 'pubsub.compat':
message = jsonapi.loads(message[0])
if isinstance(message, dict):
values = message
else:
values = message[0]
except ValueError as e:
_log.error("message for {topic} bad message string: {message_string}".format(topic=topic,
message_string=message[0]))
return
except IndexError as e:
_log.error("message for {topic} missing message string".format(topic=topic))
return
except Exception as e:
_log.exception(e)
return
meta = {}
try:
# 2.0 agents compatability layer makes sender == pubsub.compat so
# we can do the proper thing when it is here
if sender == 'pubsub.compat':
if isinstance(message[1], str):
meta = jsonapi.loads(message[1])
if not isinstance(message, dict):
meta = message[1]
except ValueError as e:
_log.warning("meta data for {topic} bad message string: {message_string}".format(topic=topic,
message_string=message[0]))
except IndexError as e:
_log.warning("meta data for {topic} missing message string".format(topic=topic))
if topic.startswith('analysis'):
source = 'analysis'
else:
source = 'scrape'
_log.debug("Queuing {topic} from {source} for publish".format(topic=topic,
source=source))
for key, value in values.iteritems():
point_topic = device + '/' + key
self._event_queue.put({'source': source,
'topic': point_topic,
'readings': [(timestamp,value)],
'meta': meta.get(key,{})})
def test_agent_last_update_increases(volttron_instance):
agent = volttron_instance.build_agent()
s = json.loads(agent.vip.health.get_status())
dt = dateparse(s['last_updated'], fuzzy=True)
agent.vip.health.set_status(STATUS_UNKNOWN, 'Unknown now!')
gevent.sleep(1)
s = json.loads(agent.vip.health.get_status())
dt2 = dateparse(s['last_updated'], fuzzy=True)
assert dt < dt2
def test_send(self):
self.publisher.send('test', 'method', '{"temp": 20, "humid": 30}')
result = self.subscriber.recv()
self.assertEqual(result[0], b'test') # data_type
msg = json.loads(result[1])
self.assertEqual(msg[0], 'method')
data = json.loads(msg[1])
self.assertEqual(data['temp'], 20)
self.assertEqual(data['humid'], 30)
def test_call(self):
self.state_handler.update_sensor({'test_temperature': 22})
self.state_handler.update_appliance({'viera': 1})
self.state_handler.__call__()
result = self.subscriber.recv()
msg = json.loads(result[1])
self.assertEqual(msg[0], 'mining')
data = json.loads(msg[1])
self.assertEqual(data['sensors']['test_temperature'], 22)
self.assertEqual(data['appliances']['viera'], 1)
def test_agent_status_changes(volttron_instance):
unknown_message = "This is unknown"
bad_message = "Bad kitty"
agent = volttron_instance.build_agent()
agent.vip.health.set_status(STATUS_UNKNOWN, unknown_message)
r = json.loads(agent.vip.health.get_status())
assert unknown_message == r['context']
assert STATUS_UNKNOWN == r['status']
agent.vip.health.set_status(STATUS_BAD, bad_message)
r = json.loads(agent.vip.health.get_status())
assert bad_message == r['context']
assert STATUS_BAD == r['status']
def pair_recv(msg):
print "Received from Server:"
try:
try:
j = jsonapi.loads(''.join(msg))
except TypeError:
j = jsonapi.loads(msg)
if j['state'] == 'naming':
#assign_names(j['mapping'])
forward_from_mesh()
elif j['state'] == 'commanding':
send_commands(j['commanding'])
finally:
pass
def call(self, cmd):
if isinstance(cmd, string_types):
raise DeprecationWarning('call() takes a mapping')
call_id = uuid.uuid4().hex
cmd['id'] = call_id
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
try:
yield tornado.gen.Task(self.stream.send, cmd)
except zmq.ZMQError as e:
raise CallError(str(e))
while True:
messages = yield tornado.gen.Task(self.stream.on_recv)
for message in messages:
try:
res = json.loads(message)
if res.get('id') != call_id:
# we got the wrong message
continue
raise tornado.gen.Return(res)
except ValueError as e:
raise CallError(str(e))
def on_message(self, message):
prefix, json_message = message.split(",", 1)
kernel_id = prefix.split("/", 1)[0]
message = jsonapi.loads(json_message)
logger.debug("KernelConnection.on_message: %s", message)
application = self.session.handler.application
if kernel_id == "complete":
if message["header"]["msg_type"] in ("complete_request",
"object_info_request"):
application.completer.registerRequest(self, message)
return
try:
if kernel_id not in self.channels:
# handler may be None in certain circumstances (it seems to only be set
# in GET requests, not POST requests, so even using it here may
# only work with JSONP because of a race condition)
kernel_info = application.km.kernel_info(kernel_id)
self.kernel_info = {'remote_ip': kernel_info['remote_ip'],
'referer': kernel_info['referer'],
'timeout': kernel_info['timeout']}
if message["header"]["msg_type"] == "execute_request":
stats_logger.info(StatsMessage(
kernel_id=kernel_id,
remote_ip=self.kernel_info['remote_ip'],
referer=self.kernel_info['referer'],
code=message["content"]["code"],
execute_type='request'))
if kernel_id not in self.channels:
self.channels[kernel_id] = SockJSChannelsHandler(self.send)
self.channels[kernel_id].open(application, kernel_id)
self.channels[kernel_id].on_message(json_message)
except KeyError:
# Ignore messages to nonexistent or killed kernels.
logger.info("%s message sent to nonexistent kernel: %s" %
(message["header"]["msg_type"], kernel_id))
def on_message(self, message):
if self.km._kernels.get(self.kernel_id) is not None:
msg = jsonapi.loads(message)
for f in self.msg_to_kernel_callbacks:
f(msg)
self.kernel["executing"] += 1
self.session.send(self.shell_stream, msg)
def handle_frontend_messages(self, data):
"""
docstring
Frame 0: [ N ][...] <- Identity of connection
Frame 1: [ N ][...] <- Data frame
"""
logging.debug(
'Received message on frontend stream: %s',
data
)
if not self.is_valid_message(data):
logging.warning('Message validation failed')
return
identity, msg = data[0], jsonapi.loads(data[1])
is_command = msg['message'].startswith('/')
# Everything except for /CONNECT should have a valid identity
if msg['message'].split()[0] != '/CONNECT':
if not self.is_valid_identity(who=msg['who'], identity=identity):
logging.warning('Invalid identity, ignoring message')
return
if is_command:
self.process_command(msg=msg, identity=identity)
else:
self.broadcast(**msg)
def is_valid_message(self, data):
"""
docstring
"""
logging.debug('Validating message: %s', data)
if len(data) != 2:
logging.warning('Too many frames received: %s', len(data))
return False
identity, msg = data
try:
msg = jsonapi.loads(msg)
except (TypeError, ValueError) as e:
logging.warning('Cannot decode message: %s', e)
return False
# Check for required message attributes
required = ('who', 'message')
if not all(k in msg for k in required):
logging.warning('Required attributes missing in message')
return False
if not all(msg.get(k) for k in required):
logging.warning('Empty data in message found')
return False
return True
def routerRecv(self, message):
"""
message = [ ... , request, image/blank]
request = {'timestamp': timestamp,
'task': 'detection'/'recognition'/'tracking',
'parameters': (...)}
"""
request = loads(message[-2])
if request["task"] == "detection":
self.logger.debug("start detection")
with open("image.jpg", "wb") as f:
f.write(message[-1])
sleep = random.randint(1, 2) # detection
time.sleep(sleep)
message[-2] = dumps("detection")
message[-1] = ""
self.rtr.send_multipart(message)
elif request["task"] == "tracking":
self.logger.debug("prepare to tracking")
message[-1] = "finish"
tracker.Tracker(self.rtr, message)
else:
self.logger.debug("requested task is not supported")
def handle_new(self, headers, message, now):
print 'handle new'
requester = headers.get('requesterID')
self.task_id = headers.get('taskID')
priority = headers.get('priority')
try:
requests = jsonapi.loads(message[0])
requests = requests[0]
except (ValueError, IndexError) as ex:
# Could be ValueError of JSONDecodeError depending
# on if simplesjson was used. JSONDecodeError
# inherits from ValueError
#We let the schedule manager tell us this is a bad request.
_log.error('bad request: {request}, {error}'.format(request=requests, error=str(ex)))
requests = []
device, start, end = requests
self.start_time = parser.parse(start, fuzzy=True)
self.end_time = parser.parse(end, fuzzy=True)
event = sched.Event(self.announce)
self.schedule(self.start_time, event)
topic = topics.ACTUATOR_SCHEDULE_RESULT()
headers = self.get_headers(requester, task_id=self.task_id)
headers['type'] = SCHEDULE_ACTION_NEW
self.publish_json(topic, headers, {'result':'SUCCESS',
'data': 'NONE',
'info': 'NONE'})
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % str(data))
topic, msg = data
try:
topic = s(topic)
watcher = topic.split('.')[1:-1][0]
action = topic.split('.')[-1]
msg = json.loads(msg)
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
# a process was added
pid = msg['process_pid']
self._append_pid(watcher, pid)
elif action == 'stop':
# the whole watcher was stopped.
self.stop_watcher(watcher)
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def _on_new_data(self, topic, headers, message, match):
"""watching for new data"""
data = jsonapi.loads(message[0])
# self.current_spacetemp = float(data["ZoneTemp"])
self.current_spacetemp = 76
droveride = bool(int(data["CoolCall2"]))
occupied = bool(int(data["Occupied"]))
if droveride and self.state not in ('IDLE', 'CLEANUP', 'STARTUP'):
print 'User Override Initiated'
self.cancel_event()
if not occupied and self.state in ('DR_EVENT', 'RESTORE'):
self.cancel_event()
if self.state == 'IDLE' or self.state=='STARTUP':
#self.default_coolingstpt = float(data["CoolingStPt"])
#self.default_heatingstpt = float(data["HeatingStPt"])
self.default_coolingstpt = 75.0
self.default_heatingstpt = 65.0
self.default_firststage_fanspeed = float(data["CoolSupplyFanSpeed1"])
self.default_secondstage_fanspeed = float(data["CoolSupplyFanSpeed2"])
self.default_damperstpt = float(data["ESMDamperMinPosition"])
if self.state == 'STARTUP':
self.state = 'IDLE'
def __init__(self, *args, **kwargs):
super().__init__(*args)
self.latency_log = None
if kwargs['latency_debug']:
self.latency_log = list()
self.logger = logging.getLogger(logger.name+'.'+self.__class__.__name__)
self.sock = ctx.socket(zmq.SUB)
self.sock.subscribe = b''
self.sock.connect(self.endpoint)
self.logger.info('Connecting to endpoint %s', self.endpoint)
msg = self.sock.recv_multipart()
meta = ChunkMeta(*jsonapi.loads(msg[0]))
self.logger.info(
'Detected remote format: chunksize=%d channels=%d srate=%d',
meta.chunksize, meta.channels, meta.srate)
self.chunksize = meta.chunksize
self.channels = meta.channels
self.srate = meta.srate
self.seq = meta.seq
self.chunk_q = queue.Queue(kwargs['max_buffer']//self.chunksize + 1)
self.ioloop = ioloop.ZMQIOLoop(time_func=time.monotonic)
self.sockstream = zmqstream.ZMQStream(self.sock, self.ioloop)
self.sockstream.on_recv(self._enqueue)
threading.Thread(target=lambda loop: loop.start(), name='AudioOutput.IOLoop',
args=(self.ioloop,), daemon=True).start()
self._open_stream('output')
def recv_json(socket, timeout = None, ignore_exiting = False):
"""
Receives JSON from a socket. Assumes socket is set to timeout properly.
Raises universal.Exiting if program is exiting, or zmq.ZMQError if
timed out.
timeout is in milliseconds
"""
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
if timeout is not None:
start_time = time.time() * 1000
poll_wait_time = 1000 if timeout is None else min(timeout, 1000)
while ignore_exiting or not universal.exiting:
if poller.poll(poll_wait_time):
msg = socket.recv_multipart()
# Decode the json in the innermost frame
msg[-1] = jsonapi.loads(msg[-1])
# If only one frame was received simply return that frame
if len(msg) == 1: msg = msg[0]
return msg
elif timeout is not None and start_time + timeout <= time.time() * 1000:
raise Timeout()
raise universal.Exiting()
def _update_entry(self, frames):
"""
NOT USED - FOR FUTURE
Update routing table entries.
:param frames:
:return:
"""
if len(frames) > 6:
sender = bytes(frames[0])
routing_table = bytes(frames[7])
routing_table = jsonapi.loads(routing_table)
_log.debug("ROUTING SERVICE Ext routing TABLE: {0}, MY {1} ".format(routing_table, self._routing_table))
for vip_id in routing_table:
if vip_id in self._routing_table.keys():
if vip_id != self._my_vip_id:
my_route_list = self._routing_table[vip_id]
if len(routing_table[vip_id]) > 0 and len(routing_table[vip_id]) < len(my_route_list):
my_route_list = [sender]
self._routing_table[vip_id] = my_route_list.extend(routing_table[vip_id])
else:
route_list = [sender]
self._routing_table[vip_id] = route_list.extend(routing_table[vip_id])
_log.debug("ROUTING SERVICE my routing TABLE: {} ".format(self._routing_table))
return True
else:
return False
def recv_message(self, flags=0):
'''Recieve a message as (topic, headers, message) tuple.'''
topic = self.recv_string(flags)
headers = self.recv_string(flags) if self.rcvmore else ''
headers = jsonapi.loads(headers) if headers else {}
message = self.recv_multipart(flags) if self.rcvmore else []
return topic, Headers(headers), message
def __call__(self, _, nframes, time_info, status):
if status != 0:
if status & 5:
self.logger.warning('Status: underflow')
elif status & 10:
self.logger.warning('Status: overflow')
msg = self.chunk_q.get()
if len(msg[1]) == 0:
self.sockstream.stop_on_recv()
self.remake.set()
return (b'', 2)
meta = ChunkMeta(*jsonapi.loads(msg[0]))
if (meta.chunksize != self.chunksize) or \
(meta.channels != self.channels) or \
(meta.srate != self.srate):
self.logger.info('Remote format changed. Signaling for restart.')
self.sockstream.stop_on_recv()
self.remake.set()
return (b'', 2)
if meta.seq != self.seq + 1:
self.logger.warning('Sequence break: expected %d got %d', self.seq+1, meta.seq)
self.seq = meta.seq
if self.latency_log is not None:
self.latency_log.append(time_info['output_buffer_dac_time'] - meta.adc_time)
if len(self.latency_log)*self.chunksize >= self.srate*5:
self.ioloop.add_callback(self._latency_report, self.latency_log)
self.latency_log = list()
data = np.frombuffer(msg[1], dtype=meta.dtype).reshape((-1, meta.channels))
return (data.astype(np.float32), 0)
def on_message(self, message):
prefix, message = message.split(",", 1)
id = prefix.split("/", 1)[0]
message = jsonapi.loads(message)
logger.debug("SockJSHandler.on_message: %s", message)
msg_type = message["header"]["msg_type"]
app = self.session.handler.application
if id == "complete":
if msg_type in ("complete_request", "object_info_request"):
app.completer.registerRequest(self, message)
return
try:
kernel = app.kernel_dealer.kernel(id)
except KeyError:
# Ignore messages to nonexistent or killed kernels.
logger.warning("%s sent to nonexistent kernel %s", msg_type, id)
return
if id not in self.channels:
self.channels[id] = SockJSChannelsHandler(self.send)
self.channels[id].connect(kernel)
if msg_type == "execute_request":
stats_logger.info(StatsMessage(
kernel_id=id,
remote_ip=kernel.remote_ip,
referer=kernel.referer,
code=message["content"]["code"],
execute_type="request"))
self.channels[id].send(message)
def schedule_result(self, topic, headers, message, match):
msg = jsonapi.loads(message[0])
self._log.info('Schedule Request Acknowledged')
self.task_timer.cancel()
task_id = headers.get('taskID', 0)
response_type = headers.get('type', 0)
schedule_start = self.device_schedule[task_id]["schedule_start"]
event_start = schedule_start + datetime.timedelta(minutes = 1)
schedule_end = self.device_schedule[task_id]["schedule_end"]
e_start = self.device_schedule[task_id]["event_start"]
e_end = self.device_schedule[task_id]["event_end"]
if response_type == 'NEW_SCHEDULE' and self.error_handler == None:
if msg.get('result',0) == 'SUCCESS':
event = sched.Event(self.pre_cool_setup, args=[e_start, e_end])
self.schedule(event_start, event)
self.all_scheduled_events[e_start] = event
elif msg.get('result',0) =='FAILURE' and schedule_start < schedule_end:
schedule_start = schedule_start + datetime.timedelta(minutes = 10)
headers = {
'type': 'NEW_SCHEDULE',
'requesterID': agent_id,
'taskID': task_id,
'priority': 'High'
}
self.task_timer = self.periodic_timer(20, self.publish_json,
topics.ACTUATOR_SCHEDULE_REQUEST(), headers,
[["{campus}/{building}/{unit}".format(**rtu_path),str(schedule_start),schedule_end]])
elif schedule_start >= schedule_end:
return
if self.error_handler is not None:
self.error_handler()
def test_connect(self):
"""Tests that we can connect and send data to a zmq puller"""
# start dummy ZMQ pull server
gevent.spawn(self._start_zmq_puller)
self.zmq_server_listning_event.wait(5)
# our local zmq logger
zmq_url = 'tcp://localhost:{0}'.format(self.zmq_tcp_port)
client_public_key = "N[DC7+%FKdW3pJUPnaCwWxt-0/jo5Lrq&U28-GG}"
client_secret_key = "Gwt%C0a8J/:9Jy$qpDNTy8wRzlnRD-HT8H>u7F{B"
server_public_key = "^4b:-bZ8seRC+m2p(sg{7{skOuK*jInNeH^/Le}Q"
zmqLogger = ZmqLogger(zmq_url, client_public_key, client_secret_key, server_public_key)
zmqLogger.start()
# inject some data into the logging relay singleton
self.reportingRelay.queueLogData({'somekey': 'somedata'})
# wait until the zmq server put something into the local testing queue
received_data = self.testing_queue.get(5)
received_data = received_data.split(' ', 1)
topic, message = received_data[0], jsonapi.loads(received_data[1])
self.assertEqual(topic, ZmqMessageTypes.HERALDING_AUTH_LOG.value)
self.assertIn('somekey', message)
self.assertEqual(message['somekey'], 'somedata')
def update_override_patterns(self):
if self._override_patterns is None:
try:
values = self.vip.config.get("override_patterns")
values = jsonapi.loads(values)
if isinstance(values, dict):
self._override_patterns = set()
for pattern, end_time in values.items():
# check the end_time
now = utils.get_aware_utc_now()
# If end time is indefinite, set override with indefinite duration
if end_time == "0.0":
self._set_override_on(pattern, 0.0, from_config_store=True)
else:
end_time = utils.parse_timestamp_string(end_time)
# If end time > current time, set override with new duration
if end_time > now:
delta = end_time - now
self._set_override_on(pattern, delta.total_seconds(), from_config_store=True)
else:
self._override_patterns = set()
except KeyError:
self._override_patterns = set()
except ValueError:
_log.error("Override patterns is not set correctly in config store")
self._override_patterns = set()
def _on_error_result(self, topic, headers, message, match):
"""ERROR result"""
point = match.group(1)
msg = jsonapi.loads(message[0])
point = match.group(1)
today = datetime.datetime.now().date()
for key,schedule in self.device_schedule.items():
if schedule["date"] == today:
schedule_start = schedule["schedule_start"]
schedule_end = schedule["schedule_end"]
task_id = key
break
self._log.info('Error Results: '+ str(point) + ' '+ str(msg))
if msg.get('type',0) == 'LockError':
headers = {
'type': 'NEW_SCHEDULE',
'requesterID': agent_id,
'taskID': task_id,
'priority': 'HIGH'
}
self.task_timer = self.periodic_timer(20, self.publish_json,
topics.ACTUATOR_SCHEDULE_REQUEST(), headers,
[["{campus}/{building}/{unit}".format(**rtu_path),str(schedule_start),str(schedule_end)]])
elif self.error_handler is not None:
self._log.info('Running error handler')
self.error_handler()
def _on_new_data(self, topic, headers, message, match):
"""watching for new data"""
data = jsonapi.loads(message[0])
self.data_queue.notify_all(data)
def query(self,
topic,
start=None,
end=None,
skip=0,
count=None,
order="FIRST_TO_LAST"):
"""This function should return the results of a query in the form:
{"values": [(timestamp1, value1), (timestamp2, value2), ...],
"metadata": {"key1": value1, "key2": value2, ...}}
metadata is not required (The caller will normalize this to {} for you)
"""
query = '''SELECT data.ts, data.value_string
FROM data, topics
{where}
{order_by}
{limit}
{offset}'''
where_clauses = [
"WHERE topics.topic_name = %s", "topics.topic_id = data.topic_id"
]
args = [topic]
if start is not None:
where_clauses.append("data.ts > %s")
args.append(start)
if end is not None:
where_clauses.append("data.ts < %s")
args.append(end)
where_statement = ' AND '.join(where_clauses)
order_by = 'ORDER BY data.ts ASC'
if order == 'LAST_TO_FIRST':
order_by = ' ORDER BY data.ts DESC'
#can't have an offset without a limit
# -1 = no limit and allows the user to
# provied just an offset
if count is None:
count = 100
limit_statement = 'LIMIT %s'
args.append(count)
offset_statement = ''
if skip > 0:
offset_statement = 'OFFSET %s'
args.append(skip)
_log.debug("About to do real_query")
real_query = query.format(where=where_statement,
limit=limit_statement,
offset=offset_statement,
order_by=order_by)
_log.debug("Real Query: " + real_query)
_log.debug("args: " + str(args))
conn = self.connect()
cur = conn.cursor()
cur.execute(real_query, args)
rows = cur.fetchall()
if rows:
values = [(ts.isoformat(), jsonapi.loads(value))
for ts, value in rows]
else:
values = {}
cur.close()
conn.close()
return {'values': values}
def schedule_result(self, topic, headers, message, match):
msg = jsonapi.loads(message[0])
_log.debug('Actuator response received')
self.task_timer.cancel()
#-----------------------------------------------------------------------------
# default values for the thresholds:
MAX_ITEMS = 64
MAX_BYTES = 1024
# ISO8601-ify datetime objects
# allow unicode
# disallow nan, because it's not actually valid JSON
json_packer = lambda obj: jsonapi.dumps(
obj,
default=date_default,
ensure_ascii=False,
allow_nan=False,
)
json_unpacker = lambda s: jsonapi.loads(s)
pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL)
pickle_unpacker = pickle.loads
default_packer = json_packer
default_unpacker = json_unpacker
DELIM = b"<IDS|MSG>"
# singleton dummy tracker, which will always report as done
DONE = zmq.MessageTracker()
#-----------------------------------------------------------------------------
# Mixin tools for apps that use Sessions
#-----------------------------------------------------------------------------
def _run(self, _, frontend, sink, *backend_socks):
def push_new_job(_job_id, _json_msg, _msg_len):
# backend_socks[0] is always at the highest priority
_sock = backend_socks[
0] if _msg_len <= self.args.priority_batch_size else rand_backend_socket
_sock.send_multipart([_job_id, _json_msg])
# bind all sockets
self.logger.info('bind all sockets')
frontend.bind('tcp://*:%d' % self.port)
addr_front2sink = auto_bind(sink)
addr_backend_list = [auto_bind(b) for b in backend_socks]
self.logger.info('open %d ventilator-worker sockets' %
len(addr_backend_list))
# start the sink process
self.logger.info('start the sink')
proc_sink = BertSink(self.args, addr_front2sink)
self.processes.append(proc_sink)
proc_sink.start()
addr_sink = sink.recv().decode('ascii')
# start the backend processes
device_map = self._get_device_map()
for idx, device_id in enumerate(device_map):
process = BertWorker(idx, self.args, addr_backend_list, addr_sink,
device_id, self.graph_path)
self.processes.append(process)
process.start()
rand_backend_socket = None
server_status = ServerStatistic()
while True:
try:
request = frontend.recv_multipart()
except ValueError:
self.logger.error(
'received a wrongly-formatted request (expected 4 frames, got %d)'
% len(request))
self.logger.error('\n'.join('field %d: %s' % (idx, k)
for idx, k in enumerate(request)),
exc_info=True)
else:
client, msg, req_id, msg_len = request
server_status.update(request)
if msg == ServerCommand.terminate:
break
elif msg == ServerCommand.show_config:
self.logger.info(
'new config request\treq id: %d\tclient: %s' %
(int(req_id), client))
status_runtime = {
'client': client.decode('ascii'),
'num_process': len(self.processes),
'ventilator -> worker': addr_backend_list,
'worker -> sink': addr_sink,
'ventilator <-> sink': addr_front2sink,
'server_current_time': str(datetime.now()),
'statistic': server_status.value,
'device_map': device_map,
'num_concurrent_socket': self.num_concurrent_socket
}
sink.send_multipart([
client, msg,
jsonapi.dumps({
**status_runtime,
**self.status_args,
**self.status_static
}), req_id
])
else:
self.logger.info(
'new encode request\treq id: %d\tsize: %d\tclient: %s'
% (int(req_id), int(msg_len), client))
# register a new job at sink
sink.send_multipart(
[client, ServerCommand.new_job, msg_len, req_id])
# renew the backend socket to prevent large job queueing up
# [0] is reserved for high priority job
# last used backennd shouldn't be selected either as it may be queued up already
rand_backend_socket = random.choice([
b for b in backend_socks[1:]
if b != rand_backend_socket
])
# push a new job, note super large job will be pushed to one socket only,
# leaving other sockets free
job_id = client + b'#' + req_id
if int(msg_len) > self.max_batch_size:
seqs = jsonapi.loads(msg)
job_gen = ((job_id + b'@%d' % i,
seqs[i:(i + self.max_batch_size)]) for i in
range(0, int(msg_len), self.max_batch_size))
for partial_job_id, job in job_gen:
push_new_job(partial_job_id, jsonapi.dumps(job),
len(job))
else:
push_new_job(job_id, msg, int(msg_len))
self.logger.info('terminated!')
def load_message(msg):
return json.loads(msg)
if __name__ == "__main__":
#signal.signal(signal.SIGTERM, exitHandler)
signal.signal(signal.SIGINT, exitHandler)
signal.signal(signal.SIGQUIT, exitHandler)
logging.info("%s started" % os.path.basename(sys.argv[0]))
if len(sys.argv) > 1:
uwscfg = UWSConfig(sys.argv[1])
else:
uwscfg = UWSConfig()
try:
with open(uwscfg.json_moveto_path, "rb") as fh:
sensor_store_ = json.loads(fh.read())
except Exception as e:
logging.debug(e)
try:
with open(uwscfg.json_backup_path, "rb") as fh:
sensor_store_ = json.loads(fh.read())
except Exception as e:
logging.debug(e)
for k in set(sensor_store_.keys()).difference(
set(uwscfg.zmq_subscribe.split(" "))):
# del old sensordata of sensor we do not subscribe to
del sensor_store_[k]
for k in sensor_store_.keys():
try:
def main():
# Process arguments
parser = argparse.ArgumentParser()
parser.add_argument('pvbase', help='EPICS PV base (e.g. DAQ:LAB2:PART:2)')
parser.add_argument('-p',
type=int,
choices=range(0, 8),
default=0,
help='platform (default 0)')
parser.add_argument('-C',
metavar='CM_HOST',
default='localhost',
help='Collection Manager host')
parser.add_argument('-u',
metavar='UNIQUE_ID',
default='control',
help='Name')
parser.add_argument('-v', action='store_true', help='be verbose')
args = parser.parse_args()
if args.v:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s - %(levelname)s - %(message)s')
else:
logging.basicConfig(level=logging.WARNING,
format='%(asctime)s - %(levelname)s - %(message)s')
logging.info('control level starting')
ctx = zmq.Context()
coll = Collection(ctx, args.C, args.p)
pybody = {}
pybody['host'] = gethostname()
pybody['pid'] = getpid()
idbody = {}
idbody['procInfo'] = pybody
mainbody = {}
mainbody['control'] = idbody
hellomsg = CollectMsg(key=CollectMsg.HELLO, body=json.dumps(mainbody))
partition = coll.partitionInfo(hellomsg)
pprint.pprint(json.loads(partition.body))
# set up our end of connections, potentially based on the information
# about who is in the partition (e.g. number of eb/drp nodes)
# control sockets (ephemeral ports)
control_router_socket = ctx.socket(zmq.ROUTER)
control_pull_socket = ctx.socket(zmq.PULL)
control_router_socket.bind("tcp://*:*")
control_pull_socket.bind("tcp://*:*")
control_router_port = Collection.parse_port(
control_router_socket.getsockopt(zmq.LAST_ENDPOINT))
control_pull_port = Collection.parse_port(
control_pull_socket.getsockopt(zmq.LAST_ENDPOINT))
logging.debug('control_router_port = %d' % control_router_port)
logging.debug('control_pull_port = %d' % control_pull_port)
pybody = {}
pybody['router_port'] = {
'adrs': gethostname(),
'port': control_router_port
}
pybody['pull_port'] = {'adrs': gethostname(), 'port': control_pull_port}
connbody = {}
connbody['connectInfo'] = pybody
mainbody = {}
mainbody['control'] = connbody
portsmsg = CollectMsg(key=CollectMsg.CONNECTINFO,
body=json.dumps(mainbody))
connect_info = coll.connectionInfo(portsmsg)
pprint.pprint(json.loads(connect_info.body))
# now make the connections and report to CM when done
# Control state
yy = ControlStateMachine(args.pvbase)
logging.debug("ControlStateMachine state: %s" % yy.state())
poller = zmq.Poller()
poller.register(control_router_socket, zmq.POLLIN)
poller.register(control_pull_socket, zmq.POLLIN)
try:
while True:
items = dict(poller.poll(1000))
# Handle control_pull_socket socket
if control_pull_socket in items:
msg = control_pull_socket.recv()
config = dgram.Dgram(view=msg)
# now it's in dgram.Dgram object
ttt = config.seq.timestamp()
print('Timestamp:', ttt) # FIXME
# Execute state command request
if control_router_socket in items:
msg = control_router_socket.recv_multipart()
identity = msg[0]
request = msg[1]
logging.debug('Received <%s> from control_router_socket' %
request.decode())
if request == ControlMsg.PING:
# Send reply to client
logging.debug("Sending <PONG> reply")
control_router_socket.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(key=ControlMsg.PONG)
cmmsg.send(control_router_socket)
continue
if request == ControlMsg.PONG:
continue
if request in [
Transition.configure, Transition.beginrun,
Transition.enable, Transition.disable,
Transition.endrun, Transition.unconfigure,
ControlMsg.GETSTATE
]:
if request != ControlMsg.GETSTATE:
oldstate = yy.state()
# Do transition
yy.on_transition(request)
newstate = yy.state()
if newstate != oldstate:
logging.debug("ControlStateMachine state: %s" %
newstate)
# Send reply to client
control_router_socket.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(key=yy.state().key())
cmmsg.send(control_router_socket)
continue
else:
logging.warning("Unknown msg <%s>" % request.decode())
# Send reply to client
logging.debug("Sending <HUH?> reply")
control_router_socket.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(key=ControlMsg.HUH)
cmmsg.send(control_router_socket)
continue
except KeyboardInterrupt:
logging.debug("Interrupt received")
# Clean up
logging.debug("Clean up control level")
# Close all sockets associated with this context, and then
# terminate the context.
ctx.destroy(0)
logging.info('control level exiting')
def _run(self, receiver, frontend, sender):
receiver_addr = auto_bind(receiver)
frontend.connect(self.front_sink_addr)
sender.bind('tcp://*:%d' % self.port)
pending_jobs = defaultdict(
lambda: SinkJob(self.max_seq_len, self.max_position_embeddings,
self.show_tokens_to_client, self.fixed_embed_length
)) # type: Dict[str, SinkJob]
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(receiver, zmq.POLLIN)
# send worker receiver address back to frontend
frontend.send(receiver_addr.encode('ascii'))
# Windows does not support logger in MP environment, thus get a new logger
# inside the process for better compability
logger = set_logger(colored('SINK', 'green'), self.verbose)
logger.info('ready')
self.is_ready.set()
while not self.exit_flag.is_set():
socks = dict(poller.poll())
if socks.get(receiver) == zmq.POLLIN:
msg = receiver.recv_multipart()
job_id = msg[0]
# parsing job_id and partial_id
job_info = job_id.split(b'@')
job_id = job_info[0]
partial_id = int(job_info[1]) if len(job_info) == 2 else 0
if msg[3] == ServerCmd.data_embed:
# parsing the ndarray
arr_info, arr_val = jsonapi.loads(msg[1]), msg[2]
x = np.frombuffer(memoryview(arr_val),
dtype=arr_info['dtype']).reshape(
arr_info['shape'])
pending_jobs[job_id].add_embed(x, partial_id)
elif msg[3] == ServerCmd.data_token:
x = jsonapi.loads(msg[1])
pending_jobs[job_id].add_token(x, partial_id)
else:
logger.error(
'received a wrongly-formatted request (expected 4 frames, got %d)'
% len(msg))
logger.error('\n'.join('field %d: %s' % (idx, k)
for idx, k in enumerate(msg)),
exc_info=True)
logger.info(
'collect %s %s (E:%d/T:%d/A:%d)' %
(msg[3], job_id, pending_jobs[job_id].progress_embeds,
pending_jobs[job_id].progress_tokens,
pending_jobs[job_id].checksum))
if socks.get(frontend) == zmq.POLLIN:
client_addr, msg_type, msg_info, req_id = frontend.recv_multipart(
)
if msg_type == ServerCmd.new_job:
job_info = client_addr + b'#' + req_id
# register a new job
pending_jobs[job_info].checksum = int(msg_info)
logger.info('job register\tsize: %d\tjob id: %s' %
(int(msg_info), job_info))
if len(pending_jobs[job_info]._pending_embeds)>0 \
and pending_jobs[job_info].final_ndarray is None:
pending_jobs[job_info].add_embed(None, 0)
elif msg_type == ServerCmd.show_config or msg_type == ServerCmd.show_status:
time.sleep(
0.1
) # dirty fix of slow-joiner: sleep so that client receiver can connect.
logger.info('send config\tclient %s' % client_addr)
sender.send_multipart([client_addr, msg_info, req_id])
# check if there are finished jobs, then send it back to workers
finished = [(k, v) for k, v in pending_jobs.items() if v.is_done]
for job_info, tmp in finished:
client_addr, req_id = job_info.split(b'#')
x, x_info = tmp.result
sender.send_multipart([client_addr, x_info, x, req_id])
logger.info('send back\tsize: %d\tjob id: %s' %
(tmp.checksum, job_info))
# release the job
tmp.clear()
pending_jobs.pop(job_info)
def get_server_config(self):
self.socket.send(b'SHOW_CONFIG')
response = self.socket.recv_multipart()
print('the server at %s:%d returns the following config:' % (self.ip, self.port))
for k, v in jsonapi.loads(response[0]).items():
print('%30s\t=\t%-30s' % (k, v))
def _read_store(self):
if os.path.exists('email.store'):
with open('email.store', 'r') as f:
self._sent_emails = jsonapi.loads(f.read())
else:
self._sent_emails = {}
def weather_response(self, topic, headers, message, match):
data = float(jsonapi.loads(message[0]))
print data
self.weather_data_queue.notify_all(data)
def __overide(self, topic, headers, message, match):
"""watch for override from controller"""
data = jsonapi.loads(message[0])
if not bool(data):
self.tasklet = greenlet.greenlet(self.__on_override)
self.tasklet.switch()
def _recv_ndarray(self):
request_id, response = self._recv()
arr_info, arr_val = jsonapi.loads(response[1]), response[2]
X = np.frombuffer(_buffer(arr_val), dtype=str(arr_info['dtype']))
return Response(request_id,
self.formatter(X.reshape(arr_info['shape'])))
def post(self):
if 'Origin' in self.request.headers:
self.set_header('Access-Control-Allow-Origin',
self.request.headers['Origin'])
self.set_header('Access-Control-Allow-Credentials', 'true')
if (config.get('requires_tos')
and self.get_argument('accepted_tos', 'false') != 'true'):
self.set_status(403)
self.finish(
'When evaluating code, you must acknowledge your acceptance '
'of the terms of service at /static/tos.html by passing the '
'parameter accepted_tos=true\n')
return
code = ''.join(self.get_arguments('code', strip=False))
if len(code) > 65000:
self.set_status(413)
self.finish('Max code size is 65000 characters')
return
remote_ip = self.request.remote_ip
referer = self.request.headers.get('Referer', '')
self.kernel = yield tornado.gen.Task(
self.application.kernel_dealer.get_kernel,
rlimits=config.get("provider_settings")["preforked_rlimits"],
lifespan=config.get("max_lifespan"),
timeout=0)
sm = StatsMessage(kernel_id=self.kernel.id,
remote_ip=remote_ip,
referer=referer,
code=code,
execute_type='service')
if remote_ip == '127.0.0.1':
stats_logger.debug(sm)
else:
stats_logger.info(sm)
self.zmq_handler = ZMQServiceHandler()
streams = self.zmq_handler.streams
self.zmq_handler.connect(self.kernel)
loop = tornado.ioloop.IOLoop.instance()
def kernel_callback(msg):
if msg['msg_type'] == 'execute_reply':
loop.remove_timeout(self.timeout_handle)
streams['success'] = msg['content']['status'] == 'ok'
streams['execute_reply'] = msg['content']
if self.kernel.status == "idle" and 'success' in streams:
logger.debug('service request finished for %s', self.kernel.id)
loop.add_callback(self.finish_request)
self.zmq_handler.msg_from_kernel_callbacks.append(kernel_callback)
def timeout_callback():
logger.debug('service request timed out for %s', self.kernel.id)
self.kernel.stop()
self.zmq_handler.streams['success'] = False
loop.add_callback(self.finish_request)
self.timeout_handle = loop.call_later(30, timeout_callback)
exec_message = {
'channel': 'shell',
'parent_header': {},
'header': {
'msg_id': str(uuid.uuid4()),
'username': '',
'session': self.kernel.id,
'msg_type': 'execute_request',
},
'content': {
'code':
code,
'silent':
False,
'user_expressions':
jsonapi.loads(self.get_argument('user_expressions', '{}')),
'allow_stdin':
False,
},
'metadata': {},
}
self.zmq_handler.send(exec_message)
def _run(self, outputs, inputs, *receivers):
# Windows does not support logger in MP environment, thus get a new logger
# inside the process for better compatibility
logger = set_logger(colored('WORKER-%d' % self.worker_id, 'yellow'))
logger.info(
'use device %s' %
('cpu' if self.device_id < 0 else 'gpu: %d' % self.device_id))
poller = zmq.Poller()
for sock, addr in zip(receivers, self.worker_address):
sock.connect(addr)
poller.register(sock, zmq.POLLIN)
outputs.connect(self.sink_address)
inputs.connect(self.sink_address)
logger.info('ready and listening!')
self.is_ready.set()
while not self.exit_flag.is_set():
events = dict(poller.poll())
for sock_idx, sock in enumerate(receivers):
if sock in events:
client_id, msg_type, raw_msg = sock.recv_multipart()
msg = jsonapi.loads(raw_msg)
if msg_type == ServerCmd.initiate:
self.model = SequenceTaggingModel()
self.modelid = str(msg)
if os.path.isfile(
os.path.join('.', 'model' + self.modelid +
'.pre')) and os.path.isfile(
os.path.join(
'.', 'model' +
self.modelid + '.pt')):
self.model.load('model' + self.modelid)
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, 1, client_id))
helper.send_test(outputs, client_id,
b'Model Loaded', ServerCmd.load)
logger.info('job done\tsize: %s\tclient: %s' %
(1, client_id))
else:
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, 1, client_id))
helper.send_test(outputs, client_id,
b'Model Initiated', msg_type)
logger.info('job done\tsize: %s\tclient: %s' %
(1, client_id))
elif msg_type == ServerCmd.online_initiate:
self.model.online_word_build(
msg[0], msg[1]
) # whole unlabeled training sentences / predefined_labels
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, len(msg[0]), client_id))
helper.send_test(outputs, client_id,
b'Online word build completed',
msg_type)
logger.info('job done\tsize: %s\tclient: %s' %
(len(msg[0]), client_id))
elif msg_type == ServerCmd.online_learning:
self.model.online_learning(msg[0], msg[1], msg[2],
msg[3])
self.model.save('model' + self.modelid)
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, len(msg[0]), client_id))
helper.send_test(outputs, client_id,
b'Online learning completed',
msg_type)
logger.info('job done\tsize: %s\tclient: %s' %
(len(msg[0]), client_id))
elif msg_type == ServerCmd.predict:
analyzed_result = self.model.analyze(msg)
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, 1, client_id))
helper.send_test(outputs, client_id,
jsonapi.dumps(analyzed_result),
msg_type)
logger.info('job done\tsize: %s\tclient: %s' %
(1, client_id))
elif msg_type == ServerCmd.active_learning:
indices, scores = self.model.active_learning(
msg[0], msg[1])
json_indices = list(map(int, indices))
json_scores = list(map(float, scores))
active_data = {
'indices': json_indices,
'scores': json_scores,
}
logger.info(
'new job\tsocket: %d\tsize: %d\tclient: %s' %
(sock_idx, len(msg[0]), client_id))
helper.send_test(outputs, client_id,
jsonapi.dumps(active_data), msg_type)
logger.info('job done\tsize: %s\tclient: %s' %
(len(msg[0]), client_id))
def convertToPython(content):
# return session.json_unpacker(content)
# Old way: this needs to be deleted if the above line is enough
return jsonapi.loads(content)
def on_message(self, msg):
self.send(jsonapi.loads(msg))
def _run(self, receiver, frontend, sender):
receiver_addr = auto_bind(receiver)
frontend.connect(self.front_sink_addr)
sender.bind('tcp://*:%d' % self.port)
pending_checksum = defaultdict(int)
pending_result = defaultdict(list)
job_checksum = defaultdict(int)
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(receiver, zmq.POLLIN)
# send worker receiver address back to frontend
frontend.send(receiver_addr.encode('ascii'))
# Windows does not support logger in MP environment, thus get a new logger
# inside the process for better compability
logger = set_logger(colored('SINK', 'green'), self.verbose)
logger.info('ready')
while not self.exit_flag.is_set():
socks = dict(poller.poll())
if socks.get(receiver) == zmq.POLLIN:
msg = receiver.recv_multipart()
job_id = msg[0]
# parsing the ndarray
arr_info, arr_val = jsonapi.loads(msg[1]), msg[2]
X = np.frombuffer(memoryview(arr_val), dtype=arr_info['dtype'])
X = X.reshape(arr_info['shape'])
job_info = job_id.split(b'@')
job_id = job_info[0]
partial_id = job_info[1] if len(job_info) == 2 else 0
pending_result[job_id].append((X, partial_id))
pending_checksum[job_id] += X.shape[0]
logger.info(
'collect job %s (%d/%d)' %
(job_id, pending_checksum[job_id], job_checksum[job_id]))
# check if there are finished jobs, send it back to workers
finished = [(k, v) for k, v in pending_result.items()
if pending_checksum[k] == job_checksum[k]]
for job_info, tmp in finished:
logger.info('send back\tsize: %d\tjob id:%s\t' %
(job_checksum[job_info], job_info))
# re-sort to the original order
tmp = [x[0] for x in sorted(tmp, key=lambda x: int(x[1]))]
client_addr, req_id = job_info.split(b'#')
send_ndarray(sender, client_addr,
np.concatenate(tmp, axis=0), req_id)
pending_result.pop(job_info)
pending_checksum.pop(job_info)
job_checksum.pop(job_info)
if socks.get(frontend) == zmq.POLLIN:
client_addr, msg_type, msg_info, req_id = frontend.recv_multipart(
)
if msg_type == ServerCommand.new_job:
job_info = client_addr + b'#' + req_id
job_checksum[job_info] = int(msg_info)
logger.info('job register\tsize: %d\tjob id: %s' %
(int(msg_info), job_info))
elif msg_type == ServerCommand.show_config:
time.sleep(
0.1
) # dirty fix of slow-joiner: sleep so that client receiver can connect.
logger.info('send config\tclient %s' % client_addr)
sender.send_multipart([client_addr, msg_info, req_id])
obj[squash_unicode(key)] = obj.pop(key)
elif isinstance(obj, list):
for i, v in enumerate(obj):
obj[i] = squash_unicode(v)
elif isinstance(obj, unicode):
obj = obj.encode('utf8')
return obj
#-----------------------------------------------------------------------------
# globals and defaults
#-----------------------------------------------------------------------------
key = 'on_unknown' if jsonapi.jsonmod.__name__ == 'jsonlib' else 'default'
json_packer = lambda obj: jsonapi.dumps(obj, **{key: date_default})
json_unpacker = lambda s: extract_dates(jsonapi.loads(s))
pickle_packer = lambda o: pickle.dumps(o, -1)
pickle_unpacker = pickle.loads
default_packer = json_packer
default_unpacker = json_unpacker
DELIM = b"<IDS|MSG>"
#-----------------------------------------------------------------------------
# Mixin tools for apps that use Sessions
#-----------------------------------------------------------------------------
session_aliases = dict(
ident='Session.session',
def on_new_data(self, topic, headers, message, match):
data = jsonapi.loads(message[0])
self.data_queue.notify_all(data)
from crate_historian import crate_utils
from volttron.platform.agent import utils
from volttron.platform.dbutils import mongoutils
logging.basicConfig(level=logging.DEBUG)
_log = logging.getLogger(__name__)
for key in logging.Logger.manager.loggerDict:
_log.debug(key)
logging.getLogger('crate.client.http').setLevel(logging.INFO)
logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO)
root = os.path.dirname(os.path.abspath(__file__))
with open('{}/crate_config'.format(root), 'r') as fp:
crate_params = jsonapi.loads(fp.read())
root = os.path.dirname(os.path.abspath(__file__))
with open('{}/mongo_config'.format(root), 'r') as fp:
mongo_params = jsonapi.loads(fp.read())
MAX_QUEUE_SIZE = 50000
QUEUE_BATCH_SIZE = 5000
class TableQueue(Queue.Queue, object):
def __init__(self, table_name):
super(TableQueue, self).__init__()
self.table_name = table_name
break
if not should_parse_json:
sys.stderr.write(
"Invalid json file detected, must start with { or [ character.\n"
)
sys.exit(1)
# Yaml failed for some reason, could be invalid yaml or could
# have embedded invalid character in a json file. So now we
# are going to try to deal with json here.
tmpconfigfile = tempfile.NamedTemporaryFile()
opts.config.seek(0)
with open(tmpconfigfile.name, 'w') as fout:
for line in opts.config:
line = line.partition('#')[0]
if line.rstrip():
fout.write(line.rstrip() + "\n")
config_file = tmpconfigfile.name
try:
with open(tmpconfigfile.name) as f:
opts.config = jsonapi.loads(f.read())
finally:
tmpconfigfile.close()
if opts.config:
install_agent(opts, opts.package, opts.config)
else:
install_agent(opts, opts.package, {})
def capture_data(self, peer, sender, bus, topic, headers, message, device):
timestamp_string = headers.get(headers_mod.DATE)
timestamp, my_tz = process_timestamp(timestamp_string)
try:
# 2.0 agents compatability layer makes sender == pubsub.compat so
# we can do the proper thing when it is here
if sender == 'pubsub.compat':
message = jsonapi.loads(message[0])
if isinstance(message, dict):
values = message
else:
values = message[0]
except ValueError as e:
_log.error(
"message for {topic} bad message string: {message_string}".
format(topic=topic, message_string=message[0]))
return
except IndexError as e:
_log.error("message for {topic} missing message string".format(
topic=topic))
return
except Exception as e:
_log.exception(e)
return
meta = {}
try:
# 2.0 agents compatability layer makes sender == pubsub.compat so
# we can do the proper thing when it is here
if sender == 'pubsub.compat':
if isinstance(message[1], str):
meta = jsonapi.loads(message[1])
if not isinstance(message, dict):
meta = message[1]
except ValueError as e:
_log.warning(
"meta data for {topic} bad message string: {message_string}".
format(topic=topic, message_string=message[0]))
except IndexError as e:
_log.warning("meta data for {topic} missing message string".format(
topic=topic))
if topic.startswith('analysis'):
source = 'analysis'
else:
source = 'scrape'
_log.debug("Queuing {topic} from {source} for publish".format(
topic=topic, source=source))
for key, value in values.iteritems():
point_topic = device + '/' + key
self._event_queue.put({
'source': source,
'topic': point_topic,
'readings': [(timestamp, value)],
'meta': meta.get(key, {})
})
def _run(self, receiver, frontend, sender):
receiver_addr = auto_bind(receiver)
frontend.connect(self.nav_to_sink_addr)
sender.bind('tcp://*:%d' % self.port)
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(receiver, zmq.POLLIN)
# send worker receiver address back to frontend
frontend.send(receiver_addr.encode('ascii'))
# Windows does not support logger in MP environment, thus get a new logger
# inside the process for better compability
logger = set_logger(colored('SINK', 'green'),
logger_dir=self.logdir,
verbose=self.verbose)
logger.info('ready')
self.is_ready.set()
sink_status = ServerStatistic()
while not self.exit_flag.is_set():
try:
socks = dict(poller.poll())
if socks.get(receiver) == zmq.POLLIN:
client, req_id, msg, msg_info = recv_from_prev_raw(
receiver)
logger.info("collected {}#{}".format(client, req_id))
send_to_next_raw(client, req_id, msg, msg_info, sender)
self.current_jobnum -= 1
self.total_processed += 1
logger.info('send back\tjob id: {}#{} \tleft: {}'.format(
client, req_id, self.current_jobnum))
if socks.get(frontend) == zmq.POLLIN:
client_addr, msg_type, msg_info, req_id = frontend.recv_multipart(
)
if msg_type == ServerCmd.new_job:
job_id = client_addr + b'#' + req_id
self.current_jobnum += 1
self.maximum_jobnum = self.current_jobnum if self.current_jobnum > self.maximum_jobnum else self.maximum_jobnum
logger.info(
'registed job\tjob id: {}\tleft: {}'.format(
job_id, self.current_jobnum))
elif msg_type == ServerCmd.show_config:
time.sleep(
0.1
) # dirty fix of slow-joiner: sleep so that client receiver can connect.
logger.info('send config\tclient %s' % client_addr)
prev_status = jsonapi.loads(msg_info)
status = {
'statistic_postsink': {
**{
'total_job_in_queue':
self.current_jobnum,
'maximum_job_in_queue':
self.maximum_jobnum,
'total_processed_job':
self.total_processed,
'util':
self.current_jobnum / (self.maximum_jobnum) if self.maximum_jobnum > 0 else 0
},
**sink_status.value
}
}
send_to_next('obj', client_addr, req_id, {
**prev_status,
**status
}, sender)
except Exception as e:
import traceback
traceback.print_exc()
tb = traceback.format_exc()
logger.error('{}\n{}'.format(e, tb))
def handle_subsystem(self, frames):
"""
Handler for incoming routing table frames. It calls appropriate action handler based on operation request.
:param frames list of frames
:type frames list
:returns: response frame to be sent back to the sender
:rtype: list
:Return Values:
response frame to be sent back to the sender
"""
response = []
result = False
try:
sender, recipient, proto, usr_id, msg_id, subsystem, op = frames[:
7]
except IndexError:
return False
subsystem = bytes(subsystem)
op = bytes(op)
# for f in frames:
# _log.debug("ROUTINGSERVICE handle subsystem {}".format(bytes(f)))
if subsystem == b'routing_table':
#If Setup mode of operation, setup authorization
if op == b'setupmode_platform_connection':
instance_config = bytes(frames[7])
instance_config = jsonapi.loads(instance_config)
self._setup_authorization(instance_config)
# If Normal mode of operation, build authorized connection
elif op == b'normalmode_platform_connection':
instance_config = bytes(frames[7])
instance_config = jsonapi.loads(instance_config)
self._build_connection(instance_config)
return False
#Respond to Hello/Welcome messages from other instances
elif op == b'hello':
handshake_request = bytes(frames[7])
try:
#Respond to 'hello' request with 'welcome'
if handshake_request == b'hello':
name = bytes(frames[8])
frames.pop(0)
_log.debug(
"HELLO Recieved hello, sending welcome to {}".
format(name))
frames[6] = 'welcome'
frames[7] = self._my_instance_name
try:
_log.debug(
"Sending welcome message to sender {}".format(
name))
self.send_external(name, frames)
except ZMQError as exc:
_log.error("ZMQ error: ")
#Respond to 'welcome' response by sending Pubsub subscription list
elif handshake_request == b'welcome':
name = bytes(frames[8])
_log.debug(
"HELLO Received welcome. Connection established with: {}"
.format(name))
try:
self._instances[name]['status'] = STATUS_CONNECTED
self._onconnect_pubsub_handler(name)
except KeyError as exc:
_log.error(
"Welcome message received from unknown platform: {}"
.format(name))
except IndexError as exc:
_log.error(
"Insufficient frames in hello message {}".format(exc))
elif op == b"web-addresses":
self._web_addresses = bytes(frames[7])
self._web_addresses = jsonapi.loads(self._web_addresses)
#Update routing table entry
elif op == b'update':
result = self._update_entry(frames)
elif op == b'request_response':
pass
else:
_log.error("Unknown operation: {}".format(op))
if result:
#Form response frame
response = [sender, recipient, proto, usr_id, msg_id, subsystem]
response.append(zmq.Frame(b'request_response'))
response.append(zmq.Frame(bytes(result)))
else:
response = False
return response
def on_message(self, msg):
if len(msg) < self.max_msg_size:
msg = jsonapi.loads(msg)
self.session.send(self.shell_stream, msg)
def deserialze_data(self, data):
return jsonapi.loads(data, object_hook=utils.json_numpy_obj_hook)
def decodeR3Message(multipart_msg):
try:
return (multipart_msg[0], json.loads(multipart_msg[1]))
except Exception as e:
logging.debug("decodeR3Message:" + str(e))
return ("", {})
def _recv_ndarray(self, wait_for_req_id=None):
request_id, response = self._recv(wait_for_req_id)
arr_info, arr_val = jsonapi.loads(response[1]), response[2]
X = np.frombuffer(_buffer(arr_val), dtype=str(arr_info['dtype']))
return Response(request_id, self.formatter(X.reshape(arr_info['shape'])), arr_info.get('tokens', ''))
