def unpack_legacy_message(headers, message): '''Unpack legacy pubsub messages for VIP agents. Loads JSON-formatted message parts and removes single-frame messages from their containing list. Does not alter headers. ''' if not isinstance(headers, Headers): headers = Headers(headers) try: content_type = headers['Content-Type'] except KeyError: return headers, message if isinstance(content_type, basestring): if content_type.lower() == 'application/json': if isinstance(message, list) and len(message) == 1: return jsonapi.loads(message[0]) if isinstance(message, basestring): return jsonapi.loads(message) if isinstance(message, list) and len(message) == 1: return message[0] if isinstance(content_type, list) and isinstance(message, list): parts = [(jsonapi.loads(msg) if str(ctype).lower() == 'application/json' else msg) for ctype, msg in zip(content_type, message)] parts.extend(message[len(parts):]) if len(parts) == len(content_type) == 1: return parts[0] return parts return message
def unpack_legacy_message(headers, message): '''Unpack legacy pubsub messages for VIP agents. Loads JSON-formatted message parts and removes single-frame messages from their containing list. Does not alter headers. ''' if not isinstance(headers, Headers): headers = Headers(headers) try: content_type = headers['Content-Type'] except KeyError: return headers, message if isinstance(content_type, str): if content_type.lower() == 'application/json': if isinstance(message, list) and len(message) == 1: return jsonapi.loads(message[0]) if isinstance(message, str): return jsonapi.loads(message) if isinstance(message, list) and len(message) == 1: return message[0] if isinstance(content_type, list) and isinstance(message, list): parts = [(jsonapi.loads(msg) if str(ctype).lower() == 'application/json' else msg) for ctype, msg in zip(content_type, message)] parts.extend(message[len(parts):]) if len(parts) == len(content_type) == 1: return parts[0] return parts return message
def test_agent_last_update_increases(volttron_instance): agent = volttron_instance.build_agent() s = json.loads(agent.vip.health.get_status()) dt = dateparse(s['last_updated'], fuzzy=True) agent.vip.health.set_status(STATUS_UNKNOWN, 'Unknown now!') gevent.sleep(1) s = json.loads(agent.vip.health.get_status()) dt2 = dateparse(s['last_updated'], fuzzy=True) assert dt < dt2
def test_agent_last_update_increases(volttron_instance): agent = volttron_instance.build_agent() s = json.loads(agent.vip.health.get_status()) dt = dateparse(s['last_updated'], fuzzy=True) agent.vip.health.set_status(STATUS_UNKNOWN, 'Unknown now!') gevent.sleep(1) s = json.loads(agent.vip.health.get_status()) dt2 = dateparse(s['last_updated'], fuzzy=True) assert dt < dt2
def _process_incoming_message(self, message): """Process incoming messages param message: VIP message from PubSubService type message: dict """ op = message.args[0].bytes if op == 'request_response': result = None try: result = self._results.pop(bytes(message.id)) except KeyError: pass if self._parameters_needed: self._send_via_rpc = False self._parameters_needed = False self._pubsubwithrpc.clear_parameters() del self._pubsubwithrpc response = message.args[1].bytes # _log.debug("Message result: {}".format(response)) if result: result.set(response) elif op == 'publish': try: topic = topic = message.args[1].bytes data = message.args[2].bytes except IndexError: return try: msg = jsonapi.loads(data) headers = msg['headers'] message = msg['message'] sender = msg['sender'] bus = msg['bus'] except KeyError as exc: _log.error("Missing keys in pubsub message: {}".format(exc)) else: self._process_callback(sender, bus, topic, headers, message) elif op == 'list_response': result = None try: result = self._results.pop(bytes(message.id)) response = jsonapi.loads(message.args[1].bytes) if result: result.set(response) except KeyError: pass else: _log.error("Unknown operation ({})".format(op))
def test_agent_status_changes(volttron_instance): unknown_message = "This is unknown" bad_message = "Bad kitty" agent = volttron_instance.build_agent() agent.vip.health.set_status(STATUS_UNKNOWN, unknown_message) r = json.loads(agent.vip.health.get_status()) assert unknown_message == r['context'] assert STATUS_UNKNOWN == r['status'] agent.vip.health.set_status(STATUS_BAD, bad_message) r = json.loads(agent.vip.health.get_status()) assert bad_message == r['context'] assert STATUS_BAD == r['status']
def test_agent_status_changes(volttron_instance): unknown_message = "This is unknown" bad_message = "Bad kitty" agent = volttron_instance.build_agent() agent.vip.health.set_status(STATUS_UNKNOWN, unknown_message) r = json.loads(agent.vip.health.get_status()) assert unknown_message == r['context'] assert STATUS_UNKNOWN == r['status'] agent.vip.health.set_status(STATUS_BAD, bad_message) r = json.loads(agent.vip.health.get_status()) assert bad_message == r['context'] assert STATUS_BAD == r['status']
def _update_external_subscriptions(self, frames): """ Store external subscriptions :param frames: frames containing external subscriptions :return: """ results = [] if len(frames) <= 7: return False else: data = frames[7].bytes msg = jsonapi.loads(data) try: for instance_name in msg: prefixes = msg[instance_name] # Store external subscription list for later use (during publish) self._ext_subscriptions[instance_name] = prefixes self._logger.debug( "PUBSUBSERVICE New external list from {0}: List: {1}". format(instance_name, self._ext_subscriptions)) except KeyError as exc: self._logger.error( "Unknown external instance name: {}".format(instance_name)) return False return True
def test_agent_status_set_when_created(volttron_instance): agent = volttron_instance.build_agent() assert agent.vip.health.get_status() is not None assert isinstance(agent.vip.health.get_status(), str) l = json.loads(agent.vip.health.get_status()) assert l['status'] == STATUS_GOOD assert l['context'] is None
def data_received(self, peer, sender, bus, topic, headers, message): def publish_external(agent, topic, headers, message): try: _log.debug('Attempting to publish remotely {}, {}, {}'.format(topic, headers, message)) agent.vip.pubsub.publish(peer='pubsub', topic=topic, headers=headers, message=message).get(timeout=30) except: _log.debug('Data dropped {}, {}, {}'.format(topic, headers, message)) if sender == 'pubsub.compat': message = jsonapi.loads(message[0]) del(headers[headers_mod.CONTENT_TYPE]) assert isinstance(message, list) assert isinstance(message[0], dict) assert isinstance(message[1], dict) print("MESSAGE VALUES ARE: {}".format(message[0])) print("DATA VALUES ARE: {}".format(message[1])) for v in message[1].values(): assert 'tz' in v.keys() assert 'units' in v.keys() assert 'type' in v.keys() #message = [jsonapi.loads(message[0]), jsonapi.loads(message[1])] if has_point_ex: for rex in point_ex: if rex.match(topic): publish_external(self._target_platform, topic, headers, message) else: publish_external(self._target_platform, topic, headers, message)
def _peer_publish(self, frames, user_id): """Publish the incoming message to all the subscribers subscribed to the specified topic. :param frames list of frames :type frames list :param user_id user id of the publishing agent. This is required for protected topics check. :type user_id UTF-8 encoded User-Id property :returns: Count of subscribers. :rtype: int :Return Values: Number of subscribers to whom the message was sent """ if len(frames) > 8: data = frames[8].bytes try: msg = jsonapi.loads(data) headers = msg['headers'] message = msg['message'] peer = frames[0].bytes bus = msg['bus'] pub_msg = jsonapi.dumps( dict(sender=peer, bus=bus, headers=headers, message=message) ) frames[8] = zmq.Frame(str(pub_msg)) except KeyError as exc: self._logger.error("Missing key in _peer_publish message {}".format(exc)) return 0 except ValueError: self._logger.error("JSON decode error. Invalid character") return 0 return self._distribute(frames, user_id)
def _allow(self, environ, start_response, data=None): _log.info('Allowing new vc instance to connect to server.') jsondata = jsonapi.loads(data) json_validate_request(jsondata) assert jsondata.get('method') == 'allowvc' assert jsondata.get('params') params = jsondata.get('params') if isinstance(params, list): vcpublickey = params[0] else: vcpublickey = params.get('vcpublickey') assert vcpublickey assert len(vcpublickey) == 43 authfile = AuthFile() authentry = AuthEntry(credentials=vcpublickey) try: authfile.add(authentry) except AuthFileEntryAlreadyExists: pass start_response('200 OK', [('Content-Type', 'application/json')]) return jsonapi.dumps(json_result(jsondata['id'], "Added"))
def _allow(self, environ, start_response, data=None): _log.info('Allowing new vc instance to connect to server.') jsondata = jsonapi.loads(data) json_validate_request(jsondata) assert jsondata.get('method') == 'allowvc' assert jsondata.get('params') params = jsondata.get('params') if isinstance(params, list): vcpublickey = params[0] else: vcpublickey = params.get('vcpublickey') assert vcpublickey assert len(vcpublickey) == 43 authfile = AuthFile() authentry = AuthEntry(credentials=vcpublickey) try: authfile.add(authentry) except AuthFileEntryAlreadyExists: pass start_response('200 OK', [('Content-Type', 'application/json')]) return jsonapi.dumps( json_result(jsondata['id'], "Added") )
def in_loop(self, sender, **kwargs): # pylint: disable=unused-argument peer = self.peer with closing(self.in_sock) as sock: sock.bind(self.publish_address) while True: message = sock.recv_multipart() #log.debug('incoming message: {!r}'.format(message)) topic = message[0] if (topic.startswith('subscriptions/list') and topic[18:19] in ['/', '']): if len(message) > 2: del message[2:] elif len(message) == 1: message.append('') prefix = topic[19:].decode('utf-8') topics = self.pubsub.list( peer, prefix, subscribed=False).get() message.extend(topic.encode('utf-8') for _, topic, _ in topics) self.out_sock.send_multipart(message) else: message = [part.decode('utf-8') for part in message] try: topic, headers = message[:2] except (ValueError, TypeError): continue headers = jsonapi.loads(headers) message = message[2:] self.pubsub.publish(peer, topic, headers, message).get()
def test_agent_status_set_when_created(volttron_instance): agent = volttron_instance.build_agent() assert agent.vip.health.get_status() is not None assert isinstance(agent.vip.health.get_status(), str) l = json.loads(agent.vip.health.get_status()) assert l['status'] == STATUS_GOOD assert l['context'] is None
def test_test_web_agent(web_instance): vi = web_instance assert vi.is_running() agent_list = vi.list_agents() assert len(agent_list) == 1 base_address = vi.bind_web_address index = base_address + "/web/index.html" text = base_address + "/web/text" rpc = base_address + "/web/jsonrpc" resp = requests.get(index) assert "<h1>The body is good</h1>" in resp.text assert "<html>" in resp.text assert "</html>" in resp.text assert resp.headers['Content-type'] == 'text/html' resp = requests.get(text) assert resp.ok print("*" * 50) print(resp.headers) assert "This is some text" == resp.text assert resp.headers['Content-type'] == 'text/plain' # now test for json rpc payload = {"data": "value", "one": 5, "three": {"two": 1.0}} resp = requests.post(rpc, json=payload) assert resp.ok assert resp.headers['Content-type'] == 'application/json' jsonresp = json.loads(resp.json()['result']) print(jsonresp) for k, v in payload.items(): assert v == jsonresp[k]
def in_loop(self, sender, **kwargs): # pylint: disable=unused-argument peer = self.peer with closing(self.in_sock) as sock: sock.bind(self.publish_address) while True: message = sock.recv_multipart() #log.debug('incoming message: {!r}'.format(message)) topic = message[0] if (topic.startswith('subscriptions/list') and topic[18:19] in ['/', '']): if len(message) > 2: del message[2:] elif len(message) == 1: message.append('') prefix = topic[19:].decode('utf-8') topics = self.pubsub.list(peer, prefix, subscribed=False).get() message.extend( topic.encode('utf-8') for _, topic, _ in topics) self.out_sock.send_multipart(message) else: message = [part.decode('utf-8') for part in message] try: topic, headers = message[:2] except (ValueError, TypeError): continue headers = jsonapi.loads(headers) message = message[2:] self.pubsub.publish(peer, topic, headers, message).get()
def _peer_unsubscribe(self, frames): """ It removes the subscription for the agent (peer) for the specified bus and prefix. :param frames list of frames :type frames list :returns: success or failure :rtype: boolean :Return Values: Return success or not """ if len(frames) < 8: return False else: data = frames[7].bytes msg = jsonapi.loads(data) peer = frames[0].bytes unsubmsg = dict() #Added for backward compatibility try: sub = msg['internal'] unsubmsg = msg except KeyError: try: sub = msg['all'] unsubmsg = msg except KeyError: unsubmsg['internal'] = msg for platform in unsubmsg: prefix = unsubmsg[platform]['prefix'] bus = unsubmsg[platform]['bus'] subscriptions = self._peer_subscriptions[platform][bus] if prefix is None: remove = [] for topic, subscribers in subscriptions.items(): subscribers.discard(peer) if not subscribers: remove.append(topic) for topic in remove: del subscriptions[topic] else: for prefix in prefix if isinstance(prefix, list) else [prefix]: subscribers = subscriptions[prefix] subscribers.discard(peer) if not subscribers: del subscriptions[prefix] if platform == 'all' and self._ext_router is not None: # Send updated subscription list to all connected platforms external_platforms = self._ext_router.get_connected_platforms( ) self._send_external_subscriptions(external_platforms) return True
def _external_to_local_publish(self, frames): """ Publish external pubsub message to local subscribers :param frames: frames containing publish message :return: count of local subscribers or error message if no local subscribers found """ results = [] subscribers_count = 0 # Check if destination is local VIP -- Todo if len(frames) > 8: publisher, receiver, proto, user_id, msg_id, subsystem, op, topic, data = frames[ 0:9] data = frames[8].bytes msg = jsonapi.loads(data) # Check if peer is authorized to publish the topic errmsg = self._check_if_protected_topic(bytes(user_id), bytes(topic)) # peer is not authorized to publish to the topic, send error message to the peer if errmsg is not None: try: frames = [ publisher, b'', proto, user_id, msg_id, subsystem, b'error', zmq.Frame(bytes(UNAUTHORIZED)), zmq.Frame(str(errmsg)) ] self._ext_router.send_external(publisher, frames) return except ValueError: self._logger.debug("Value error") # Make it an internal publish frames[6] = 'publish' subscribers_count = 1 if self._rabbitmq_agent: self._publish_on_rmq_bus(frames) else: subscribers_count = self._distribute_internal(frames) # There are no subscribers, send error message back to source platform if not subscribers_count: try: errmsg = 'NO SUBSCRIBERS' frames = [ publisher, b'', proto, user_id, msg_id, subsystem, zmq.Frame(b'error'), zmq.Frame(bytes(INVALID_REQUEST)), topic ] self._ext_router.send_external(publisher, frames) except ValueError: self._logger.debug("Value error") else: self._logger.debug("Incorrect frames {}".format(len(frames))) return subscribers_count
def _peer_subscribe(self, frames): """It stores the subscription information sent by the agent. It unpacks the frames to get identity of the subscriber, prefix and bus and saves it for future use. :param frames list of frames :type frames list """ # for f in frames: # self._logger.debug("sub frames: {}".format(bytes(f))) if len(frames) < 8: return False else: # self._logger.debug("Subscribe before: {}".format(self._peer_subscriptions)) if isinstance(frames[7], str): data = bytes(frames[7]) else: data = frames[7].bytes msg = jsonapi.loads(data) if isinstance(frames[0], str): peer = bytes(frames[0]) else: peer = frames[0].bytes try: prefix = msg['prefix'] bus = msg['bus'] except KeyError as exc: self._logger.error( "Missing key in _peer_subscribe message {}".format(exc)) return False is_all = msg.get('all_platforms', False) if is_all: platform = 'all' else: platform = 'internal' if self._rabbitmq_agent: # Subscribe to RMQ bus self._rabbitmq_agent.vip.pubsub.subscribe( 'pubsub', prefix, self.publish_callback, all_platforms=is_all) for prefix in prefix if isinstance(prefix, list) else [prefix]: self._add_peer_subscription(peer, bus, prefix, platform) # self._logger.debug("Subscribe after: {}".format(self._peer_subscriptions)) if is_all and self._ext_router is not None: # Send subscription message to all connected platforms external_platforms = self._ext_router.get_connected_platforms() self._send_external_subscriptions(external_platforms) return True
def _distribute_internal(self, frames): """ Distribute the publish message to local subscribers :param frames: list of frames :return: Number of local subscribers """ publisher = bytes(frames[0]) topic = bytes(frames[7]) data = bytes(frames[8]) try: msg = jsonapi.loads(data) bus = msg['bus'] except KeyError as exc: self._logger.error( "Missing key in _peer_publish message {}".format(exc)) return 0 except ValueError: self._logger.error("JSON decode error. Invalid character") return 0 all_subscriptions = dict() subscriptions = dict() subs = dict() # Get subscriptions for all platforms try: all_subscriptions = self._peer_subscriptions['all'][bus] except KeyError: pass try: subscriptions = self._peer_subscriptions['internal'][bus] except KeyError: pass subs.update(all_subscriptions) subs.update(subscriptions) subscribers = set() # Check for local subscribers for prefix, subscription in subs.iteritems(): if subscription and topic.startswith(prefix): subscribers |= subscription if subscribers: # self._logger.debug("PUBSUBSERVICE: found subscribers: {}".format(subscribers)) for subscriber in subscribers: frames[0] = zmq.Frame(subscriber) try: # Send the message to the subscriber for sub in self._send(frames, publisher): # Drop the subscriber if unreachable self.peer_drop(sub) except ZMQError: raise return len(subscribers)
def _read_protected_topics_file(self): #Read protected topics file and send to router try: create_file_if_missing(self._protected_topics_file) with open(self._protected_topics_file) as fil: # Use gevent FileObject to avoid blocking the thread data = FileObject(fil, close=False).read() self._protected_topics = jsonapi.loads(data) if data else {} self._send_protected_update_to_pubsub(self._protected_topics) except Exception: _log.exception('error loading %s', self._protected_topics_file)
def _handle_result(self, message): if message.args and not message.args[0]: try: result = self._results.pop(bytes(message.id)) except KeyError: return try: value = jsonapi.loads(bytes(message.args[1])) except IndexError: value = None result.set(value)
def _read_protected_topics_file(self): #Read protected topics file and send to router try: create_file_if_missing(self._protected_topics_file) with open(self._protected_topics_file) as fil: # Use gevent FileObject to avoid blocking the thread data = FileObject(fil, close=False).read() self._protected_topics = jsonapi.loads(data) if data else {} self._send_protected_update_to_pubsub(self._protected_topics) except Exception: _log.exception('error loading %s', self._protected_topics_file)
def build_remote_connection_param(self, rmq_user, rmq_address, ssl_auth=None, retry_attempt=30, retry_delay=2): """ Build Pika Connection parameters :param rmq_user: RabbitMQ user :param ssl_auth: If SSL based connection or not :return: """ from urlparse import urlparse parsed_addr = urlparse(rmq_address) ssl_auth = ssl_auth if ssl_auth is not None else self.is_ssl _, virtual_host = parsed_addr.path.split('/') try: if ssl_auth: certfile = self.certs.cert_file(rmq_user, True) metafile = certfile[:-4] + ".json" metadata = jsonapi.loads(open(metafile).read()) local_keyfile = metadata['local_keyname'] ca_file = self.certs.cert_file(metadata['remote_ca_name'], True) ssl_options = dict( ssl_version=ssl.PROTOCOL_TLSv1, ca_certs=ca_file, keyfile=self.certs.private_key_file(local_keyfile), certfile=self.certs.cert_file(rmq_user, True), cert_reqs=ssl.CERT_REQUIRED) conn_params = pika.ConnectionParameters( host=parsed_addr.hostname, port=parsed_addr.port, virtual_host=virtual_host, ssl=True, connection_attempts=retry_attempt, retry_delay=retry_delay, ssl_options=ssl_options, credentials=pika.credentials.ExternalCredentials()) else: conn_params = pika.ConnectionParameters( host=parsed_addr.hostname, port=parsed_addr.port, virtual_host=virtual_host, credentials=pika.credentials.PlainCredentials( rmq_user, rmq_user)) except KeyError: return None return conn_params
def ext_route(self, socket): """ Handler function for message received through external socket connection :param socket: socket :return: """ # Expecting incoming frames to follow this VIP format: # [SENDER, PROTO, USER_ID, MSG_ID, SUBSYS, ...] frames = socket.recv_multipart(copy=False) # for f in frames: # _log.debug("PUBSUBSERVICE Frames: {}".format(bytes(f))) if len(frames) < 6: return sender, proto, user_id, msg_id, subsystem = frames[:5] if proto.bytes != b'VIP1': return # Handle 'EXT_RPC' subsystem messages name = subsystem.bytes if name == 'external_rpc': # Reframe the frames sender, proto, usr_id, msg_id, subsystem, msg = frames[:6] msg_data = jsonapi.loads(msg.bytes) peer = msg_data['to_peer'] # Send to destionation agent/peer # Form new frame for local frames[:9] = [ peer, sender, proto, usr_id, msg_id, 'external_rpc', msg ] try: self.socket.send_multipart(frames, flags=NOBLOCK, copy=False) except ZMQError as ex: _log.debug("ZMQ error: {}".format(ex)) pass # Handle 'pubsub' subsystem messages elif name == 'pubsub': if bytes(frames[1]) == b'VIP1': recipient = b'' frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')] # for f in frames: # _log.debug("frames: {}".format(bytes(f))) result = self._pubsub.handle_subsystem(frames, user_id) return result # Handle 'routing_table' subsystem messages elif name == 'routing_table': # for f in frames: # _log.debug("frames: {}".format(bytes(f))) if bytes(frames[1]) == b'VIP1': frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')] result = self._ext_routing.handle_subsystem(frames) return result
def _handle_external_rpc_subsystem(self, message): ret_msg = dict() _log.debug("EXT_RPC subsystem handler IN message {0}".format(message)) op = message.args[0].bytes rpc_msg = jsonapi.loads(message.args[1].bytes) try: # _log.debug("EXT_RPC subsystem handler IN message {0}, {1}".format(message.peer, rpc_msg)) method_args = rpc_msg['args'] #message.args = [method_args] message.args = method_args dispatch = self._dispatcher.dispatch # _log.debug("External RPC IN message args {}".format(message)) responses = [ response for response in (dispatch(bytes(msg), message) for msg in message.args) if response ] # _log.debug("External RPC Resonses {}".format(responses)) if responses: message.user = '' try: message.peer = '' message.subsystem = 'external_rpc' frames = [] op = b'send_platform' frames.append(op) msg = jsonapi.dumps( dict(to_platform=rpc_msg['from_platform'], to_peer=rpc_msg['from_peer'], from_platform=rpc_msg['to_platform'], from_peer=rpc_msg['to_peer'], args=responses)) frames.append(msg) except KeyError: _log.error( "External RPC message did not contain proper message format" ) message.args = jsonapi.dumps(ret_msg) ret_msg = jsonapi.dumps(ret_msg) #_log.debug("EXT_RPC subsystem handler OUT message {}".format(message)) try: self.core().connection.send_vip(peer=b'', subsystem=b'external_rpc', args=frames, msg_id=message.id, user=message.user) except ZMQError as ex: _log.error("ZMQ error: {}".format(ex)) pass except KeyError: pass
def _update_protected_topics(self, frames): """ Update the protected topics and capabilities as per message received from AuthService. :peer frames list of frames :type frames list """ if len(frames) > 7: data = frames[7].bytes try: msg = jsonapi.loads(data) self._load_protected_topics(msg) except ValueError: pass
def rmq_callback(ch, method, properties, body): # Strip prefix from routing key topic = self._get_original_topic(str(method.routing_key)) try: msg = jsonapi.loads(body) headers = msg['headers'] message = msg['message'] bus = msg['bus'] sender = msg['sender'] self.core().spawn(callback, 'pubsub', sender, bus, topic, headers, message) except KeyError as esc: self._logger.error( "Missing keys in pubsub message {}".format(esc))
def ext_route(self, socket): """ Handler function for message received through external socket connection :param socket: socket :return: """ # Expecting incoming frames to follow this VIP format: # [SENDER, PROTO, USER_ID, MSG_ID, SUBSYS, ...] frames = socket.recv_multipart(copy=False) # for f in frames: # _log.debug("PUBSUBSERVICE Frames: {}".format(bytes(f))) if len(frames) < 6: return sender, proto, user_id, msg_id, subsystem = frames[:5] if proto.bytes != b'VIP1': return # Handle 'EXT_RPC' subsystem messages name = subsystem.bytes if name == 'external_rpc': # Reframe the frames sender, proto, usr_id, msg_id, subsystem, msg = frames[:6] msg_data = jsonapi.loads(msg.bytes) peer = msg_data['to_peer'] # Send to destionation agent/peer # Form new frame for local frames[:9] = [peer, sender, proto, usr_id, msg_id, 'external_rpc', msg] try: self.socket.send_multipart(frames, flags=NOBLOCK, copy=False) except ZMQError as ex: _log.debug("ZMQ error: {}".format(ex)) pass # Handle 'pubsub' subsystem messages elif name == 'pubsub': if bytes(frames[1]) == b'VIP1': recipient = b'' frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')] # for f in frames: # _log.debug("frames: {}".format(bytes(f))) result = self._pubsub.handle_subsystem(frames, user_id) return result # Handle 'routing_table' subsystem messages elif name == 'routing_table': # for f in frames: # _log.debug("frames: {}".format(bytes(f))) if bytes(frames[1]) == b'VIP1': frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')] result = self._ext_routing.handle_subsystem(frames) return result
def _read_auth_file(self): auth_path = os.path.join(self.volttron_home, 'auth.json') try: with open(auth_path, 'r') as fd: data = strip_comments(FileObject(fd, close=False).read()) if data: auth = jsonapi.loads(data) else: auth = {} except IOError: auth = {} if 'allow' not in auth: auth['allow'] = [] return auth, auth_path
def _update_caps_users(self, frames): """ Stores the user capabilities sent by the Auth Service :param frames list of frames :type frames list """ if len(frames) > 7: data = frames[7].bytes try: msg = jsonapi.loads(data) self._user_capabilities = msg['capabilities'] except KeyError as exc: self._logger.error("Missing key in update auth capabilities message {}".format(exc)) except ValueError: pass
def _peer_list(self, frames): """Returns a list of subscriptions for a specific bus. If bus is None, then it returns list of subscriptions for all the buses. :param frames list of frames :type frames list :returns: list of tuples of bus, topic and flag to indicate if peer is a subscriber or not :rtype: list :Return Values: List of tuples [(bus, topic, flag to indicate if peer is a subscriber or not)]. """ results = [] if len(frames) > 7: data = frames[7].bytes msg = jsonapi.loads(data) peer = frames[0].bytes try: prefix = msg['prefix'] bus = msg['bus'] subscribed = msg['subscribed'] reverse = msg['reverse'] except KeyError as exc: self._logger.error( "Missing key in _peer_list message {}".format(exc)) return results is_all = msg.get('all_platforms', False) if not is_all: platform = 'internal' else: platform = 'all' if bus is None: buses = self._peer_subscriptions[platform].iteritems() else: buses = [(bus, self._peer_subscriptions[platform][bus])] if reverse: test = prefix.startswith else: test = lambda t: t.startswith(prefix) for bus, subscriptions in buses: for topic, subscribers in subscriptions.iteritems(): if test(topic): member = peer in subscribers if not subscribed or member: results.append((bus, topic, member)) results = jsonapi.dumps(results) return results
def query(self, topic_ids, id_name_map, start=None, end=None, skip=0, agg_type=None, agg_period=None, count=None, order='FIRST_TO_LAST'): if agg_type and agg_period: table_name = agg_type + '_' + agg_period else: table_name = self.data_table topic_id = Literal(0) query = [ SQL('''SELECT to_char(ts, 'YYYY-MM-DD"T"HH24:MI:SS.USOF:00'), ''' 'value_string\n' 'FROM {}\n' 'WHERE topic_id = {}').format(Identifier(table_name), topic_id) ] if start and start.tzinfo != pytz.UTC: start = start.astimezone(pytz.UTC) if end and end.tzinfo != pytz.UTC: end = end.astimezone(pytz.UTC) if start and start == end: query.append(SQL(' AND ts = {}').format(Literal(start))) else: if start: query.append(SQL(' AND ts >= {}').format(Literal(start))) if end: query.append(SQL(' AND ts < {}').format(Literal(end))) query.append( SQL('ORDER BY ts {}'.format('DESC' if order == 'LAST_TO_FIRST' else 'ASC'))) if skip or count: query.append( SQL('LIMIT {} OFFSET {}').format( Literal(None if not count or count < 0 else count), Literal(None if not skip or skip < 0 else skip))) query = SQL('\n').join(query) values = {} for topic_id._wrapped in topic_ids: name = id_name_map[topic_id.wrapped] with self.select(query, fetch_all=False) as cursor: values[name] = [(ts, jsonapi.loads(value)) for ts, value in cursor] return values
def _read_protected_topics_file(self): # Read protected topics file and send to router try: create_file_if_missing(self._protected_topics_file) with open(self._protected_topics_file) as fil: # Use gevent FileObject to avoid blocking the thread data = FileObject(fil, close=False).read() self._protected_topics = jsonapi.loads(data) if data else {} if self.core.messagebus == 'rmq': self._load_protected_topics_for_rmq() # Deferring the RMQ topic permissions to after "onstart" event else: self._send_protected_update_to_pubsub( self._protected_topics) except Exception: _log.exception('error loading %s', self._protected_topics_file)
def parse(jsonstr): data = jsonapi.loads(jsonstr) id = data.get('id', None) version = data.get('jsonrpc', None) method = data.get('method', None) params = data.get('params', None) authorization = data.get('authorization', None) if id == None: raise ParseError("Invalid id") if version != '2.0': print("VERSION IS: {}".format(version)) raise ParseError('Invalid jsonrpc version') if method == None: raise ParseError('Method not specified.') return JsonRpcData(id, version, method, params, authorization)
def parse(jsonstr): data = jsonapi.loads(jsonstr) id = data.get('id', None) version = data.get('jsonrpc', None) method = data.get('method', None) params = data.get('params', None) authorization = data.get('authorization', None) if id == None: raise ParseError("Invalid id") if version != '2.0': print("VERSION IS: {}".format(version)) raise ParseError('Invalid jsonrpc version') if method == None: raise ParseError('Method not specified.') return JsonRpcData(id, version, method, params, authorization)
def _peer_sync(self, frames): """ Synchronizes the subscriptions with the calling agent. :param frames list of frames :type frames list """ if len(frames) > 8: conn = frames[7].bytes if conn == b'connected': data = frames[8].bytes msg = jsonapi.loads(data) peer = frames[0].bytes try: items = msg['subscriptions'] assert isinstance(items, dict) self._sync(peer, items) except KeyError as exc: self._logger.error("Missing key in _peer_sync message {}".format(exc))
def rpc_message_handler(self, ch, method, props, body): """ :param ch: :param method: :param props: :param body: :return: """ zmq_frames = [] frames = jsonapi.loads(body) for frame in frames: zmq_frames.append(bytes(frame)) try: self.zmq_router.socket.send_multipart(zmq_frames, copy=False) except ZMQError as ex: _log.error("ZMQ Error {}".format(ex))
def test_instance_writes_to_instances_file(volttron_instance): vi = volttron_instance assert vi is not None assert vi.is_running() instances_file = os.path.expanduser("~/.volttron_instances") with open(instances_file, 'r') as fp: result = jsonapi.loads(fp.read()) assert result.get(vi.volttron_home) the_instance_entry = result.get(vi.volttron_home) for key in ('pid', 'vip-address', 'volttron-home', 'start-args'): assert the_instance_entry.get(key) assert the_instance_entry['pid'] == vi.p_process.pid assert the_instance_entry['vip-address'][0] == vi.vip_address assert the_instance_entry['volttron-home'] == vi.volttron_home
def _process_incoming_message(self, message): """Process incoming messages param message: VIP message from PubSubService type message: dict """ op = message.args[0].bytes if op == 'request_response': result = None try: result = self._results.pop(bytes(message.id)) except KeyError: pass if self._parameters_needed: self._send_via_rpc = False self._parameters_needed = False self._pubsubwithrpc.clear_parameters() del self._pubsubwithrpc response = message.args[1].bytes #_log.debug("Message result: {}".format(response)) if result: result.set(response) elif op == 'publish': try: topic = topic = message.args[1].bytes data = message.args[2].bytes except IndexError: return try: msg = jsonapi.loads(data) headers = msg['headers'] message = msg['message'] sender = msg['sender'] bus = msg['bus'] except KeyError as exc: _log.error("Missing keys in pubsub message: {}".format(exc)) else: self._process_callback(sender, bus, topic, headers, message) else: _log.error("Unknown operation ({})".format(op))
def _handle_external_rpc_subsystem(self, message): ret_msg = dict() #_log.debug("EXT_RPC subsystem handler IN message {0}".format(message)) op = message.args[0].bytes rpc_msg = jsonapi.loads(message.args[1].bytes) try: #_log.debug("EXT_RPC subsystem handler IN message {0}, {1}".format(message.peer, rpc_msg)) method_args = rpc_msg['args'] #message.args = [method_args] message.args = method_args dispatch = self._dispatcher.dispatch #_log.debug("External RPC IN message args {}".format(message)) responses = [response for response in ( dispatch(bytes(msg), message) for msg in message.args) if response] #_log.debug("External RPC Resonses {}".format(responses)) if responses: message.user = '' try: message.peer = '' message.subsystem = 'external_rpc' frames = [] op = b'send_platform' frames.append(op) msg = jsonapi.dumps(dict(to_platform=rpc_msg['from_platform'], to_peer=rpc_msg['from_peer'], from_platform=rpc_msg['to_platform'], from_peer=rpc_msg['to_peer'], args=responses)) frames.append(msg) except KeyError: _log.error("External RPC message did not contain proper message format") message.args = jsonapi.dumps(ret_msg) ret_msg = jsonapi.dumps(ret_msg) #_log.debug("EXT_RPC subsystem handler OUT message {}".format(message)) try: self.core().socket.send_vip(b'', 'external_rpc', frames, user=message.user, msg_id=message.id, copy=False) except ZMQError as ex: _log.error("ZMQ error: {}".format(ex)) pass except KeyError: pass
def from_json(data, status_changed_callback=None): """ Deserializes a `Status` object and returns it to the caller. :param data: :param status_changed_callback: :return: """ _log.debug("from_json {}".format(data)) statusobj = Status() cp = jsonapi.loads(data) cp['_status'] = cp['status'] cp['_last_updated'] = cp['last_updated'] cp['_context'] = cp['context'] del cp['status'] del cp['last_updated'] del cp['context'] statusobj.__dict__ = cp statusobj._status_changed_callback = status_changed_callback return statusobj
def _read(self): auth_data = {} try: create_file_if_missing(self.auth_file) with open(self.auth_file) as fil: # Use gevent FileObject to avoid blocking the thread before_strip_comments = FileObject(fil, close=False).read() data = strip_comments(before_strip_comments) if data != before_strip_comments: _log.warn('Comments in %s are deprecated and will not be ' 'preserved', self.auth_file) if data: auth_data = jsonapi.loads(data) except Exception: _log.exception('error loading %s', self.auth_file) allow_list = auth_data.get('allow', []) groups = auth_data.get('groups', {}) roles = auth_data.get('roles', {}) version = auth_data.get('version', {'major': 0, 'minor': 0}) return allow_list, groups, roles, version
def query(self, topic_ids, id_name_map, start=None, end=None, skip=0, agg_type=None, agg_period=None, count=None, order='FIRST_TO_LAST'): if agg_type and agg_period: table_name = agg_type + '_' + agg_period else: table_name = self.data_table topic_id = Literal(0) query = [SQL( '''SELECT to_char(ts, 'YYYY-MM-DD"T"HH24:MI:SS.USOF:00'), ''' 'value_string\n' 'FROM {}\n' 'WHERE topic_id = {}' ).format(Identifier(table_name), topic_id)] if start and start.tzinfo != pytz.UTC: start = start.astimezone(pytz.UTC) if end and end.tzinfo != pytz.UTC: end = end.astimezone(pytz.UTC) if start and start == end: query.append(SQL(' AND ts = {}').format(Literal(start))) else: if start: query.append(SQL(' AND ts >= {}').format(Literal(start))) if end: query.append(SQL(' AND ts < {}').format(Literal(end))) query.append(SQL('ORDER BY ts {}'.format( 'DESC' if order == 'LAST_TO_FIRST' else 'ASC'))) if skip or count: query.append(SQL('LIMIT {} OFFSET {}').format( Literal(None if not count or count < 0 else count), Literal(None if not skip or skip < 0 else skip))) query = SQL('\n').join(query) values = {} for topic_id._wrapped in topic_ids: name = id_name_map[topic_id.wrapped] with self.select(query, fetch_all=False) as cursor: values[name] = [(ts, jsonapi.loads(value)) for ts, value in cursor] return values
def is_instance_running(volttron_home=None): from volttron.platform.agent import json as jsonapi if volttron_home is None: volttron_home = get_home() instance_file = os.path.expanduser("~/.volttron_instances") if not os.path.isfile(instance_file): return False with open(instance_file, 'r') as fp: jsonobj = jsonapi.loads(fp.read()) if volttron_home not in jsonobj: return False obj = jsonobj[volttron_home] pid = obj.get('pid', None) if not pid: return False return psutil.pid_exists(pid)
def parse_json_config(config_str): """Parse a JSON-encoded configuration file.""" return jsonapi.loads(strip_comments(config_str))
from crate_historian import crate_utils from volttron.platform.agent import utils from volttron.platform.dbutils import mongoutils logging.basicConfig(level=logging.DEBUG) _log = logging.getLogger(__name__) for key in logging.Logger.manager.loggerDict: _log.debug(key) logging.getLogger('crate.client.http').setLevel(logging.INFO) logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO) root = os.path.dirname(os.path.abspath(__file__)) with open('{}/crate_config'.format(root), 'r') as fp: crate_params = jsonapi.loads(fp.read()) root = os.path.dirname(os.path.abspath(__file__)) with open('{}/mongo_config'.format(root), 'r') as fp: mongo_params = jsonapi.loads(fp.read()) MAX_QUEUE_SIZE = 50000 QUEUE_BATCH_SIZE = 5000 class TableQueue(Queue.Queue, object): def __init__(self, table_name): super(TableQueue, self).__init__() self.table_name = table_name
def store_agent_config(self, session_user, params): required = ('agent_identity', 'config_name', 'raw_contents') message_id = params.pop('message_id') errors = [] for r in required: if r not in params: errors.append('Missing {}'.format(r)) config_type = params.get('config_type', None) if config_type: if config_type not in ('raw', 'json', 'csv'): errors.append('Invalid config_type parameter') if errors: return jsonrpc.json_error(message_id, INVALID_PARAMS, "\n".join(errors)) try: self._log.debug("Calling store_agent_config on external platform.") self.call("store_agent_config", **params) except Exception as e: self._log.error(repr(e)) return jsonrpc.json_error(message_id, INTERNAL_ERROR, str(e)) config_name = params.get("config_name") agent_identity = params.get("agent_identity") if config_name.startswith("devices"): # Since we start with devices, we assume that we are attempting # to save a master driver config file. rawdict = jsonapi.loads(params['raw_contents']) # if this is not a bacnet device_type then we cannot do anything # more than save and retrieve it from the store. driver_type = rawdict.get('driver_type', None) if driver_type is None or driver_type not in ('bacnet', 'modbus'): return jsonrpc.json_result(message_id, "SUCCESS") # Registry config starts with config:// registry_config = rawdict['registry_config'][len('config://'):] try: self._log.debug("Retrieving registry_config for new device.") point_config = self.call("get_agent_config", agent_identity, registry_config, raw=False) except Exception as e: self._log.error(str(e)) return jsonrpc.json_error(message_id, INTERNAL_ERROR, "Couldn't retrieve registry_config " "from connection.") else: new_device = dict( device_address=rawdict['driver_config']['device_address'], device_id=rawdict['driver_config']['device_id'], points=[], path=config_name, health=Status.build(UNKNOWN_STATUS, context="Unpublished").as_dict() ) points = [p['Volttron Point Name'] for p in point_config] new_device['points'] = points self._vc.send_management_message("NEW_DEVICE", new_device) status = Status.build(UNKNOWN_STATUS, context="Not published since update") device_config_name = params.get('config_name') device_no_prefix = device_config_name[len('devices/'):] the_device = self._current_devices.get(device_no_prefix, {}) if not the_device: self._current_devices[device_no_prefix] = dict( last_publish_utc=None, health=status.as_dict(), points=points ) else: self._current_devices[device_no_prefix]['points'] = points return jsonrpc.json_result(message_id, "SUCCESS")
def deserialize(self, json_string): return jsonapi.loads(json_string)
def install_agent(self, agent_wheel=None, agent_dir=None, config_file=None, start=True, vip_identity=None): """ Install and optionally start an agent on the instance. This function allows installation from an agent wheel or an agent directory (NOT BOTH). If an agent_wheel is specified then it is assumed to be ready for installation (has a config file). If an agent_dir is specified then a config_file file must be specified or if it is not specified then it is assumed that the file agent_dir/config is to be used as the configuration file. If none of these exist then an assertion error will be thrown. This function will return with a uuid of the installed agent. :param agent_wheel: :param agent_dir: :param config_file: :param start: :param vip_identity: :return: """ assert self.is_running(), "Instance must be running to install agent." assert agent_wheel or agent_dir, "Invalid agent_wheel or agent_dir." if agent_wheel: assert not agent_dir assert not config_file assert os.path.exists(agent_wheel) wheel_file = agent_wheel agent_uuid = self._install_agent(wheel_file, start, vip_identity) # Now if the agent_dir is specified. if agent_dir: assert not agent_wheel if isinstance(config_file, dict): from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps(config_file)) config_file = temp_config elif not config_file: if os.path.exists(os.path.join(agent_dir, "config")): config_file = os.path.join(agent_dir, "config") else: from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps({})) config_file = temp_config elif os.path.exists(config_file): pass # config_file already set! else: raise ValueError("Can't determine correct config file.") script = os.path.join(self.volttron_root, "scripts/install-agent.py") cmd = [self.python, script, "--volttron-home", self.volttron_home, "--volttron-root", self.volttron_root, "--agent-source", agent_dir, "--config", config_file, "--json"] if vip_identity: cmd.extend(["--vip-identity", vip_identity]) if start: cmd.extend(["--start"]) try: response = subprocess.check_output(cmd) except Exception as e: _log.error(repr(e)) raise e self.logit(response) # Because we are no longer silencing output from the install, the # the results object is now much more verbose. Our assumption is # that the result we are looking for is the only JSON block in # the output match = re.search(r'^({.*})', response, flags=re.M | re.S) if match: results = match.group(0) else: raise ValueError( "The results were not found in the command output") self.logit("here are the results: {}".format(results)) # # Response from results is expected as follows depending on # parameters, note this is a json string so parse to get dictionary # { # "started": true, # "agent_pid": 26241, # "starting": true, # "agent_uuid": "ec1fd94e-922a-491f-9878-c392b24dbe50" # } assert results resultobj = jsonapi.loads(str(results)) if start: assert resultobj['started'] agent_uuid = resultobj['agent_uuid'] assert agent_uuid is not None if start: assert self.is_agent_running(agent_uuid) return agent_uuid
def configure_main(self, config_name, action, contents): config = self.default_config.copy() config.update(contents) if action == "NEW": try: self.max_open_sockets = config["max_open_sockets"] if self.max_open_sockets is not None: max_open_sockets = int(self.max_open_sockets) configure_socket_lock(max_open_sockets) _log.info("maximum concurrently open sockets limited to " + str(max_open_sockets)) elif self.system_socket_limit is not None: max_open_sockets = int(self.system_socket_limit * 0.8) _log.info("maximum concurrently open sockets limited to " + str(max_open_sockets) + " (derived from system limits)") configure_socket_lock(max_open_sockets) else: configure_socket_lock() _log.warn("No limit set on the maximum number of concurrently open sockets. " "Consider setting max_open_sockets if you plan to work with 800+ modbus devices.") self.max_concurrent_publishes = config['max_concurrent_publishes'] max_concurrent_publishes = int(self.max_concurrent_publishes) if max_concurrent_publishes < 1: _log.warn("No limit set on the maximum number of concurrent driver publishes. " "Consider setting max_concurrent_publishes if you plan to work with many devices.") else: _log.info("maximum concurrent driver publishes limited to " + str(max_concurrent_publishes)) configure_publish_lock(max_concurrent_publishes) self.scalability_test = bool(config["scalability_test"]) self.scalability_test_iterations = int(config["scalability_test_iterations"]) if self.scalability_test: self.waiting_to_finish = set() self.test_iterations = 0 self.test_results = [] self.current_test_start = None except ValueError as e: _log.error("ERROR PROCESSING STARTUP CRITICAL CONFIGURATION SETTINGS: {}".format(e)) _log.error("MASTER DRIVER SHUTTING DOWN") sys.exit(1) else: if self.max_open_sockets != config["max_open_sockets"]: _log.info("The master driver must be restarted for changes to the max_open_sockets setting to take effect") if self.max_concurrent_publishes != config["max_concurrent_publishes"]: _log.info("The master driver must be restarted for changes to the max_concurrent_publishes setting to take effect") if self.scalability_test != bool(config["scalability_test"]): if not self.scalability_test: _log.info( "The master driver must be restarted with scalability_test set to true in order to run a test.") if self.scalability_test: _log.info( "A scalability test may not be interrupted. Restarting the driver is required to stop the test.") try: if self.scalability_test_iterations != int(config["scalability_test_iterations"]) and self.scalability_test: _log.info( "A scalability test must be restarted for the scalability_test_iterations setting to take effect.") except ValueError: pass #update override patterns if self._override_patterns is None: try: values = self.vip.config.get("override_patterns") values = jsonapi.loads(values) if isinstance(values, dict): self._override_patterns = set() for pattern, end_time in values.items(): #check the end_time now = utils.get_aware_utc_now() #If end time is indefinite, set override with indefinite duration if end_time == "0.0": self._set_override_on(pattern, 0.0, from_config_store=True) else: end_time = utils.parse_timestamp_string(end_time) # If end time > current time, set override with new duration if end_time > now: delta = end_time - now self._set_override_on(pattern, delta.total_seconds(), from_config_store=True) else: self._override_patterns = set() except KeyError: self._override_patterns = set() except ValueError: _log.error("Override patterns is not set correctly in config store") self._override_patterns = set() try: driver_scrape_interval = float(config["driver_scrape_interval"]) except ValueError as e: _log.error("ERROR PROCESSING CONFIGURATION: {}".format(e)) _log.error("Master driver scrape interval settings unchanged") # TODO: set a health status for the agent try: group_offset_interval = float(config["group_offset_interval"]) except ValueError as e: _log.error("ERROR PROCESSING CONFIGURATION: {}".format(e)) _log.error("Master driver group interval settings unchanged") # TODO: set a health status for the agent if self.scalability_test and action == "UPDATE": _log.info("Running scalability test. Settings may not be changed without restart.") return if (self.driver_scrape_interval != driver_scrape_interval or self.group_offset_interval != group_offset_interval): self.driver_scrape_interval = driver_scrape_interval self.group_offset_interval = group_offset_interval _log.info("Setting time delta between driver device scrapes to " + str(driver_scrape_interval)) #Reset all scrape schedules self.freed_time_slots.clear() self.group_counts.clear() for driver in self.instances.itervalues(): time_slot = self.group_counts[driver.group] driver.update_scrape_schedule(time_slot, self.driver_scrape_interval, driver.group, self.group_offset_interval) self.group_counts[driver.group] += 1 self.publish_depth_first_all = bool(config["publish_depth_first_all"]) self.publish_breadth_first_all = bool(config["publish_breadth_first_all"]) self.publish_depth_first = bool(config["publish_depth_first"]) self.publish_breadth_first = bool(config["publish_breadth_first"]) #Update the publish settings on running devices. for driver in self.instances.itervalues(): driver.update_publish_types(self.publish_depth_first_all, self.publish_breadth_first_all, self.publish_depth_first, self.publish_breadth_first)
from crate import client import os from volttron.platform.agent import json as jsonapi root = os.path.dirname(os.path.abspath(__file__)) with open('{}/crate_config'.format(root), 'r') as fp: data = jsonapi.loads(fp.read()) host = data['connection']['params']['host'] conn = client.connect(host, error_trace=True) cursor = conn.cursor() schema = 'test_import' tables = ['analysis', 'analysis_string', 'datalogger', 'datalogger_string', 'device', 'device_string', 'topic', 'meta', 'record'] for t in tables: try: if schema: full_table_name = "{schema}.{table}".format(schema=schema, table=t) else: full_table_name = t cursor.execute("DROP TABLE {}".format(full_table_name)) except Exception as ex: print(ex.message)