def unpack_legacy_message(headers, message):
'''Unpack legacy pubsub messages for VIP agents.
Loads JSON-formatted message parts and removes single-frame messages
from their containing list. Does not alter headers.
'''
if not isinstance(headers, Headers):
headers = Headers(headers)
try:
content_type = headers['Content-Type']
except KeyError:
return headers, message
if isinstance(content_type, basestring):
if content_type.lower() == 'application/json':
if isinstance(message, list) and len(message) == 1:
return jsonapi.loads(message[0])
if isinstance(message, basestring):
return jsonapi.loads(message)
if isinstance(message, list) and len(message) == 1:
return message[0]
if isinstance(content_type, list) and isinstance(message, list):
parts = [(jsonapi.loads(msg)
if str(ctype).lower() == 'application/json' else msg)
for ctype, msg in zip(content_type, message)]
parts.extend(message[len(parts):])
if len(parts) == len(content_type) == 1:
return parts[0]
return parts
return message
def unpack_legacy_message(headers, message):
'''Unpack legacy pubsub messages for VIP agents.
Loads JSON-formatted message parts and removes single-frame messages
from their containing list. Does not alter headers.
'''
if not isinstance(headers, Headers):
headers = Headers(headers)
try:
content_type = headers['Content-Type']
except KeyError:
return headers, message
if isinstance(content_type, str):
if content_type.lower() == 'application/json':
if isinstance(message, list) and len(message) == 1:
return jsonapi.loads(message[0])
if isinstance(message, str):
return jsonapi.loads(message)
if isinstance(message, list) and len(message) == 1:
return message[0]
if isinstance(content_type, list) and isinstance(message, list):
parts = [(jsonapi.loads(msg)
if str(ctype).lower() == 'application/json' else msg)
for ctype, msg in zip(content_type, message)]
parts.extend(message[len(parts):])
if len(parts) == len(content_type) == 1:
return parts[0]
return parts
return message
def test_agent_last_update_increases(volttron_instance):
agent = volttron_instance.build_agent()
s = json.loads(agent.vip.health.get_status())
dt = dateparse(s['last_updated'], fuzzy=True)
agent.vip.health.set_status(STATUS_UNKNOWN, 'Unknown now!')
gevent.sleep(1)
s = json.loads(agent.vip.health.get_status())
dt2 = dateparse(s['last_updated'], fuzzy=True)
assert dt < dt2
def test_agent_last_update_increases(volttron_instance):
agent = volttron_instance.build_agent()
s = json.loads(agent.vip.health.get_status())
dt = dateparse(s['last_updated'], fuzzy=True)
agent.vip.health.set_status(STATUS_UNKNOWN, 'Unknown now!')
gevent.sleep(1)
s = json.loads(agent.vip.health.get_status())
dt2 = dateparse(s['last_updated'], fuzzy=True)
assert dt < dt2
def _process_incoming_message(self, message):
"""Process incoming messages
param message: VIP message from PubSubService
type message: dict
"""
op = message.args[0].bytes
if op == 'request_response':
result = None
try:
result = self._results.pop(bytes(message.id))
except KeyError:
pass
if self._parameters_needed:
self._send_via_rpc = False
self._parameters_needed = False
self._pubsubwithrpc.clear_parameters()
del self._pubsubwithrpc
response = message.args[1].bytes
# _log.debug("Message result: {}".format(response))
if result:
result.set(response)
elif op == 'publish':
try:
topic = topic = message.args[1].bytes
data = message.args[2].bytes
except IndexError:
return
try:
msg = jsonapi.loads(data)
headers = msg['headers']
message = msg['message']
sender = msg['sender']
bus = msg['bus']
except KeyError as exc:
_log.error("Missing keys in pubsub message: {}".format(exc))
else:
self._process_callback(sender, bus, topic, headers, message)
elif op == 'list_response':
result = None
try:
result = self._results.pop(bytes(message.id))
response = jsonapi.loads(message.args[1].bytes)
if result:
result.set(response)
except KeyError:
pass
else:
_log.error("Unknown operation ({})".format(op))
def test_agent_status_changes(volttron_instance):
unknown_message = "This is unknown"
bad_message = "Bad kitty"
agent = volttron_instance.build_agent()
agent.vip.health.set_status(STATUS_UNKNOWN, unknown_message)
r = json.loads(agent.vip.health.get_status())
assert unknown_message == r['context']
assert STATUS_UNKNOWN == r['status']
agent.vip.health.set_status(STATUS_BAD, bad_message)
r = json.loads(agent.vip.health.get_status())
assert bad_message == r['context']
assert STATUS_BAD == r['status']
def test_agent_status_changes(volttron_instance):
unknown_message = "This is unknown"
bad_message = "Bad kitty"
agent = volttron_instance.build_agent()
agent.vip.health.set_status(STATUS_UNKNOWN, unknown_message)
r = json.loads(agent.vip.health.get_status())
assert unknown_message == r['context']
assert STATUS_UNKNOWN == r['status']
agent.vip.health.set_status(STATUS_BAD, bad_message)
r = json.loads(agent.vip.health.get_status())
assert bad_message == r['context']
assert STATUS_BAD == r['status']
def _update_external_subscriptions(self, frames):
"""
Store external subscriptions
:param frames: frames containing external subscriptions
:return:
"""
results = []
if len(frames) <= 7:
return False
else:
data = frames[7].bytes
msg = jsonapi.loads(data)
try:
for instance_name in msg:
prefixes = msg[instance_name]
# Store external subscription list for later use (during publish)
self._ext_subscriptions[instance_name] = prefixes
self._logger.debug(
"PUBSUBSERVICE New external list from {0}: List: {1}".
format(instance_name, self._ext_subscriptions))
except KeyError as exc:
self._logger.error(
"Unknown external instance name: {}".format(instance_name))
return False
return True
def test_agent_status_set_when_created(volttron_instance):
agent = volttron_instance.build_agent()
assert agent.vip.health.get_status() is not None
assert isinstance(agent.vip.health.get_status(), str)
l = json.loads(agent.vip.health.get_status())
assert l['status'] == STATUS_GOOD
assert l['context'] is None
def data_received(self, peer, sender, bus, topic, headers, message):
def publish_external(agent, topic, headers, message):
try:
_log.debug('Attempting to publish remotely {}, {}, {}'.format(topic, headers, message))
agent.vip.pubsub.publish(peer='pubsub',
topic=topic,
headers=headers,
message=message).get(timeout=30)
except:
_log.debug('Data dropped {}, {}, {}'.format(topic, headers, message))
if sender == 'pubsub.compat':
message = jsonapi.loads(message[0])
del(headers[headers_mod.CONTENT_TYPE])
assert isinstance(message, list)
assert isinstance(message[0], dict)
assert isinstance(message[1], dict)
print("MESSAGE VALUES ARE: {}".format(message[0]))
print("DATA VALUES ARE: {}".format(message[1]))
for v in message[1].values():
assert 'tz' in v.keys()
assert 'units' in v.keys()
assert 'type' in v.keys()
#message = [jsonapi.loads(message[0]), jsonapi.loads(message[1])]
if has_point_ex:
for rex in point_ex:
if rex.match(topic):
publish_external(self._target_platform, topic, headers, message)
else:
publish_external(self._target_platform, topic, headers, message)
def _peer_publish(self, frames, user_id):
"""Publish the incoming message to all the subscribers subscribed to the specified topic.
:param frames list of frames
:type frames list
:param user_id user id of the publishing agent. This is required for protected topics check.
:type user_id UTF-8 encoded User-Id property
:returns: Count of subscribers.
:rtype: int
:Return Values:
Number of subscribers to whom the message was sent
"""
if len(frames) > 8:
data = frames[8].bytes
try:
msg = jsonapi.loads(data)
headers = msg['headers']
message = msg['message']
peer = frames[0].bytes
bus = msg['bus']
pub_msg = jsonapi.dumps(
dict(sender=peer, bus=bus, headers=headers, message=message)
)
frames[8] = zmq.Frame(str(pub_msg))
except KeyError as exc:
self._logger.error("Missing key in _peer_publish message {}".format(exc))
return 0
except ValueError:
self._logger.error("JSON decode error. Invalid character")
return 0
return self._distribute(frames, user_id)
def _allow(self, environ, start_response, data=None):
_log.info('Allowing new vc instance to connect to server.')
jsondata = jsonapi.loads(data)
json_validate_request(jsondata)
assert jsondata.get('method') == 'allowvc'
assert jsondata.get('params')
params = jsondata.get('params')
if isinstance(params, list):
vcpublickey = params[0]
else:
vcpublickey = params.get('vcpublickey')
assert vcpublickey
assert len(vcpublickey) == 43
authfile = AuthFile()
authentry = AuthEntry(credentials=vcpublickey)
try:
authfile.add(authentry)
except AuthFileEntryAlreadyExists:
pass
start_response('200 OK', [('Content-Type', 'application/json')])
return jsonapi.dumps(json_result(jsondata['id'], "Added"))
def _allow(self, environ, start_response, data=None):
_log.info('Allowing new vc instance to connect to server.')
jsondata = jsonapi.loads(data)
json_validate_request(jsondata)
assert jsondata.get('method') == 'allowvc'
assert jsondata.get('params')
params = jsondata.get('params')
if isinstance(params, list):
vcpublickey = params[0]
else:
vcpublickey = params.get('vcpublickey')
assert vcpublickey
assert len(vcpublickey) == 43
authfile = AuthFile()
authentry = AuthEntry(credentials=vcpublickey)
try:
authfile.add(authentry)
except AuthFileEntryAlreadyExists:
pass
start_response('200 OK',
[('Content-Type', 'application/json')])
return jsonapi.dumps(
json_result(jsondata['id'], "Added")
)
def in_loop(self, sender, **kwargs): # pylint: disable=unused-argument
peer = self.peer
with closing(self.in_sock) as sock:
sock.bind(self.publish_address)
while True:
message = sock.recv_multipart()
#log.debug('incoming message: {!r}'.format(message))
topic = message[0]
if (topic.startswith('subscriptions/list') and
topic[18:19] in ['/', '']):
if len(message) > 2:
del message[2:]
elif len(message) == 1:
message.append('')
prefix = topic[19:].decode('utf-8')
topics = self.pubsub.list(
peer, prefix, subscribed=False).get()
message.extend(topic.encode('utf-8')
for _, topic, _ in topics)
self.out_sock.send_multipart(message)
else:
message = [part.decode('utf-8') for part in message]
try:
topic, headers = message[:2]
except (ValueError, TypeError):
continue
headers = jsonapi.loads(headers)
message = message[2:]
self.pubsub.publish(peer, topic, headers, message).get()
def test_agent_status_set_when_created(volttron_instance):
agent = volttron_instance.build_agent()
assert agent.vip.health.get_status() is not None
assert isinstance(agent.vip.health.get_status(), str)
l = json.loads(agent.vip.health.get_status())
assert l['status'] == STATUS_GOOD
assert l['context'] is None
def test_test_web_agent(web_instance):
vi = web_instance
assert vi.is_running()
agent_list = vi.list_agents()
assert len(agent_list) == 1
base_address = vi.bind_web_address
index = base_address + "/web/index.html"
text = base_address + "/web/text"
rpc = base_address + "/web/jsonrpc"
resp = requests.get(index)
assert "<h1>The body is good</h1>" in resp.text
assert "<html>" in resp.text
assert "</html>" in resp.text
assert resp.headers['Content-type'] == 'text/html'
resp = requests.get(text)
assert resp.ok
print("*" * 50)
print(resp.headers)
assert "This is some text" == resp.text
assert resp.headers['Content-type'] == 'text/plain'
# now test for json rpc
payload = {"data": "value", "one": 5, "three": {"two": 1.0}}
resp = requests.post(rpc, json=payload)
assert resp.ok
assert resp.headers['Content-type'] == 'application/json'
jsonresp = json.loads(resp.json()['result'])
print(jsonresp)
for k, v in payload.items():
assert v == jsonresp[k]
def in_loop(self, sender, **kwargs): # pylint: disable=unused-argument
peer = self.peer
with closing(self.in_sock) as sock:
sock.bind(self.publish_address)
while True:
message = sock.recv_multipart()
#log.debug('incoming message: {!r}'.format(message))
topic = message[0]
if (topic.startswith('subscriptions/list')
and topic[18:19] in ['/', '']):
if len(message) > 2:
del message[2:]
elif len(message) == 1:
message.append('')
prefix = topic[19:].decode('utf-8')
topics = self.pubsub.list(peer, prefix,
subscribed=False).get()
message.extend(
topic.encode('utf-8') for _, topic, _ in topics)
self.out_sock.send_multipart(message)
else:
message = [part.decode('utf-8') for part in message]
try:
topic, headers = message[:2]
except (ValueError, TypeError):
continue
headers = jsonapi.loads(headers)
message = message[2:]
self.pubsub.publish(peer, topic, headers, message).get()
def _peer_unsubscribe(self, frames):
"""
It removes the subscription for the agent (peer) for the specified bus and prefix.
:param frames list of frames
:type frames list
:returns: success or failure
:rtype: boolean
:Return Values:
Return success or not
"""
if len(frames) < 8:
return False
else:
data = frames[7].bytes
msg = jsonapi.loads(data)
peer = frames[0].bytes
unsubmsg = dict()
#Added for backward compatibility
try:
sub = msg['internal']
unsubmsg = msg
except KeyError:
try:
sub = msg['all']
unsubmsg = msg
except KeyError:
unsubmsg['internal'] = msg
for platform in unsubmsg:
prefix = unsubmsg[platform]['prefix']
bus = unsubmsg[platform]['bus']
subscriptions = self._peer_subscriptions[platform][bus]
if prefix is None:
remove = []
for topic, subscribers in subscriptions.items():
subscribers.discard(peer)
if not subscribers:
remove.append(topic)
for topic in remove:
del subscriptions[topic]
else:
for prefix in prefix if isinstance(prefix,
list) else [prefix]:
subscribers = subscriptions[prefix]
subscribers.discard(peer)
if not subscribers:
del subscriptions[prefix]
if platform == 'all' and self._ext_router is not None:
# Send updated subscription list to all connected platforms
external_platforms = self._ext_router.get_connected_platforms(
)
self._send_external_subscriptions(external_platforms)
return True
def _external_to_local_publish(self, frames):
"""
Publish external pubsub message to local subscribers
:param frames: frames containing publish message
:return: count of local subscribers or error message if no local subscribers found
"""
results = []
subscribers_count = 0
# Check if destination is local VIP -- Todo
if len(frames) > 8:
publisher, receiver, proto, user_id, msg_id, subsystem, op, topic, data = frames[
0:9]
data = frames[8].bytes
msg = jsonapi.loads(data)
# Check if peer is authorized to publish the topic
errmsg = self._check_if_protected_topic(bytes(user_id),
bytes(topic))
# peer is not authorized to publish to the topic, send error message to the peer
if errmsg is not None:
try:
frames = [
publisher, b'', proto, user_id, msg_id, subsystem,
b'error',
zmq.Frame(bytes(UNAUTHORIZED)),
zmq.Frame(str(errmsg))
]
self._ext_router.send_external(publisher, frames)
return
except ValueError:
self._logger.debug("Value error")
# Make it an internal publish
frames[6] = 'publish'
subscribers_count = 1
if self._rabbitmq_agent:
self._publish_on_rmq_bus(frames)
else:
subscribers_count = self._distribute_internal(frames)
# There are no subscribers, send error message back to source platform
if not subscribers_count:
try:
errmsg = 'NO SUBSCRIBERS'
frames = [
publisher, b'', proto, user_id, msg_id, subsystem,
zmq.Frame(b'error'),
zmq.Frame(bytes(INVALID_REQUEST)), topic
]
self._ext_router.send_external(publisher, frames)
except ValueError:
self._logger.debug("Value error")
else:
self._logger.debug("Incorrect frames {}".format(len(frames)))
return subscribers_count
def _peer_subscribe(self, frames):
"""It stores the subscription information sent by the agent. It unpacks the frames to get identity of the
subscriber, prefix and bus and saves it for future use.
:param frames list of frames
:type frames list
"""
# for f in frames:
# self._logger.debug("sub frames: {}".format(bytes(f)))
if len(frames) < 8:
return False
else:
# self._logger.debug("Subscribe before: {}".format(self._peer_subscriptions))
if isinstance(frames[7], str):
data = bytes(frames[7])
else:
data = frames[7].bytes
msg = jsonapi.loads(data)
if isinstance(frames[0], str):
peer = bytes(frames[0])
else:
peer = frames[0].bytes
try:
prefix = msg['prefix']
bus = msg['bus']
except KeyError as exc:
self._logger.error(
"Missing key in _peer_subscribe message {}".format(exc))
return False
is_all = msg.get('all_platforms', False)
if is_all:
platform = 'all'
else:
platform = 'internal'
if self._rabbitmq_agent:
# Subscribe to RMQ bus
self._rabbitmq_agent.vip.pubsub.subscribe(
'pubsub',
prefix,
self.publish_callback,
all_platforms=is_all)
for prefix in prefix if isinstance(prefix, list) else [prefix]:
self._add_peer_subscription(peer, bus, prefix, platform)
# self._logger.debug("Subscribe after: {}".format(self._peer_subscriptions))
if is_all and self._ext_router is not None:
# Send subscription message to all connected platforms
external_platforms = self._ext_router.get_connected_platforms()
self._send_external_subscriptions(external_platforms)
return True
def _distribute_internal(self, frames):
"""
Distribute the publish message to local subscribers
:param frames: list of frames
:return: Number of local subscribers
"""
publisher = bytes(frames[0])
topic = bytes(frames[7])
data = bytes(frames[8])
try:
msg = jsonapi.loads(data)
bus = msg['bus']
except KeyError as exc:
self._logger.error(
"Missing key in _peer_publish message {}".format(exc))
return 0
except ValueError:
self._logger.error("JSON decode error. Invalid character")
return 0
all_subscriptions = dict()
subscriptions = dict()
subs = dict()
# Get subscriptions for all platforms
try:
all_subscriptions = self._peer_subscriptions['all'][bus]
except KeyError:
pass
try:
subscriptions = self._peer_subscriptions['internal'][bus]
except KeyError:
pass
subs.update(all_subscriptions)
subs.update(subscriptions)
subscribers = set()
# Check for local subscribers
for prefix, subscription in subs.iteritems():
if subscription and topic.startswith(prefix):
subscribers |= subscription
if subscribers:
# self._logger.debug("PUBSUBSERVICE: found subscribers: {}".format(subscribers))
for subscriber in subscribers:
frames[0] = zmq.Frame(subscriber)
try:
# Send the message to the subscriber
for sub in self._send(frames, publisher):
# Drop the subscriber if unreachable
self.peer_drop(sub)
except ZMQError:
raise
return len(subscribers)
def _read_protected_topics_file(self):
#Read protected topics file and send to router
try:
create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
self._protected_topics = jsonapi.loads(data) if data else {}
self._send_protected_update_to_pubsub(self._protected_topics)
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
def _handle_result(self, message):
if message.args and not message.args[0]:
try:
result = self._results.pop(bytes(message.id))
except KeyError:
return
try:
value = jsonapi.loads(bytes(message.args[1]))
except IndexError:
value = None
result.set(value)
def _read_protected_topics_file(self):
#Read protected topics file and send to router
try:
create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
self._protected_topics = jsonapi.loads(data) if data else {}
self._send_protected_update_to_pubsub(self._protected_topics)
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
def build_remote_connection_param(self,
rmq_user,
rmq_address,
ssl_auth=None,
retry_attempt=30,
retry_delay=2):
"""
Build Pika Connection parameters
:param rmq_user: RabbitMQ user
:param ssl_auth: If SSL based connection or not
:return:
"""
from urlparse import urlparse
parsed_addr = urlparse(rmq_address)
ssl_auth = ssl_auth if ssl_auth is not None else self.is_ssl
_, virtual_host = parsed_addr.path.split('/')
try:
if ssl_auth:
certfile = self.certs.cert_file(rmq_user, True)
metafile = certfile[:-4] + ".json"
metadata = jsonapi.loads(open(metafile).read())
local_keyfile = metadata['local_keyname']
ca_file = self.certs.cert_file(metadata['remote_ca_name'],
True)
ssl_options = dict(
ssl_version=ssl.PROTOCOL_TLSv1,
ca_certs=ca_file,
keyfile=self.certs.private_key_file(local_keyfile),
certfile=self.certs.cert_file(rmq_user, True),
cert_reqs=ssl.CERT_REQUIRED)
conn_params = pika.ConnectionParameters(
host=parsed_addr.hostname,
port=parsed_addr.port,
virtual_host=virtual_host,
ssl=True,
connection_attempts=retry_attempt,
retry_delay=retry_delay,
ssl_options=ssl_options,
credentials=pika.credentials.ExternalCredentials())
else:
conn_params = pika.ConnectionParameters(
host=parsed_addr.hostname,
port=parsed_addr.port,
virtual_host=virtual_host,
credentials=pika.credentials.PlainCredentials(
rmq_user, rmq_user))
except KeyError:
return None
return conn_params
def ext_route(self, socket):
"""
Handler function for message received through external socket connection
:param socket: socket
:return:
"""
# Expecting incoming frames to follow this VIP format:
# [SENDER, PROTO, USER_ID, MSG_ID, SUBSYS, ...]
frames = socket.recv_multipart(copy=False)
# for f in frames:
# _log.debug("PUBSUBSERVICE Frames: {}".format(bytes(f)))
if len(frames) < 6:
return
sender, proto, user_id, msg_id, subsystem = frames[:5]
if proto.bytes != b'VIP1':
return
# Handle 'EXT_RPC' subsystem messages
name = subsystem.bytes
if name == 'external_rpc':
# Reframe the frames
sender, proto, usr_id, msg_id, subsystem, msg = frames[:6]
msg_data = jsonapi.loads(msg.bytes)
peer = msg_data['to_peer']
# Send to destionation agent/peer
# Form new frame for local
frames[:9] = [
peer, sender, proto, usr_id, msg_id, 'external_rpc', msg
]
try:
self.socket.send_multipart(frames, flags=NOBLOCK, copy=False)
except ZMQError as ex:
_log.debug("ZMQ error: {}".format(ex))
pass
# Handle 'pubsub' subsystem messages
elif name == 'pubsub':
if bytes(frames[1]) == b'VIP1':
recipient = b''
frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')]
# for f in frames:
# _log.debug("frames: {}".format(bytes(f)))
result = self._pubsub.handle_subsystem(frames, user_id)
return result
# Handle 'routing_table' subsystem messages
elif name == 'routing_table':
# for f in frames:
# _log.debug("frames: {}".format(bytes(f)))
if bytes(frames[1]) == b'VIP1':
frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')]
result = self._ext_routing.handle_subsystem(frames)
return result
def _handle_external_rpc_subsystem(self, message):
ret_msg = dict()
_log.debug("EXT_RPC subsystem handler IN message {0}".format(message))
op = message.args[0].bytes
rpc_msg = jsonapi.loads(message.args[1].bytes)
try:
# _log.debug("EXT_RPC subsystem handler IN message {0}, {1}".format(message.peer, rpc_msg))
method_args = rpc_msg['args']
#message.args = [method_args]
message.args = method_args
dispatch = self._dispatcher.dispatch
# _log.debug("External RPC IN message args {}".format(message))
responses = [
response for response in (dispatch(bytes(msg), message)
for msg in message.args) if response
]
# _log.debug("External RPC Resonses {}".format(responses))
if responses:
message.user = ''
try:
message.peer = ''
message.subsystem = 'external_rpc'
frames = []
op = b'send_platform'
frames.append(op)
msg = jsonapi.dumps(
dict(to_platform=rpc_msg['from_platform'],
to_peer=rpc_msg['from_peer'],
from_platform=rpc_msg['to_platform'],
from_peer=rpc_msg['to_peer'],
args=responses))
frames.append(msg)
except KeyError:
_log.error(
"External RPC message did not contain proper message format"
)
message.args = jsonapi.dumps(ret_msg)
ret_msg = jsonapi.dumps(ret_msg)
#_log.debug("EXT_RPC subsystem handler OUT message {}".format(message))
try:
self.core().connection.send_vip(peer=b'',
subsystem=b'external_rpc',
args=frames,
msg_id=message.id,
user=message.user)
except ZMQError as ex:
_log.error("ZMQ error: {}".format(ex))
pass
except KeyError:
pass
def _update_protected_topics(self, frames):
"""
Update the protected topics and capabilities as per message received from AuthService.
:peer frames list of frames
:type frames list
"""
if len(frames) > 7:
data = frames[7].bytes
try:
msg = jsonapi.loads(data)
self._load_protected_topics(msg)
except ValueError:
pass
def rmq_callback(ch, method, properties, body):
# Strip prefix from routing key
topic = self._get_original_topic(str(method.routing_key))
try:
msg = jsonapi.loads(body)
headers = msg['headers']
message = msg['message']
bus = msg['bus']
sender = msg['sender']
self.core().spawn(callback, 'pubsub', sender, bus, topic,
headers, message)
except KeyError as esc:
self._logger.error(
"Missing keys in pubsub message {}".format(esc))
def ext_route(self, socket):
"""
Handler function for message received through external socket connection
:param socket: socket
:return:
"""
# Expecting incoming frames to follow this VIP format:
# [SENDER, PROTO, USER_ID, MSG_ID, SUBSYS, ...]
frames = socket.recv_multipart(copy=False)
# for f in frames:
# _log.debug("PUBSUBSERVICE Frames: {}".format(bytes(f)))
if len(frames) < 6:
return
sender, proto, user_id, msg_id, subsystem = frames[:5]
if proto.bytes != b'VIP1':
return
# Handle 'EXT_RPC' subsystem messages
name = subsystem.bytes
if name == 'external_rpc':
# Reframe the frames
sender, proto, usr_id, msg_id, subsystem, msg = frames[:6]
msg_data = jsonapi.loads(msg.bytes)
peer = msg_data['to_peer']
# Send to destionation agent/peer
# Form new frame for local
frames[:9] = [peer, sender, proto, usr_id, msg_id, 'external_rpc', msg]
try:
self.socket.send_multipart(frames, flags=NOBLOCK, copy=False)
except ZMQError as ex:
_log.debug("ZMQ error: {}".format(ex))
pass
# Handle 'pubsub' subsystem messages
elif name == 'pubsub':
if bytes(frames[1]) == b'VIP1':
recipient = b''
frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')]
# for f in frames:
# _log.debug("frames: {}".format(bytes(f)))
result = self._pubsub.handle_subsystem(frames, user_id)
return result
# Handle 'routing_table' subsystem messages
elif name == 'routing_table':
# for f in frames:
# _log.debug("frames: {}".format(bytes(f)))
if bytes(frames[1]) == b'VIP1':
frames[:1] = [zmq.Frame(b''), zmq.Frame(b'')]
result = self._ext_routing.handle_subsystem(frames)
return result
def _read_auth_file(self):
auth_path = os.path.join(self.volttron_home, 'auth.json')
try:
with open(auth_path, 'r') as fd:
data = strip_comments(FileObject(fd, close=False).read())
if data:
auth = jsonapi.loads(data)
else:
auth = {}
except IOError:
auth = {}
if 'allow' not in auth:
auth['allow'] = []
return auth, auth_path
def _update_caps_users(self, frames):
"""
Stores the user capabilities sent by the Auth Service
:param frames list of frames
:type frames list
"""
if len(frames) > 7:
data = frames[7].bytes
try:
msg = jsonapi.loads(data)
self._user_capabilities = msg['capabilities']
except KeyError as exc:
self._logger.error("Missing key in update auth capabilities message {}".format(exc))
except ValueError:
pass
def _peer_list(self, frames):
"""Returns a list of subscriptions for a specific bus. If bus is None, then it returns list of subscriptions
for all the buses.
:param frames list of frames
:type frames list
:returns: list of tuples of bus, topic and flag to indicate if peer is a subscriber or not
:rtype: list
:Return Values:
List of tuples [(bus, topic, flag to indicate if peer is a subscriber or not)].
"""
results = []
if len(frames) > 7:
data = frames[7].bytes
msg = jsonapi.loads(data)
peer = frames[0].bytes
try:
prefix = msg['prefix']
bus = msg['bus']
subscribed = msg['subscribed']
reverse = msg['reverse']
except KeyError as exc:
self._logger.error(
"Missing key in _peer_list message {}".format(exc))
return results
is_all = msg.get('all_platforms', False)
if not is_all:
platform = 'internal'
else:
platform = 'all'
if bus is None:
buses = self._peer_subscriptions[platform].iteritems()
else:
buses = [(bus, self._peer_subscriptions[platform][bus])]
if reverse:
test = prefix.startswith
else:
test = lambda t: t.startswith(prefix)
for bus, subscriptions in buses:
for topic, subscribers in subscriptions.iteritems():
if test(topic):
member = peer in subscribers
if not subscribed or member:
results.append((bus, topic, member))
results = jsonapi.dumps(results)
return results
def query(self,
topic_ids,
id_name_map,
start=None,
end=None,
skip=0,
agg_type=None,
agg_period=None,
count=None,
order='FIRST_TO_LAST'):
if agg_type and agg_period:
table_name = agg_type + '_' + agg_period
else:
table_name = self.data_table
topic_id = Literal(0)
query = [
SQL('''SELECT to_char(ts, 'YYYY-MM-DD"T"HH24:MI:SS.USOF:00'), '''
'value_string\n'
'FROM {}\n'
'WHERE topic_id = {}').format(Identifier(table_name), topic_id)
]
if start and start.tzinfo != pytz.UTC:
start = start.astimezone(pytz.UTC)
if end and end.tzinfo != pytz.UTC:
end = end.astimezone(pytz.UTC)
if start and start == end:
query.append(SQL(' AND ts = {}').format(Literal(start)))
else:
if start:
query.append(SQL(' AND ts >= {}').format(Literal(start)))
if end:
query.append(SQL(' AND ts < {}').format(Literal(end)))
query.append(
SQL('ORDER BY ts {}'.format('DESC' if order ==
'LAST_TO_FIRST' else 'ASC')))
if skip or count:
query.append(
SQL('LIMIT {} OFFSET {}').format(
Literal(None if not count or count < 0 else count),
Literal(None if not skip or skip < 0 else skip)))
query = SQL('\n').join(query)
values = {}
for topic_id._wrapped in topic_ids:
name = id_name_map[topic_id.wrapped]
with self.select(query, fetch_all=False) as cursor:
values[name] = [(ts, jsonapi.loads(value))
for ts, value in cursor]
return values
def _read_protected_topics_file(self):
# Read protected topics file and send to router
try:
create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
self._protected_topics = jsonapi.loads(data) if data else {}
if self.core.messagebus == 'rmq':
self._load_protected_topics_for_rmq()
# Deferring the RMQ topic permissions to after "onstart" event
else:
self._send_protected_update_to_pubsub(
self._protected_topics)
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
def parse(jsonstr):
data = jsonapi.loads(jsonstr)
id = data.get('id', None)
version = data.get('jsonrpc', None)
method = data.get('method', None)
params = data.get('params', None)
authorization = data.get('authorization', None)
if id == None:
raise ParseError("Invalid id")
if version != '2.0':
print("VERSION IS: {}".format(version))
raise ParseError('Invalid jsonrpc version')
if method == None:
raise ParseError('Method not specified.')
return JsonRpcData(id, version, method, params, authorization)
def parse(jsonstr):
data = jsonapi.loads(jsonstr)
id = data.get('id', None)
version = data.get('jsonrpc', None)
method = data.get('method', None)
params = data.get('params', None)
authorization = data.get('authorization', None)
if id == None:
raise ParseError("Invalid id")
if version != '2.0':
print("VERSION IS: {}".format(version))
raise ParseError('Invalid jsonrpc version')
if method == None:
raise ParseError('Method not specified.')
return JsonRpcData(id, version, method, params, authorization)
def _peer_sync(self, frames):
"""
Synchronizes the subscriptions with the calling agent.
:param frames list of frames
:type frames list
"""
if len(frames) > 8:
conn = frames[7].bytes
if conn == b'connected':
data = frames[8].bytes
msg = jsonapi.loads(data)
peer = frames[0].bytes
try:
items = msg['subscriptions']
assert isinstance(items, dict)
self._sync(peer, items)
except KeyError as exc:
self._logger.error("Missing key in _peer_sync message {}".format(exc))
def rpc_message_handler(self, ch, method, props, body):
"""
:param ch:
:param method:
:param props:
:param body:
:return:
"""
zmq_frames = []
frames = jsonapi.loads(body)
for frame in frames:
zmq_frames.append(bytes(frame))
try:
self.zmq_router.socket.send_multipart(zmq_frames, copy=False)
except ZMQError as ex:
_log.error("ZMQ Error {}".format(ex))
def test_instance_writes_to_instances_file(volttron_instance):
vi = volttron_instance
assert vi is not None
assert vi.is_running()
instances_file = os.path.expanduser("~/.volttron_instances")
with open(instances_file, 'r') as fp:
result = jsonapi.loads(fp.read())
assert result.get(vi.volttron_home)
the_instance_entry = result.get(vi.volttron_home)
for key in ('pid', 'vip-address', 'volttron-home', 'start-args'):
assert the_instance_entry.get(key)
assert the_instance_entry['pid'] == vi.p_process.pid
assert the_instance_entry['vip-address'][0] == vi.vip_address
assert the_instance_entry['volttron-home'] == vi.volttron_home
def _process_incoming_message(self, message):
"""Process incoming messages
param message: VIP message from PubSubService
type message: dict
"""
op = message.args[0].bytes
if op == 'request_response':
result = None
try:
result = self._results.pop(bytes(message.id))
except KeyError:
pass
if self._parameters_needed:
self._send_via_rpc = False
self._parameters_needed = False
self._pubsubwithrpc.clear_parameters()
del self._pubsubwithrpc
response = message.args[1].bytes
#_log.debug("Message result: {}".format(response))
if result:
result.set(response)
elif op == 'publish':
try:
topic = topic = message.args[1].bytes
data = message.args[2].bytes
except IndexError:
return
try:
msg = jsonapi.loads(data)
headers = msg['headers']
message = msg['message']
sender = msg['sender']
bus = msg['bus']
except KeyError as exc:
_log.error("Missing keys in pubsub message: {}".format(exc))
else:
self._process_callback(sender, bus, topic, headers, message)
else:
_log.error("Unknown operation ({})".format(op))
def _handle_external_rpc_subsystem(self, message):
ret_msg = dict()
#_log.debug("EXT_RPC subsystem handler IN message {0}".format(message))
op = message.args[0].bytes
rpc_msg = jsonapi.loads(message.args[1].bytes)
try:
#_log.debug("EXT_RPC subsystem handler IN message {0}, {1}".format(message.peer, rpc_msg))
method_args = rpc_msg['args']
#message.args = [method_args]
message.args = method_args
dispatch = self._dispatcher.dispatch
#_log.debug("External RPC IN message args {}".format(message))
responses = [response for response in (
dispatch(bytes(msg), message) for msg in message.args) if response]
#_log.debug("External RPC Resonses {}".format(responses))
if responses:
message.user = ''
try:
message.peer = ''
message.subsystem = 'external_rpc'
frames = []
op = b'send_platform'
frames.append(op)
msg = jsonapi.dumps(dict(to_platform=rpc_msg['from_platform'],
to_peer=rpc_msg['from_peer'],
from_platform=rpc_msg['to_platform'],
from_peer=rpc_msg['to_peer'], args=responses))
frames.append(msg)
except KeyError:
_log.error("External RPC message did not contain proper message format")
message.args = jsonapi.dumps(ret_msg)
ret_msg = jsonapi.dumps(ret_msg)
#_log.debug("EXT_RPC subsystem handler OUT message {}".format(message))
try:
self.core().socket.send_vip(b'', 'external_rpc', frames,
user=message.user, msg_id=message.id, copy=False)
except ZMQError as ex:
_log.error("ZMQ error: {}".format(ex))
pass
except KeyError:
pass
def from_json(data, status_changed_callback=None):
"""
Deserializes a `Status` object and returns it to the caller.
:param data:
:param status_changed_callback:
:return:
"""
_log.debug("from_json {}".format(data))
statusobj = Status()
cp = jsonapi.loads(data)
cp['_status'] = cp['status']
cp['_last_updated'] = cp['last_updated']
cp['_context'] = cp['context']
del cp['status']
del cp['last_updated']
del cp['context']
statusobj.__dict__ = cp
statusobj._status_changed_callback = status_changed_callback
return statusobj
def _read(self):
auth_data = {}
try:
create_file_if_missing(self.auth_file)
with open(self.auth_file) as fil:
# Use gevent FileObject to avoid blocking the thread
before_strip_comments = FileObject(fil, close=False).read()
data = strip_comments(before_strip_comments)
if data != before_strip_comments:
_log.warn('Comments in %s are deprecated and will not be '
'preserved', self.auth_file)
if data:
auth_data = jsonapi.loads(data)
except Exception:
_log.exception('error loading %s', self.auth_file)
allow_list = auth_data.get('allow', [])
groups = auth_data.get('groups', {})
roles = auth_data.get('roles', {})
version = auth_data.get('version', {'major': 0, 'minor': 0})
return allow_list, groups, roles, version
def query(self, topic_ids, id_name_map, start=None, end=None, skip=0,
agg_type=None, agg_period=None, count=None,
order='FIRST_TO_LAST'):
if agg_type and agg_period:
table_name = agg_type + '_' + agg_period
else:
table_name = self.data_table
topic_id = Literal(0)
query = [SQL(
'''SELECT to_char(ts, 'YYYY-MM-DD"T"HH24:MI:SS.USOF:00'), '''
'value_string\n'
'FROM {}\n'
'WHERE topic_id = {}'
).format(Identifier(table_name), topic_id)]
if start and start.tzinfo != pytz.UTC:
start = start.astimezone(pytz.UTC)
if end and end.tzinfo != pytz.UTC:
end = end.astimezone(pytz.UTC)
if start and start == end:
query.append(SQL(' AND ts = {}').format(Literal(start)))
else:
if start:
query.append(SQL(' AND ts >= {}').format(Literal(start)))
if end:
query.append(SQL(' AND ts < {}').format(Literal(end)))
query.append(SQL('ORDER BY ts {}'.format(
'DESC' if order == 'LAST_TO_FIRST' else 'ASC')))
if skip or count:
query.append(SQL('LIMIT {} OFFSET {}').format(
Literal(None if not count or count < 0 else count),
Literal(None if not skip or skip < 0 else skip)))
query = SQL('\n').join(query)
values = {}
for topic_id._wrapped in topic_ids:
name = id_name_map[topic_id.wrapped]
with self.select(query, fetch_all=False) as cursor:
values[name] = [(ts, jsonapi.loads(value))
for ts, value in cursor]
return values
def is_instance_running(volttron_home=None):
from volttron.platform.agent import json as jsonapi
if volttron_home is None:
volttron_home = get_home()
instance_file = os.path.expanduser("~/.volttron_instances")
if not os.path.isfile(instance_file):
return False
with open(instance_file, 'r') as fp:
jsonobj = jsonapi.loads(fp.read())
if volttron_home not in jsonobj:
return False
obj = jsonobj[volttron_home]
pid = obj.get('pid', None)
if not pid:
return False
return psutil.pid_exists(pid)
def parse_json_config(config_str):
"""Parse a JSON-encoded configuration file."""
return jsonapi.loads(strip_comments(config_str))
from crate_historian import crate_utils
from volttron.platform.agent import utils
from volttron.platform.dbutils import mongoutils
logging.basicConfig(level=logging.DEBUG)
_log = logging.getLogger(__name__)
for key in logging.Logger.manager.loggerDict:
_log.debug(key)
logging.getLogger('crate.client.http').setLevel(logging.INFO)
logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO)
root = os.path.dirname(os.path.abspath(__file__))
with open('{}/crate_config'.format(root), 'r') as fp:
crate_params = jsonapi.loads(fp.read())
root = os.path.dirname(os.path.abspath(__file__))
with open('{}/mongo_config'.format(root), 'r') as fp:
mongo_params = jsonapi.loads(fp.read())
MAX_QUEUE_SIZE = 50000
QUEUE_BATCH_SIZE = 5000
class TableQueue(Queue.Queue, object):
def __init__(self, table_name):
super(TableQueue, self).__init__()
self.table_name = table_name
def store_agent_config(self, session_user, params):
required = ('agent_identity', 'config_name', 'raw_contents')
message_id = params.pop('message_id')
errors = []
for r in required:
if r not in params:
errors.append('Missing {}'.format(r))
config_type = params.get('config_type', None)
if config_type:
if config_type not in ('raw', 'json', 'csv'):
errors.append('Invalid config_type parameter')
if errors:
return jsonrpc.json_error(message_id, INVALID_PARAMS,
"\n".join(errors))
try:
self._log.debug("Calling store_agent_config on external platform.")
self.call("store_agent_config", **params)
except Exception as e:
self._log.error(repr(e))
return jsonrpc.json_error(message_id, INTERNAL_ERROR,
str(e))
config_name = params.get("config_name")
agent_identity = params.get("agent_identity")
if config_name.startswith("devices"):
# Since we start with devices, we assume that we are attempting
# to save a master driver config file.
rawdict = jsonapi.loads(params['raw_contents'])
# if this is not a bacnet device_type then we cannot do anything
# more than save and retrieve it from the store.
driver_type = rawdict.get('driver_type', None)
if driver_type is None or driver_type not in ('bacnet', 'modbus'):
return jsonrpc.json_result(message_id, "SUCCESS")
# Registry config starts with config://
registry_config = rawdict['registry_config'][len('config://'):]
try:
self._log.debug("Retrieving registry_config for new device.")
point_config = self.call("get_agent_config",
agent_identity,
registry_config, raw=False)
except Exception as e:
self._log.error(str(e))
return jsonrpc.json_error(message_id, INTERNAL_ERROR,
"Couldn't retrieve registry_config "
"from connection.")
else:
new_device = dict(
device_address=rawdict['driver_config']['device_address'],
device_id=rawdict['driver_config']['device_id'],
points=[],
path=config_name,
health=Status.build(UNKNOWN_STATUS,
context="Unpublished").as_dict()
)
points = [p['Volttron Point Name'] for p in point_config]
new_device['points'] = points
self._vc.send_management_message("NEW_DEVICE", new_device)
status = Status.build(UNKNOWN_STATUS,
context="Not published since update")
device_config_name = params.get('config_name')
device_no_prefix = device_config_name[len('devices/'):]
the_device = self._current_devices.get(device_no_prefix, {})
if not the_device:
self._current_devices[device_no_prefix] = dict(
last_publish_utc=None,
health=status.as_dict(),
points=points
)
else:
self._current_devices[device_no_prefix]['points'] = points
return jsonrpc.json_result(message_id, "SUCCESS")
def deserialize(self, json_string):
return jsonapi.loads(json_string)
def install_agent(self, agent_wheel=None, agent_dir=None, config_file=None,
start=True, vip_identity=None):
"""
Install and optionally start an agent on the instance.
This function allows installation from an agent wheel or an
agent directory (NOT BOTH). If an agent_wheel is specified then
it is assumed to be ready for installation (has a config file).
If an agent_dir is specified then a config_file file must be
specified or if it is not specified then it is assumed that the
file agent_dir/config is to be used as the configuration file. If
none of these exist then an assertion error will be thrown.
This function will return with a uuid of the installed agent.
:param agent_wheel:
:param agent_dir:
:param config_file:
:param start:
:param vip_identity:
:return:
"""
assert self.is_running(), "Instance must be running to install agent."
assert agent_wheel or agent_dir, "Invalid agent_wheel or agent_dir."
if agent_wheel:
assert not agent_dir
assert not config_file
assert os.path.exists(agent_wheel)
wheel_file = agent_wheel
agent_uuid = self._install_agent(wheel_file, start, vip_identity)
# Now if the agent_dir is specified.
if agent_dir:
assert not agent_wheel
if isinstance(config_file, dict):
from os.path import join, basename
temp_config = join(self.volttron_home,
basename(agent_dir) + "_config_file")
with open(temp_config, "w") as fp:
fp.write(json.dumps(config_file))
config_file = temp_config
elif not config_file:
if os.path.exists(os.path.join(agent_dir, "config")):
config_file = os.path.join(agent_dir, "config")
else:
from os.path import join, basename
temp_config = join(self.volttron_home,
basename(agent_dir) + "_config_file")
with open(temp_config, "w") as fp:
fp.write(json.dumps({}))
config_file = temp_config
elif os.path.exists(config_file):
pass # config_file already set!
else:
raise ValueError("Can't determine correct config file.")
script = os.path.join(self.volttron_root,
"scripts/install-agent.py")
cmd = [self.python, script,
"--volttron-home", self.volttron_home,
"--volttron-root", self.volttron_root,
"--agent-source", agent_dir,
"--config", config_file,
"--json"]
if vip_identity:
cmd.extend(["--vip-identity", vip_identity])
if start:
cmd.extend(["--start"])
try:
response = subprocess.check_output(cmd)
except Exception as e:
_log.error(repr(e))
raise e
self.logit(response)
# Because we are no longer silencing output from the install, the
# the results object is now much more verbose. Our assumption is
# that the result we are looking for is the only JSON block in
# the output
match = re.search(r'^({.*})', response, flags=re.M | re.S)
if match:
results = match.group(0)
else:
raise ValueError(
"The results were not found in the command output")
self.logit("here are the results: {}".format(results))
#
# Response from results is expected as follows depending on
# parameters, note this is a json string so parse to get dictionary
# {
# "started": true,
# "agent_pid": 26241,
# "starting": true,
# "agent_uuid": "ec1fd94e-922a-491f-9878-c392b24dbe50"
# }
assert results
resultobj = jsonapi.loads(str(results))
if start:
assert resultobj['started']
agent_uuid = resultobj['agent_uuid']
assert agent_uuid is not None
if start:
assert self.is_agent_running(agent_uuid)
return agent_uuid
def configure_main(self, config_name, action, contents):
config = self.default_config.copy()
config.update(contents)
if action == "NEW":
try:
self.max_open_sockets = config["max_open_sockets"]
if self.max_open_sockets is not None:
max_open_sockets = int(self.max_open_sockets)
configure_socket_lock(max_open_sockets)
_log.info("maximum concurrently open sockets limited to " + str(max_open_sockets))
elif self.system_socket_limit is not None:
max_open_sockets = int(self.system_socket_limit * 0.8)
_log.info("maximum concurrently open sockets limited to " + str(max_open_sockets) +
" (derived from system limits)")
configure_socket_lock(max_open_sockets)
else:
configure_socket_lock()
_log.warn("No limit set on the maximum number of concurrently open sockets. "
"Consider setting max_open_sockets if you plan to work with 800+ modbus devices.")
self.max_concurrent_publishes = config['max_concurrent_publishes']
max_concurrent_publishes = int(self.max_concurrent_publishes)
if max_concurrent_publishes < 1:
_log.warn("No limit set on the maximum number of concurrent driver publishes. "
"Consider setting max_concurrent_publishes if you plan to work with many devices.")
else:
_log.info("maximum concurrent driver publishes limited to " + str(max_concurrent_publishes))
configure_publish_lock(max_concurrent_publishes)
self.scalability_test = bool(config["scalability_test"])
self.scalability_test_iterations = int(config["scalability_test_iterations"])
if self.scalability_test:
self.waiting_to_finish = set()
self.test_iterations = 0
self.test_results = []
self.current_test_start = None
except ValueError as e:
_log.error("ERROR PROCESSING STARTUP CRITICAL CONFIGURATION SETTINGS: {}".format(e))
_log.error("MASTER DRIVER SHUTTING DOWN")
sys.exit(1)
else:
if self.max_open_sockets != config["max_open_sockets"]:
_log.info("The master driver must be restarted for changes to the max_open_sockets setting to take effect")
if self.max_concurrent_publishes != config["max_concurrent_publishes"]:
_log.info("The master driver must be restarted for changes to the max_concurrent_publishes setting to take effect")
if self.scalability_test != bool(config["scalability_test"]):
if not self.scalability_test:
_log.info(
"The master driver must be restarted with scalability_test set to true in order to run a test.")
if self.scalability_test:
_log.info(
"A scalability test may not be interrupted. Restarting the driver is required to stop the test.")
try:
if self.scalability_test_iterations != int(config["scalability_test_iterations"]) and self.scalability_test:
_log.info(
"A scalability test must be restarted for the scalability_test_iterations setting to take effect.")
except ValueError:
pass
#update override patterns
if self._override_patterns is None:
try:
values = self.vip.config.get("override_patterns")
values = jsonapi.loads(values)
if isinstance(values, dict):
self._override_patterns = set()
for pattern, end_time in values.items():
#check the end_time
now = utils.get_aware_utc_now()
#If end time is indefinite, set override with indefinite duration
if end_time == "0.0":
self._set_override_on(pattern, 0.0, from_config_store=True)
else:
end_time = utils.parse_timestamp_string(end_time)
# If end time > current time, set override with new duration
if end_time > now:
delta = end_time - now
self._set_override_on(pattern, delta.total_seconds(), from_config_store=True)
else:
self._override_patterns = set()
except KeyError:
self._override_patterns = set()
except ValueError:
_log.error("Override patterns is not set correctly in config store")
self._override_patterns = set()
try:
driver_scrape_interval = float(config["driver_scrape_interval"])
except ValueError as e:
_log.error("ERROR PROCESSING CONFIGURATION: {}".format(e))
_log.error("Master driver scrape interval settings unchanged")
# TODO: set a health status for the agent
try:
group_offset_interval = float(config["group_offset_interval"])
except ValueError as e:
_log.error("ERROR PROCESSING CONFIGURATION: {}".format(e))
_log.error("Master driver group interval settings unchanged")
# TODO: set a health status for the agent
if self.scalability_test and action == "UPDATE":
_log.info("Running scalability test. Settings may not be changed without restart.")
return
if (self.driver_scrape_interval != driver_scrape_interval or
self.group_offset_interval != group_offset_interval):
self.driver_scrape_interval = driver_scrape_interval
self.group_offset_interval = group_offset_interval
_log.info("Setting time delta between driver device scrapes to " + str(driver_scrape_interval))
#Reset all scrape schedules
self.freed_time_slots.clear()
self.group_counts.clear()
for driver in self.instances.itervalues():
time_slot = self.group_counts[driver.group]
driver.update_scrape_schedule(time_slot, self.driver_scrape_interval,
driver.group, self.group_offset_interval)
self.group_counts[driver.group] += 1
self.publish_depth_first_all = bool(config["publish_depth_first_all"])
self.publish_breadth_first_all = bool(config["publish_breadth_first_all"])
self.publish_depth_first = bool(config["publish_depth_first"])
self.publish_breadth_first = bool(config["publish_breadth_first"])
#Update the publish settings on running devices.
for driver in self.instances.itervalues():
driver.update_publish_types(self.publish_depth_first_all,
self.publish_breadth_first_all,
self.publish_depth_first,
self.publish_breadth_first)
from crate import client
import os
from volttron.platform.agent import json as jsonapi
root = os.path.dirname(os.path.abspath(__file__))
with open('{}/crate_config'.format(root), 'r') as fp:
data = jsonapi.loads(fp.read())
host = data['connection']['params']['host']
conn = client.connect(host, error_trace=True)
cursor = conn.cursor()
schema = 'test_import'
tables = ['analysis', 'analysis_string',
'datalogger', 'datalogger_string',
'device', 'device_string', 'topic',
'meta', 'record']
for t in tables:
try:
if schema:
full_table_name = "{schema}.{table}".format(schema=schema,
table=t)
else:
full_table_name = t
cursor.execute("DROP TABLE {}".format(full_table_name))
except Exception as ex:
print(ex.message)
