class KeyDiscoveryAgent(Agent):
"""
Class to get server key, instance name and vip address of external/remote platforms
"""
def __init__(self, address, serverkey, identity, external_address_config,
setup_mode, bind_web_address, *args, **kwargs):
super(KeyDiscoveryAgent, self).__init__(identity, address, **kwargs)
self._external_address_file = external_address_config
self._ext_addresses = dict()
self._grnlets = dict()
self._vip_socket = None
self._my_web_address = bind_web_address
self.r = random.random()
self._setup_mode = setup_mode
if self._setup_mode:
_log.debug("RUNNING IN MULTI-PLATFORM SETUP MODE")
self._store_path = os.path.join(os.environ['VOLTTRON_HOME'],
'external_platform_discovery.json')
self._ext_addresses_store = dict()
self._ext_addresses_store_lock = Semaphore()
@Core.receiver('onstart')
def startup(self, sender, **kwargs):
"""
Try to get platform discovery info of all the remote platforms. If unsuccessful, setup events to try again later
:param sender: caller
:param kwargs: optional arguments
:return:
"""
self._vip_socket = self.core.socket
#If in setup mode, read the external_addresses.json to get web addresses to set up authorized connection with
# external platforms.
if self._setup_mode:
if self._my_web_address is None:
_log.error(
"Web bind address is needed in multiplatform setup mode")
return
with self._ext_addresses_store_lock:
try:
self._ext_addresses_store = PersistentDict(
filename=self._store_path, flag='c', format='json')
except ValueError as exc:
_log.error("Error in json format: {0}".format(exc))
#Delete the existing store.
if self._ext_addresses_store:
self._ext_addresses_store.clear()
self._ext_addresses_store.async_sync()
web_addresses = dict()
#Read External web addresses file
try:
web_addresses = self._read_platform_address_file()
try:
web_addresses.remove(self._my_web_address)
except ValueError:
_log.debug(
"My web address is not in the external bind web adress list"
)
op = b'web-addresses'
self._send_to_router(op, web_addresses)
except IOError as exc:
_log.error("Error in reading file: {}".format(exc))
return
sec = random.random() * self.r + 10
delay = utils.get_aware_utc_now() + timedelta(seconds=sec)
grnlt = self.core.schedule(delay, self._key_collection,
web_addresses)
else:
#Use the existing store for platform discovery information
with self._ext_addresses_store_lock:
try:
self._ext_addresses_store = PersistentDict(
filename=self._store_path, flag='c', format='json')
except ValueError as exc:
_log.error("Error in json file format: {0}".format(exc))
for name, discovery_info in self._ext_addresses_store.items():
op = b'normalmode_platform_connection'
self._send_to_router(op, discovery_info)
def _key_collection(self, web_addresses):
"""
Collect platform discovery information (server key, instance name, vip-address) for all platforms.
:param web_addresses: List of web addresses to get discovery info
:return:
"""
for web_address in web_addresses:
if web_address not in self._my_web_address:
self._collect_key(web_address)
def _collect_key(self, web_address):
"""
Try to get (server key, instance name, vip-address) of remote instance and send it to RoutingService
to connect to the remote instance. If unsuccessful, try again later.
:param name: instance name
:param web_address: web address of remote instance
:return:
"""
platform_info = dict()
try:
platform_info = self._get_platform_discovery(web_address)
with self._ext_addresses_store_lock:
_log.debug("Platform discovery info: {}".format(platform_info))
name = platform_info['instance-name']
self._ext_addresses_store[name] = platform_info
self._ext_addresses_store.async_sync()
except KeyError as exc:
_log.error(
"Discovery info does not contain instance name {}".format(exc))
except DiscoveryError:
# If discovery error, try again later
sec = random.random() * self.r + 30
delay = utils.get_aware_utc_now() + timedelta(seconds=sec)
grnlet = self.core.schedule(delay, self._collect_key, web_address)
except ConnectionError as e:
_log.error("HTTP connection error {}".format(e))
#If platform discovery is successful, send the info to RoutingService
#to establish connection with remote platform.
if platform_info:
op = b'setupmode_platform_connection'
connection_settings = dict(platform_info)
connection_settings['web-address'] = web_address
self._send_to_router(op, connection_settings)
def _send_to_router(self, op, platform_info):
"""
Send the platform discovery stats to the router to establish new connection
:param platform_info: platform discovery stats
:return:
"""
address = jsonapi.dumps(platform_info)
frames = [op, address]
try:
self._vip_socket.send_vip(b'', 'routing_table', frames, copy=False)
except ZMQError as ex:
# Try sending later
_log.error(
"ZMQ error while sending external platform info to router: {}".
format(ex))
def _read_platform_address_file(self):
"""
Read the external addresses file
:return:
"""
try:
with open(self._external_address_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
web_addresses = jsonapi.loads(data) if data else {}
return web_addresses
except IOError as e:
_log.error("Error opening file {}".format(
self._external_address_file))
raise
except Exception:
_log.exception('error loading %s', self._external_address_file)
raise
def _get_platform_discovery(self, web_address):
"""
Use http discovery call to get serverkey, instance name and vip-address of remote instance
:param web_address: web address of remote instance
:return:
"""
r = {}
try:
parsed = urlparse(web_address)
assert parsed.scheme
assert not parsed.path
real_url = urljoin(web_address, "/discovery/")
req = grequests.get(real_url)
responses = grequests.map([req])
responses[0].raise_for_status()
r = responses[0].json()
return r
except requests.exceptions.HTTPError:
raise DiscoveryError(
"Invalid discovery response from {}".format(real_url))
except requests.exceptions.Timeout:
raise DiscoveryError("Timeout error from {}".format(real_url))
except AttributeError as e:
raise DiscoveryError(
"Invalid web_address passed {}".format(web_address))
except (ConnectionError, NewConnectionError) as e:
raise DiscoveryError(
"Connection to {} not available".format(real_url))
except Exception as e:
raise DiscoveryError("Unknown Exception: {}".format(str(e)))
class AdminEndpoints(object):
def __init__(self, rmq_mgmt, ssl_public_key):
self._rmq_mgmt = rmq_mgmt
self._ssl_public_key = ssl_public_key
self._userdict = None
self.reload_userdict()
self._observer = Observer()
self._observer.schedule(
FileReloader("web-users.json", self.reload_userdict), get_home())
self._observer.start()
self._certs = Certs()
def reload_userdict(self):
webuserpath = os.path.join(get_home(), 'web-users.json')
self._userdict = PersistentDict(webuserpath)
def get_routes(self):
"""
Returns a list of tuples with the routes for the adminstration endpoints
available in it.
:return:
"""
return [(re.compile('^/admin.*'), 'callable', self.admin)]
def admin(self, env, data):
if len(self._userdict) == 0:
if env.get('REQUEST_METHOD') == 'POST':
decoded = dict((k, v if len(v) > 1 else v[0])
for k, v in urlparse.parse_qs(data).iteritems())
username = decoded.get('username')
pass1 = decoded.get('password1')
pass2 = decoded.get('password2')
if pass1 == pass2 and pass1 is not None:
_log.debug("Setting master password")
self.add_user(username, pass1, groups=['admin'])
return Response('',
status='302',
headers={'Location': '/admin/login.html'})
template = template_env(env).get_template('first.html')
return Response(template.render())
if 'login.html' in env.get('PATH_INFO') or '/admin/' == env.get(
'PATH_INFO'):
template = template_env(env).get_template('login.html')
return Response(template.render())
return self.verify_and_dispatch(env, data)
def verify_and_dispatch(self, env, data):
""" Verify that the user is an admin and dispatch
:param env: web environment
:param data: data associated with a web form or json/xml request data
:return: Response object.
"""
from volttron.platform.web import get_user_claims, NotAuthorized
try:
claims = get_user_claims(env)
except NotAuthorized:
_log.error("Unauthorized user attempted to connect to {}".format(
env.get('PATH_INFO')))
return Response('<h1>Unauthorized User</h1>',
status="401 Unauthorized")
# Make sure we have only admins for viewing this.
if 'admin' not in claims.get('groups'):
return Response('<h1>Unauthorized User</h1>',
status="401 Unauthorized")
# Make sure we have only admins for viewing this.
if 'admin' not in claims.get('groups'):
return Response('<h1>Unauthorized User</h1>',
status="401 Unauthorized")
path_info = env.get('PATH_INFO')
if path_info.startswith('/admin/api/'):
return self.__api_endpoint(path_info[len('/admin/api/'):], data)
if path_info.endswith('html'):
page = path_info.split('/')[-1]
try:
template = template_env(env).get_template(page)
except TemplateNotFound:
return Response("<h1>404 Not Found</h1>",
status="404 Not Found")
if page == 'list_certs.html':
html = template.render(
certs=self._certs.get_all_cert_subjects())
elif page == 'pending_csrs.html':
html = template.render(
csrs=self._certs.get_pending_csr_requests())
else:
# A template with no params.
html = template.render()
return Response(html)
template = template_env(env).get_template('index.html')
resp = template.render()
return Response(resp)
def __api_endpoint(self, endpoint, data):
_log.debug("Doing admin endpoint {}".format(endpoint))
if endpoint == 'certs':
response = self.__cert_list_api()
elif endpoint == 'pending_csrs':
response = self.__pending_csrs_api()
elif endpoint.startswith('approve_csr/'):
response = self.__approve_csr_api(endpoint.split('/')[1])
elif endpoint.startswith('deny_csr/'):
response = self.__deny_csr_api(endpoint.split('/')[1])
elif endpoint.startswith('delete_csr/'):
response = self.__delete_csr_api(endpoint.split('/')[1])
else:
response = Response(
'{"status": "Unknown endpoint {}"}'.format(endpoint),
content_type="application/json")
return response
def __approve_csr_api(self, common_name):
try:
_log.debug("Creating cert and permissions for user: {}".format(
common_name))
self._certs.approve_csr(common_name)
permissions = self._rmq_mgmt.get_default_permissions(common_name)
self._rmq_mgmt.create_user_with_permissions(
common_name, permissions, True)
data = dict(status=self._certs.get_csr_status(common_name),
cert=self._certs.get_cert_from_csr(common_name))
except ValueError as e:
data = dict(status="ERROR", message=e.message)
return Response(json.dumps(data), content_type="application/json")
def __deny_csr_api(self, common_name):
try:
self._certs.deny_csr(common_name)
data = dict(status="DENIED",
message="The administrator has denied the request")
except ValueError as e:
data = dict(status="ERROR", message=e.message)
return Response(json.dumps(data), content_type="application/json")
def __delete_csr_api(self, common_name):
try:
self._certs.delete_csr(common_name)
data = dict(status="DELETED",
message="The administrator has denied the request")
except ValueError as e:
data = dict(status="ERROR", message=e.message)
return Response(json.dumps(data), content_type="application/json")
def __pending_csrs_api(self):
csrs = [c for c in self._certs.get_pending_csr_requests()]
return Response(json.dumps(csrs), content_type="application/json")
def __cert_list_api(self):
subjects = [
dict(common_name=x.common_name)
for x in self._certs.get_all_cert_subjects()
]
return Response(json.dumps(subjects), content_type="application/json")
def add_user(self, username, unencrypted_pw, groups=[], overwrite=False):
if self._userdict.get(username):
raise ValueError("Already exists!")
if groups is None:
groups = []
hashed_pass = argon2.hash(unencrypted_pw)
self._userdict[username] = dict(hashed_password=hashed_pass,
groups=groups)
self._userdict.async_sync()
class KeyDiscoveryAgent(Agent):
"""
Class to get server key, instance name and vip address of external/remote platforms
"""
def __init__(self, address, serverkey, identity, external_address_config,
setup_mode, bind_web_address, *args, **kwargs):
super(KeyDiscoveryAgent, self).__init__(identity, address, **kwargs)
self._external_address_file = external_address_config
self._ext_addresses = dict()
self._grnlets = dict()
self._vip_socket = None
self._my_web_address = bind_web_address
self.r = random.random()
self._setup_mode = setup_mode
if self._setup_mode:
_log.debug("RUNNING IN MULTI-PLATFORM SETUP MODE")
self._store_path = os.path.join(os.environ['VOLTTRON_HOME'],
'external_platform_discovery.json')
self._ext_addresses_store = dict()
self._ext_addresses_store_lock = Semaphore()
@Core.receiver('onstart')
def startup(self, sender, **kwargs):
"""
Try to get platform discovery info of all the remote platforms. If unsuccessful, setup events to try again later
:param sender: caller
:param kwargs: optional arguments
:return:
"""
self._vip_socket = self.core.socket
#If in setup mode, read the external_addresses.json to get web addresses to set up authorized connection with
# external platforms.
if self._setup_mode:
if self._my_web_address is None:
_log.error("Web bind address is needed in multiplatform setup mode")
return
with self._ext_addresses_store_lock:
try:
self._ext_addresses_store = PersistentDict(filename=self._store_path, flag='c', format='json')
except ValueError as exc:
_log.error("Error in json format: {0}".format(exc))
#Delete the existing store.
if self._ext_addresses_store:
self._ext_addresses_store.clear()
self._ext_addresses_store.async_sync()
web_addresses = dict()
#Read External web addresses file
try:
web_addresses = self._read_platform_address_file()
try:
web_addresses.remove(self._my_web_address)
except ValueError:
_log.debug("My web address is not in the external bind web adress list")
op = b'web-addresses'
self._send_to_router(op, web_addresses)
except IOError as exc:
_log.error("Error in reading file: {}".format(exc))
return
sec = random.random() * self.r + 10
delay = utils.get_aware_utc_now() + timedelta(seconds=sec)
grnlt = self.core.schedule(delay, self._key_collection, web_addresses)
else:
#Use the existing store for platform discovery information
with self._ext_addresses_store_lock:
try:
self._ext_addresses_store = PersistentDict(filename=self._store_path, flag='c', format='json')
except ValueError as exc:
_log.error("Error in json file format: {0}".format(exc))
for name, discovery_info in self._ext_addresses_store.items():
op = b'normalmode_platform_connection'
self._send_to_router(op, discovery_info)
def _key_collection(self, web_addresses):
"""
Collect platform discovery information (server key, instance name, vip-address) for all platforms.
:param web_addresses: List of web addresses to get discovery info
:return:
"""
for web_address in web_addresses:
if web_address not in self._my_web_address:
self._collect_key(web_address)
def _collect_key(self, web_address):
"""
Try to get (server key, instance name, vip-address) of remote instance and send it to RoutingService
to connect to the remote instance. If unsuccessful, try again later.
:param name: instance name
:param web_address: web address of remote instance
:return:
"""
platform_info = dict()
try:
platform_info = self._get_platform_discovery(web_address)
with self._ext_addresses_store_lock:
_log.debug("Platform discovery info: {}".format(platform_info))
name = platform_info['instance-name']
self._ext_addresses_store[name] = platform_info
self._ext_addresses_store.async_sync()
except KeyError as exc:
_log.error("Discovery info does not contain instance name {}".format(exc))
except DiscoveryError:
# If discovery error, try again later
sec = random.random() * self.r + 30
delay = utils.get_aware_utc_now() + timedelta(seconds=sec)
grnlet = self.core.schedule(delay, self._collect_key, web_address)
except ConnectionError as e:
_log.error("HTTP connection error {}".format(e))
#If platform discovery is successful, send the info to RoutingService
#to establish connection with remote platform.
if platform_info:
op = b'setupmode_platform_connection'
connection_settings = dict(platform_info)
connection_settings['web-address'] = web_address
self._send_to_router(op, connection_settings)
def _send_to_router(self, op, platform_info):
"""
Send the platform discovery stats to the router to establish new connection
:param platform_info: platform discovery stats
:return:
"""
address = jsonapi.dumps(platform_info)
frames = [op, address]
try:
self._vip_socket.send_vip(b'', 'routing_table', frames, copy=False)
except ZMQError as ex:
# Try sending later
_log.error("ZMQ error while sending external platform info to router: {}".format(ex))
def _read_platform_address_file(self):
"""
Read the external addresses file
:return:
"""
try:
with open(self._external_address_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
web_addresses = jsonapi.loads(data) if data else {}
return web_addresses
except IOError as e:
_log.error("Error opening file {}".format(self._external_address_file))
raise
except Exception:
_log.exception('error loading %s', self._external_address_file)
raise
def _get_platform_discovery(self, web_address):
"""
Use http discovery call to get serverkey, instance name and vip-address of remote instance
:param web_address: web address of remote instance
:return:
"""
r = {}
try:
parsed = urlparse(web_address)
assert parsed.scheme
assert not parsed.path
real_url = urljoin(web_address, "/discovery/")
req = grequests.get(real_url)
responses = grequests.map([req])
responses[0].raise_for_status()
r = responses[0].json()
return r
except requests.exceptions.HTTPError:
raise DiscoveryError(
"Invalid discovery response from {}".format(real_url)
)
except requests.exceptions.Timeout:
raise DiscoveryError(
"Timeout error from {}".format(real_url)
)
except AttributeError as e:
raise DiscoveryError(
"Invalid web_address passed {}"
.format(web_address)
)
except (ConnectionError, NewConnectionError) as e:
raise DiscoveryError(
"Connection to {} not available".format(real_url)
)
except Exception as e:
raise DiscoveryError(
"Unknown Exception".format(e.message)
)
