def __init__(self, certificate_dir=None):
"""Creates a Certs instance"""
self.default_certs_dir = os.path.join(get_home(), 'certificates')
self.root_ca_name = get_platform_instance_name() + '-root-ca'
self.trusted_ca_name = get_platform_instance_name() + '-trusted-cas'
self.default_root_ca_cn = '{} {}'.format(gethostname(),
self.root_ca_name)
if not certificate_dir:
certificate_dir = self.default_certs_dir
# If user provided explicit directory then it should exist
if not os.path.exists(certificate_dir):
if certificate_dir != self.default_certs_dir:
raise ValueError('Invalid cert_dir {}'.format(self.cert_dir))
self.cert_dir = os.path.join(os.path.expanduser(certificate_dir),
'certs')
self.private_dir = os.path.join(os.path.expanduser(certificate_dir),
'private')
self.ca_db_dir = os.path.join(os.path.expanduser(certificate_dir),
'ca_db')
self.csr_pending_dir = os.path.join(os.path.expanduser(certificate_dir),
'pending_csr')
self.remote_cert_dir = os.path.join(os.path.expanduser(certificate_dir),
'remote_certs')
self.certs_pending_dir = os.path.join(os.path.expanduser(certificate_dir),
'pending_certs')
self.rejected_dir = os.path.join(os.path.expanduser(certificate_dir),
'rejected')
required_paths = (self.cert_dir, self.private_dir, self.ca_db_dir,
self.csr_pending_dir, self.remote_cert_dir, self.certs_pending_dir)
try:
dir_created = False
for p in required_paths:
if not os.path.exists(p):
# explicitly provide rx to others since agent users should
# have read access to these dirs
os.makedirs(p)
os.chmod(p, 0o755)
dir_created = True
else:
# if one exists all of them should exist. break
break
if dir_created:
os.chmod(os.path.expanduser(certificate_dir), 0o755)
except Exception:
raise RuntimeError("No permission to create certificates directory")
def __init__(self):
self.instance_name = get_platform_instance_name()
# This is written durn the bootstrap phase of the rabbitmq installation
# however for docker we don't write this at all so we need to
# use the default location for this
rmq_home_file = os.path.expanduser("~/.volttron_rmq_home")
if os.path.isfile(rmq_home_file):
with open(os.path.expanduser("~/.volttron_rmq_home")) as f:
self.rabbitmq_server = f.read().strip()
else:
self.rabbitmq_server = os.path.expanduser(
"~/rabbitmq_server/rabbitmq_server-3.7.7/")
assert os.path.isdir(self.rabbitmq_server
), "Missing rabbitmq server directory{}".format(
self.rabbitmq_server)
self.crts = certs.Certs()
self.volttron_home = get_home()
self.volttron_rmq_config = os.path.join(self.volttron_home,
'rabbitmq_config.yml')
self.default_pass = "******"
self.config_opts = None
try:
self.load_rmq_config()
except (IOError, yaml.YAMLError) as exc:
self.config_opts = {}
self._set_default_config()
def _create_web_certs():
global config_opts
"""
Utility to create web server certificates
Designed to be used in conjecture with get_cert_and_key
As such, it assumes that the program has already checked
for existing certs, and prompted the user to enter in
certificates that they have generated separately.
"""
crts = certs.Certs()
try:
crts.ca_cert()
except certs.CertError:
print("WARNING! CA certificate does not exist.")
prompt_str = "Create new root CA?"
prompt = prompt_response(prompt_str, valid_answers=y_or_n, default='Y')
if prompt in y:
cert_data = {}
print(
"\nPlease enter the following details for web server certificate:"
)
prompt = '\tCountry:'
cert_data['country'] = prompt_response(prompt, default='US')
prompt = '\tState:'
cert_data['state'] = prompt_response(prompt, mandatory=True)
prompt = '\tLocation:'
cert_data['location'] = prompt_response(prompt, mandatory=True)
prompt = '\tOrganization:'
cert_data['organization'] = prompt_response(prompt, mandatory=True)
prompt = '\tOrganization Unit:'
cert_data['organization-unit'] = prompt_response(prompt,
mandatory=True)
cert_data['common-name'] = get_platform_instance_name(
) + '-root-ca'
data = {
'C': cert_data.get('country'),
'ST': cert_data.get('state'),
'L': cert_data.get('location'),
'O': cert_data.get('organization'),
'OU': cert_data.get('organization-unit'),
'CN': cert_data.get('common-name')
}
crts.create_root_ca(overwrite=False, **data)
copy(crts.cert_file(crts.root_ca_name),
crts.cert_file(crts.trusted_ca_name))
else:
return 1
print("Creating new web server certificate.")
crts.create_signed_cert_files(name=PLATFORM_WEB + "-server",
cert_type='server',
ca_name=crts.root_ca_name,
fqdn=get_hostname())
return 0
def __init__(self, certificate_dir=None):
"""Creates a Certs instance"""
self.default_certs_dir = os.path.join(get_home(), 'certificates')
self.root_ca_name = get_platform_instance_name() + '-root-ca'
self.trusted_ca_name = get_platform_instance_name() + '-trusted-cas'
self.default_root_ca_cn = '{} {}'.format(gethostname(),
self.root_ca_name)
if not certificate_dir:
certificate_dir = self.default_certs_dir
# If user provided explicit directory then it should exist
if not os.path.exists(certificate_dir):
if certificate_dir != self.default_certs_dir:
raise ValueError('Invalid cert_dir {}'.format(self.cert_dir))
self.cert_dir = os.path.join(os.path.expanduser(certificate_dir),
'certs')
self.private_dir = os.path.join(os.path.expanduser(certificate_dir),
'private')
self.ca_db_dir = os.path.join(os.path.expanduser(certificate_dir),
'ca_db')
self.csr_pending_dir = os.path.join(os.path.expanduser(certificate_dir),
'pending_csr')
self.remote_cert_dir = os.path.join(os.path.expanduser(certificate_dir),
'remote_certs')
self.certs_pending_dir = os.path.join(os.path.expanduser(certificate_dir),
'pending_certs')
self.rejected_dir = os.path.join(os.path.expanduser(certificate_dir),
'rejected')
required_paths = (self.cert_dir, self.private_dir, self.ca_db_dir,
self.csr_pending_dir, self.remote_cert_dir, self.certs_pending_dir)
for p in required_paths:
if not os.path.exists(p):
os.makedirs(p, 0o755)
def _build_connection_parameters(self):
param = None
if self.identity is None:
raise ValueError("Agent's VIP identity is not set")
else:
try:
if self.instance_name == get_platform_instance_name():
param = self.rmq_mgmt.build_agent_connection(self.identity,
self.instance_name)
else:
param = self.rmq_mgmt.build_remote_connection_param(self.rmq_user,
self.rmq_address,
True)
except AttributeError:
_log.error("RabbitMQ broker may not be running. Restart the broker first")
param = None
return param
def _update_external_subscriptions(self, frames):
"""
Store external subscriptions
:param frames: frames containing external subscriptions
:return:
"""
results = []
if len(frames) <= 7:
return False
else:
_log.debug(f"external subscription frames {frames}")
msg = frames[7]
if not isinstance(msg, dict):
raise ValueError(f"Invalid frame passed for frame {frames[7]}")
try:
this_platform_instance_name = get_platform_instance_name()
for instance_name in msg:
if instance_name == this_platform_instance_name:
_log.error(
"Invalid configuraiton of external instances!\n"
f"The name {instance_name} is specified as local and "
"external instance name. Please fix this issue in the "
"external_platform_discovery.json file in the "
"the VOLTTRON_HOME of the external instance.")
continue
prefixes = msg[instance_name]
# Store external subscription list for later use (during publish)
self._ext_subscriptions[instance_name] = prefixes
self._logger.debug(
"PUBSUBSERVICE New external list from {0}: List: {1}".
format(instance_name, self._ext_subscriptions))
if self._rabbitmq_agent:
for prefix in prefixes:
self._rabbitmq_agent.vip.pubsub.subscribe(
'pubsub', prefix, self.publish_callback)
except KeyError as exc:
self._logger.error(
"Unknown external instance name: {}".format(instance_name))
return False
return True
def _csr_request_new(self, env, data):
_log.debug("New csr request")
if not isinstance(data, dict):
try:
request_data = jsonapi.loads(data)
except:
_log.error(
"Invalid data for csr request. Must be json serializable")
return Response()
else:
request_data = data.copy()
csr = request_data.get('csr').encode("utf-8")
identity = self._certs.get_csr_common_name(csr)
# The identity must start with the current instances name or it is a failure.
if not identity.startswith(get_platform_instance_name() + "."):
json_response = dict(
status="ERROR",
message="CSR must start with instance name: {}".format(
get_platform_instance_name()))
Response(jsonapi.dumps(json_response),
content_type='application/json',
headers={'Content-type': 'application/json'})
csr_file = self._certs.save_pending_csr_request(
env.get('REMOTE_ADDR'), identity, csr)
if self._auto_allow_csr:
_log.debug(
"Creating cert and permissions for user: {}".format(identity))
status = self._certs.get_csr_status(identity)
json_response = dict(status=status)
if status == 'APPROVED':
try:
json_response['cert'] = self._certs.get_cert_from_csr(
identity)
except Exception as e:
_log.error(f"Exception getting cert from csr {e}")
else:
try:
cert = self._certs.approve_csr(identity)
#permissions = self._core().rmq_mgmt.get_default_permissions(identity)
_log.debug(f"CREATING NEW RMQ USER: {identity}")
permissions = dict(configure=".*", read=".*", write=".*")
self._core().rmq_mgmt.create_user_with_permissions(
identity, permissions, True)
json_response = dict(status="SUCCESSFUL", cert=cert)
except BaseException as e:
_log.error(
f"Exception in approving csr/creating user in auto_allow_csr mode: {e}"
)
else:
status = self._certs.get_csr_status(identity)
cert = self._certs.get_cert_from_csr(identity)
json_response = dict(status=status)
if status == "APPROVED":
json_response['cert'] = self._certs.get_cert_from_csr(identity)
elif status == "PENDING":
json_response[
'message'] = "The request is pending admininstrator approval."
elif status == "DENIED":
json_response[
'message'] = "The request has been denied by the administrator."
elif status == "UNKNOWN":
json_response[
'message'] = "An unknown common name was specified to the server {}".format(
identity)
else:
json_response[
'message'] = "An unkonwn error has occured during the respons phase"
response = None
try:
if json_response.get('cert', None):
json_response['cert'] = json_response['cert'].decode('utf-8')
response = Response(jsonapi.dumps(json_response),
content_type='application/json',
headers={'Content-type': 'application/json'})
except BaseException as e:
_log.error(f"Exception creating Response {e}")
return response
def connect_remote_platform(
self,
address,
serverkey=None,
agent_class=None
):
"""
Agent attempts to connect to a remote platform to exchange data.
address must start with http, https, tcp, ampq, or ampqs or a
ValueError will be
raised
If this function is successful it will return an instance of the
`agent_class`
parameter if not then this function will return None.
If the address parameter begins with http or https
TODO: use the known host functionality here
the agent will attempt to use Discovery to find the values
associated with it.
Discovery should return either an rmq-address or a vip-address or
both. In
that situation the connection will be made using zmq. In the event
that
fails then rmq will be tried. If both fail then None is returned
from this
function.
"""
from volttron.platform.vip.agent.utils import build_agent
from volttron.platform.vip.agent import Agent
if agent_class is None:
agent_class = Agent
parsed_address = urlparse(address)
_log.debug("Begining auth.connect_remote_platform: {}".format(address))
value = None
if parsed_address.scheme == "tcp":
# ZMQ connection
hosts = KnownHostsStore()
temp_serverkey = hosts.serverkey(address)
if not temp_serverkey:
_log.info(
"Destination serverkey not found in known hosts file, "
"using config"
)
destination_serverkey = serverkey
elif not serverkey:
destination_serverkey = temp_serverkey
else:
if temp_serverkey != serverkey:
raise ValueError(
"server_key passed and known hosts serverkey do not "
""
"match!"
)
destination_serverkey = serverkey
publickey, secretkey = (
self._core().publickey,
self._core().secretkey,
)
_log.debug(
"Connecting using: %s", get_fq_identity(self._core().identity)
)
value = build_agent(
agent_class=agent_class,
identity=get_fq_identity(self._core().identity),
serverkey=destination_serverkey,
publickey=publickey,
secretkey=secretkey,
message_bus="zmq",
address=address,
)
elif parsed_address.scheme in ("https", "http"):
from volttron.platform.web import DiscoveryInfo
from volttron.platform.web import DiscoveryError
try:
# TODO: Use known host instead of looking up for discovery
# info if possible.
# We need to discover which type of bus is at the other end.
info = DiscoveryInfo.request_discovery_info(address)
remote_identity = "{}.{}.{}".format(
info.instance_name,
get_platform_instance_name(),
self._core().identity,
)
# if the current message bus is zmq then we need
# to connect a zmq on the remote, whether that be the
# rmq router or proxy. Also note that we are using the
# fully qualified
# version of the identity because there will be conflicts if
# volttron central has more than one platform.agent connecting
if get_messagebus() == "zmq":
if not info.vip_address or not info.serverkey:
err = (
"Discovery from {} did not return serverkey "
"and/or vip_address".format(address)
)
raise ValueError(err)
_log.debug(
"Connecting using: %s",
get_fq_identity(self._core().identity),
)
# use fully qualified identity
value = build_agent(
identity=get_fq_identity(self._core().identity),
address=info.vip_address,
serverkey=info.serverkey,
secretkey=self._core().secretkey,
publickey=self._core().publickey,
agent_class=agent_class,
)
else: # we are on rmq messagebus
# This is if both remote and local are rmq message buses.
if info.messagebus_type == "rmq":
_log.debug("Both remote and local are rmq messagebus.")
fqid_local = get_fq_identity(self._core().identity)
# Check if we already have the cert, if so use it
# instead of requesting cert again
remote_certs_dir = self.get_remote_certs_dir()
remote_cert_name = "{}.{}".format(
info.instance_name, fqid_local
)
certfile = os.path.join(
remote_certs_dir, remote_cert_name + ".crt"
)
if os.path.exists(certfile):
response = certfile
else:
response = self.request_cert(
address, fqid_local, info
)
if response is None:
_log.error("there was no response from the server")
value = None
elif isinstance(response, tuple):
if response[0] == "PENDING":
_log.info(
"Waiting for administrator to accept a "
"CSR request."
)
value = None
# elif isinstance(response, dict):
# response
elif os.path.exists(response):
# info = DiscoveryInfo.request_discovery_info(
# address)
# From the remote platforms perspective the
# remote user name is
# remoteinstance.localinstance.identity,
# this is what we must
# pass to the build_remote_connection_params
# for a successful
remote_rmq_user = get_fq_identity(
fqid_local, info.instance_name
)
_log.debug(
"REMOTE RMQ USER IS: %s", remote_rmq_user
)
remote_rmq_address = self._core().rmq_mgmt.build_remote_connection_param(
remote_rmq_user,
info.rmq_address,
ssl_auth=True,
cert_dir=self.get_remote_certs_dir(),
)
value = build_agent(
identity=fqid_local,
address=remote_rmq_address,
instance_name=info.instance_name,
publickey=self._core().publickey,
secretkey=self._core().secretkey,
message_bus="rmq",
enable_store=False,
agent_class=agent_class,
)
else:
raise ValueError(
"Unknown path through discovery process!"
)
else:
# TODO: cache the connection so we don't always have
# to ping the server to connect.
# This branch happens when the message bus is not
# the same note
# this writes to the agent-data directory of this
# agent if the agent
# is installed.
if get_messagebus() == "rmq":
if not os.path.exists("keystore.json"):
with open("keystore.json", "w") as file_pointer:
file_pointer.write(
jsonapi.dumps(
KeyStore.generate_keypair_dict()
)
)
with open("keystore.json") as file_pointer:
keypair = jsonapi.loads(file_pointer.read())
value = build_agent(
agent_class=agent_class,
identity=remote_identity,
serverkey=info.serverkey,
publickey=keypair.get("publickey"),
secretkey=keypair.get("secretekey"),
message_bus="zmq",
address=info.vip_address,
)
except DiscoveryError:
_log.error(
"Couldn't connect to %s or incorrect response returned "
"response was %s",
address,
value,
)
else:
raise ValueError(
"Invalid configuration found the address: {} has an invalid "
"scheme".format(address)
)
return value
def setup_rabbitmq_volttron(setup_type, verbose=False, prompt=False, instance_name=None,
rmq_conf_file=None, env=None):
"""
Setup VOLTTRON instance to run with RabbitMQ message bus.
:param setup_type:
single - Setup to run as single instance
federation - Setup to connect multiple VOLTTRON instances as
a federation
shovel - Setup shovels to forward local messages to remote instances
:param verbose
:param prompt
:raises RabbitMQSetupAlreadyError
"""
if not instance_name:
instance_name = get_platform_instance_name(prompt=True)
# Store config this is checked at startup
store_message_bus_config(message_bus='rmq', instance_name=instance_name)
rmq_config = RMQConfig()
if verbose:
_log.setLevel(logging.DEBUG)
_log.debug("verbose set to True")
_log.debug(get_home())
logging.getLogger("requests.packages.urllib3.connectionpool"
"").setLevel(logging.DEBUG)
else:
_log.setLevel(logging.INFO)
logging.getLogger("requests.packages.urllib3.connectionpool"
"").setLevel(logging.WARN)
if prompt:
# ignore any existing rabbitmq_config.yml in vhome. Prompt user and
# generate a new rabbitmq_config.yml
_create_rabbitmq_config(rmq_config, setup_type)
# Load either the newly created config or config passed
try:
rmq_config.load_rmq_config()
except (yaml.parser.ParserError, yaml.scanner.ScannerError, yaml.YAMLError) as exc:
_log.error("Error: YAML file cannot parsed properly. Check the contents of the file")
return exc
except IOError as exc:
_log.error("Error opening {}. Please create a rabbitmq_config.yml "
"file in your volttron home. If you want to point to a "
"volttron home other than {} please set it as the "
"environment variable VOLTTRON_HOME".format(
rmq_config.volttron_rmq_config, rmq_config.volttron_home))
_log.error("\nFor single setup, configuration file must at least "
"contain host and ssl certificate details. For federation "
"and shovel setup, config should contain details about the "
"volttron instance with which communication needs "
"to be established. Please refer to example config file "
"at examples/configurations/rabbitmq/rabbitmq_config.yml")
raise
if not rmq_conf_file:
rmq_conf_file = os.path.join(rmq_config.rmq_home, "etc/rabbitmq/rabbitmq.conf")
invalid = True
if setup_type in ["all", "single"]:
invalid = False
# Verify that the rmq_conf_file if exists is removed before continuing.
message = f"A rabbitmq conf file {rmq_conf_file} already exists.\n" \
"In order for setup to proceed it must be removed.\n"
if os.path.exists(rmq_conf_file):
print(message)
while os.path.exists(rmq_conf_file):
value = prompt_response(f"Remove {rmq_conf_file}? ", y_or_n)
if value in y:
os.remove(rmq_conf_file)
_start_rabbitmq_without_ssl(rmq_config, rmq_conf_file, env=env)
_log.debug("Creating rabbitmq virtual hosts and required users for "
"volttron")
# Create local RabbitMQ setup - vhost, exchange etc.
# should be called after _start_rabbitmq_without_ssl
rmq_mgmt = RabbitMQMgmt()
success = rmq_mgmt.init_rabbitmq_setup()
if success and rmq_config.is_ssl:
_setup_for_ssl_auth(rmq_config, rmq_conf_file, env=env)
# Create utility scripts
script_path = os.path.dirname(os.path.realpath(__file__))
src_home = os.path.dirname(os.path.dirname(script_path))
start_script = os.path.join(src_home, 'start-rabbitmq')
with open(start_script, 'w+') as f:
f.write(os.path.join(rmq_config.rmq_home, 'sbin',
'rabbitmq-server') + ' -detached')
f.write(os.linesep)
f.write("sleep 5") # give a few seconds for all plugins to be ready
os.chmod(start_script, 0o755)
stop_script = os.path.join(src_home, 'stop-rabbitmq')
with open(stop_script, 'w+') as f:
f.write(os.path.join(rmq_config.rmq_home, 'sbin',
'rabbitmqctl') + ' stop')
os.chmod(stop_script, 0o755)
# symlink to rmq log
log_name = os.path.join(src_home, 'rabbitmq.log')
if os.path.lexists(log_name):
os.unlink(log_name)
os.symlink(os.path.join(rmq_config.rmq_home,
'var/log/rabbitmq',
rmq_config.node_name + "@" +
rmq_config.hostname.split('.')[0] + ".log"),
log_name)
if setup_type in ["all", "federation"]:
# Create a multi-platform federation setup
invalid = False
_create_federation_setup(rmq_config.admin_user,
rmq_config.admin_pwd,
rmq_config.is_ssl,
rmq_config.virtual_host,
rmq_config.volttron_home)
if setup_type in ["all", "shovel"]:
# Create shovel setup
invalid = False
if rmq_config.is_ssl:
port = rmq_config.amqp_port_ssl
else:
port = rmq_config.amqp_port
_create_shovel_setup(rmq_config.instance_name,
rmq_config.hostname,
port,
rmq_config.virtual_host,
rmq_config.volttron_home,
rmq_config.is_ssl)
if invalid:
_log.error("Unknown option. Exiting....")
