def setup_control_connection(request, get_volttron_instances): """ Creates a single instance of VOLTTRON for testing purposes """ global wrapper, control_connection wrapper = get_volttron_instances(1) request.addfinalizer(wrapper.shutdown_platform) assert wrapper assert wrapper.is_running() if get_volttron_instances.param == 'encrypted': if wrapper.encrypt: wrapper.allow_all_connections() # Connect using keys ks = KeyStore() ks.generate() control_connection = build_connection(identity="foo", address=wrapper.vip_address, peer=CONTROL, serverkey=wrapper.serverkey, publickey=ks.public, secretkey=ks.secret) else: control_connection = Connection(address=wrapper.local_vip_address, peer=CONTROL, developer_mode=True) # Sleep a couple seconds to wait for things to startup gevent.sleep(2) return wrapper, control_connection
def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, generatekeys=False, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if address is None: if use_ipc: self.logit('Using IPC vip-address') address = "ipc://@"+self.volttron_home+"/run/vip.socket" else: self.logit('Using vip-address '+self.vip_address) address = self.vip_address if generatekeys: self.logit('generating new public secret key pair') tf = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) ks.generate() publickey = ks.public() secretkey = ks.secret() if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = Agent(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event)#.join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) return agent
def build_connection(self, peer=None, address=None, identity=None, publickey=None, secretkey=None, serverkey=None, capabilities=[], **kwargs): self.logit('Building connection to {}'.format(peer)) self.allow_all_connections() if address is None: self.logit( 'Default address was None so setting to current instances') address = self.vip_address serverkey = self.serverkey if serverkey is None: self.logit("serverkey wasn't set but the address was.") raise Exception("Invalid state.") if publickey is None or secretkey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret entry = AuthEntry(capabilities=capabilities, comments="Added by test", credentials=keys.public) file = AuthFile(self.volttron_home + "/auth.json") file.add(entry) conn = Connection(address=address, peer=peer, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home) return conn
def forwarder(request, volttron_instances): #print "Fixture forwarder" global volttron_instance1, volttron_instance2 global forwarder_uuid, forwarder_config # 1. Update destination address in forwarder configuration if volttron_instance1.encrypt: tf = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) # generate public private key pair for instance1 ks.generate() # add public key of instance1 to instance2 auth file authfile = AuthFile(volttron_instance2.volttron_home + "/auth.json") entry = AuthEntry(credentials=ks.public()) authfile.add(entry) # setup destination address to include keys forwarder_config["destination-vip"] =\ "{}?serverkey={}&publickey={}&secretkey={}".format( volttron_instance2.vip_address, volttron_instance2.serverkey, ks.public(), ks.secret()) else: forwarder_config["destination-vip"] = volttron_instance2.vip_address # 1: Install historian agent # Install and start sqlhistorian agent in instance2 forwarder_uuid = volttron_instance1.install_agent( agent_dir="services/core/ForwardHistorian", config_file=forwarder_config, start=True) print("forwarder agent id: ", forwarder_uuid)
def start_wrapper_platform(wrapper, with_http=False, with_tcp=True, volttron_central_address=None, volttron_central_serverkey=None, add_local_vc_address=False): """ Customize easily customize the platform wrapper before starting it. """ assert not wrapper.is_running() vc_http = get_rand_http_address() if with_http else None vc_tcp = get_rand_tcp_address() if with_tcp else None if add_local_vc_address: ks = KeyStore(os.path.join(wrapper.volttron_home, 'keystore')) ks.generate() volttron_central_address = vc_tcp volttron_central_serverkey = ks.public wrapper.startup_platform(vip_address=vc_tcp, bind_web_address=vc_http, volttron_central_address=volttron_central_address, volttron_central_serverkey=volttron_central_serverkey) if with_http: discovery = "{}/discovery/".format(vc_http) response = requests.get(discovery) assert response.ok assert wrapper.is_running()
def forwarder(request, volttron_instances): #print "Fixture forwarder" global volttron_instance1, volttron_instance2 global forwarder_uuid, forwarder_config # 1. Update destination address in forwarder configuration if volttron_instance1.encrypt: tf = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) # generate public private key pair for instance1 ks.generate() # add public key of instance1 to instance2 auth file authfile = AuthFile(volttron_instance2.volttron_home + "/auth.json") entry = AuthEntry(credentials=ks.public) authfile.add(entry) # setup destination address to include keys forwarder_config["destination-vip"] =\ "{}?serverkey={}&publickey={}&secretkey={}".format( volttron_instance2.vip_address, volttron_instance2.serverkey, ks.public, ks.secret) else: forwarder_config["destination-vip"] = volttron_instance2.vip_address # 1: Install historian agent # Install and start sqlhistorian agent in instance2 forwarder_uuid = volttron_instance1.install_agent( agent_dir="services/core/ForwardHistorian", config_file=forwarder_config, start=True) print("forwarder agent id: ", forwarder_uuid)
def build_connection(self, peer=None, address=None, identity=None, publickey=None, secretkey=None, serverkey=None, **kwargs): if self.encrypt: self.allow_all_connections() if address is None: address = self.vip_address serverkey = self.serverkey if publickey is None or secretkey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public() secretkey = keys.secret() if self.encrypt: conn = Connection(address=address, peer=peer, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home) else: conn = Connection(address=self.local_vip_address, peer=peer, volttron_home=self.volttron_home) return conn
def start_wrapper_platform(wrapper, with_http=False, with_tcp=True, volttron_central_address=None, volttron_central_serverkey=None, add_local_vc_address=False): """ Customize easily customize the platform wrapper before starting it. """ assert not wrapper.is_running() vc_http = get_rand_http_address() if with_http else None vc_tcp = get_rand_tcp_address() if with_tcp else None if add_local_vc_address: ks = KeyStore(os.path.join(wrapper.volttron_home, 'keystore')) ks.generate() volttron_central_address = vc_tcp volttron_central_serverkey = ks.public wrapper.startup_platform( vip_address=vc_tcp, bind_web_address=vc_http, volttron_central_address=volttron_central_address, volttron_central_serverkey=volttron_central_serverkey) if with_http: discovery = "{}/discovery/".format(vc_http) response = requests.get(discovery) assert response.ok assert wrapper.is_running()
def setup_control_connection(request, get_volttron_instances): """ Creates a single instance of VOLTTRON for testing purposes """ global wrapper, control_connection wrapper = get_volttron_instances(1) request.addfinalizer(wrapper.shutdown_platform) assert wrapper assert wrapper.is_running() wrapper.allow_all_connections() # Connect using keys ks = KeyStore() ks.generate() control_connection = build_connection(identity="foo", address=wrapper.vip_address, peer=CONTROL, serverkey=wrapper.serverkey, publickey=ks.public, secretkey=ks.secret, instance_name=wrapper.instance_name, message_bus=wrapper.messagebus) # Sleep a couple seconds to wait for things to startup gevent.sleep(2) return wrapper, control_connection
def tcp_to(instance): tmp = tempfile.NamedTemporaryFile() key = KeyStore(tmp.name) key.generate() return "{}?serverkey={}&publickey={}&secretkey={}".format( instance.vip_address, instance.serverkey, key.public, key.secret)
def build_agent_with_key(platform: PlatformWrapper, identity=None): """Create an agent instance that has a generated public and private key. The passed platform will be the vip-address of the agent and the identity will be set. If the identity is set to None then a random identity will be created. """ os.environ['VOLTTRON_HOME'] = platform.volttron_home keys = KeyStore(os.path.join(platform.volttron_home, identity + '.keys')) keys.generate() agent = platform.build_agent(identity=identity, serverkey=platform.publickey, publickey=keys.public, secretkey=keys.secret) # Make publickey easily accessible for these tests agent.publickey = keys.public gevent.sleep(0.1) # switch context for a bit os.environ.pop('VOLTTRON_HOME') return agent
def build_agent_with_key(platform, identity=None): """Create an agent instance that has a generated public and private key. The passed platform will be the vip-address of the agent and the identity will be set. If the identity is set to None then a random identity will be created. """ keys = KeyStore(os.path.join(platform.volttron_home, identity + '.keys')) keys.generate() agent = platform.build_agent(identity=identity, serverkey=platform.publickey, publickey=keys.public(), secretkey=keys.secret()) # Make publickey easily accessible for these tests agent.publickey = keys.public() gevent.sleep(0.1) # switch context for a bit return agent
def test_forwarding(volttron_instance1_encrypt, volttron_instance2_encrypt): global FORWARDER_CONFIG tf = tempfile.NamedTemporaryFile() tf2 = tempfile.NamedTemporaryFile() tf3 = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) ks.generate() ks2 = KeyStore(tf2.name) ks2.generate() ks3 = KeyStore(tf2.name) ks3.generate() wrap1 = volttron_instance1_encrypt wrap2 = volttron_instance2_encrypt authfile1 = AuthFile(wrap1.volttron_home+"/auth.json") entry1 = AuthEntry( credentials="CURVE:{}".format(ks3.public()) ) authfile1.add(entry1) authfile = AuthFile(wrap2.volttron_home+"/auth.json") entry = AuthEntry( credentials="CURVE:{}".format(ks.public())) authfile.add(entry) entry = AuthEntry( credentials="CURVE:{}".format(ks2.public())) authfile.add(entry) forward_to_vip = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap2.vip_address, wrap2.publickey, ks.public(), ks.secret() ) FORWARDER_CONFIG["destination-vip"] = forward_to_vip forwarder_config = FORWARDER_CONFIG print("THE CONFIG = {}".format(forwarder_config)) wrap1.install_agent( agent_dir="services/core/ForwardHistorian", config_file=forwarder_config ) connect_to_wrap2 = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap2.vip_address, wrap2.publickey, ks2.public(), ks2.secret() ) connect_to_wrap1 = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap1.vip_address, wrap1.publickey, ks3.public(), ks3.secret() ) agent_connected1 = wrap1.build_agent(address=connect_to_wrap1) agent_connected2 = wrap2.build_agent(address=connect_to_wrap2) message = '' agent_connected2.vip.pubsub.subscribe('pubsub', '', callback=onmessage) gevent.sleep(0.2) do_publish(agent1=agent_connected1) gevent.sleep(1) assert allforwardedmessage
def test_forwarding(volttron_instance1_encrypt, volttron_instance2_encrypt): global FORWARDER_CONFIG tf = tempfile.NamedTemporaryFile() tf2 = tempfile.NamedTemporaryFile() tf3 = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) ks.generate() ks2 = KeyStore(tf2.name) ks2.generate() ks3 = KeyStore(tf2.name) ks3.generate() wrap1 = volttron_instance1_encrypt wrap2 = volttron_instance2_encrypt authfile1 = AuthFile(wrap1.volttron_home + "/auth.json") entry1 = AuthEntry(credentials="CURVE:{}".format(ks3.public())) authfile1.add(entry1) authfile = AuthFile(wrap2.volttron_home + "/auth.json") entry = AuthEntry(credentials="CURVE:{}".format(ks.public())) authfile.add(entry) entry = AuthEntry(credentials="CURVE:{}".format(ks2.public())) authfile.add(entry) forward_to_vip = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap2.vip_address, wrap2.publickey, ks.public(), ks.secret()) FORWARDER_CONFIG["destination-vip"] = forward_to_vip forwarder_config = FORWARDER_CONFIG print("THE CONFIG = {}".format(forwarder_config)) wrap1.install_agent(agent_dir="services/core/ForwardHistorian", config_file=forwarder_config) connect_to_wrap2 = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap2.vip_address, wrap2.publickey, ks2.public(), ks2.secret()) connect_to_wrap1 = "{}?serverkey={}&publickey={}&secretkey={}".format( wrap1.vip_address, wrap1.publickey, ks3.public(), ks3.secret()) agent_connected1 = wrap1.build_agent(address=connect_to_wrap1) agent_connected2 = wrap2.build_agent(address=connect_to_wrap2) message = '' agent_connected2.vip.pubsub.subscribe('pubsub', '', callback=onmessage) gevent.sleep(0.2) do_publish(agent1=agent_connected1) gevent.sleep(1) assert allforwardedmessage
def get_new_keypair(): tf = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) ks.generate() return ks.public, ks.secret
class PlatformWrapper: def __init__(self): """ Initializes a new VOLTTRON instance Creates a temporary VOLTTRON_HOME directory with a packaged directory for agents that are built. """ self.volttron_home = tempfile.mkdtemp() self.packaged_dir = os.path.join(self.volttron_home, "packaged") os.makedirs(self.packaged_dir) # in the context of this platform it is very important not to # use the main os.environ for anything. self.env = { 'VOLTTRON_HOME': self.volttron_home, 'PACKAGED_DIR': self.packaged_dir, 'DEBUG_MODE': os.environ.get('DEBUG_MODE', ''), 'DEBUG': os.environ.get('DEBUG', ''), 'PATH': VOLTTRON_ROOT + ':' + os.environ['PATH'] } # By default no web server should be started. self.bind_web_address = None self.discovery_address = None self.jsonrpc_endpoint = None self.volttron_central_address = None self.instance_name = None self.serverkey = None self.p_process = None self.t_process = None self.started_agent_pids = [] self.local_vip_address = None self.vip_address = None self.encrypt = False self.logit('Creating platform wrapper') # This was used when we are testing the SMAP historian. self.use_twistd = False # Added restricted code properties self.certsobj = None # Control whether the instance directory is cleaned up when shutdown. # if the environment variable DEBUG is set to a True value then the # instance is not cleaned up. self.skip_cleanup = False # This is used as command line entry replacement. Especially working # with older 2.0 agents. self.opts = None keystorefile = os.path.join(self.volttron_home, 'keystore') self.keystore = KeyStore(keystorefile) self.keystore.generate() def logit(self, message): print('{}: {}'.format(self.volttron_home, message)) def allow_all_connections(self): """ Add a /.*/ entry to the auth.json file. """ entry = AuthEntry(credentials="/.*/") authfile = AuthFile(self.volttron_home + "/auth.json") authfile.add(entry) def build_connection(self, peer=None, address=None, identity=None, publickey=None, secretkey=None, serverkey=None, **kwargs): if self.encrypt: self.allow_all_connections() if address is None: address = self.vip_address serverkey = self.serverkey if publickey is None or secretkey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public() secretkey = keys.secret() if self.encrypt: conn = Connection(address=address, peer=peer, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home) else: conn = Connection(address=self.local_vip_address, peer=peer, volttron_home=self.volttron_home) return conn def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, agent_class=Agent, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :param agent_class: Agent class to build :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if self.encrypt: if serverkey is None: serverkey = self.serverkey if publickey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public() secretkey = keys.secret() if address is None: if not self.encrypt: self.logit('Using IPC vip-address') address = "ipc://@" + self.volttron_home + "/run/vip.socket" else: self.logit('Using vip-address ' + self.vip_address) address = self.vip_address if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = agent_class(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event) #.join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) agent.publickey = publickey return agent def _read_auth_file(self): auth_path = os.path.join(self.volttron_home, 'auth.json') try: with open(auth_path, 'r') as fd: data = strip_comments(FileObject(fd, close=False).read()) if data: auth = jsonapi.loads(data) else: auth = {} except IOError: auth = {} if not 'allow' in auth: auth['allow'] = [] return auth, auth_path def _append_allow_curve_key(self, publickey): entry = AuthEntry(credentials=publickey) authfile = AuthFile(self.volttron_home + "/auth.json") authfile.add(entry) def add_vc(self): return add_vc_to_instance(self) def add_capabilities(self, publickey, capabilities): if isinstance(capabilities, basestring): capabilities = [capabilities] auth, auth_path = self._read_auth_file() cred = publickey allow = auth['allow'] entry = next((item for item in allow if item['credentials'] == cred), {}) caps = entry.get('capabilities', []) entry['capabilities'] = list(set(caps + capabilities)) with open(auth_path, 'w+') as fd: json.dump(auth, fd) def set_auth_dict(self, auth_dict): if auth_dict: with open(os.path.join(self.volttron_home, 'auth.json'), 'w') as fd: fd.write(json.dumps(auth_dict)) def startup_platform(self, vip_address, auth_dict=None, use_twistd=False, mode=UNRESTRICTED, encrypt=False, bind_web_address=None, volttron_central_address=None, volttron_central_serverkey=None): # if not isinstance(vip_address, list): # self.vip_address = [vip_address] # else: # self.vip_address = vip_address self.vip_address = vip_address self.encrypt = encrypt self.mode = mode self.bind_web_address = bind_web_address if self.bind_web_address: self.discovery_address = "{}/discovery/".format( self.bind_web_address) # Only available if vc is installed! self.jsonrpc_endpoint = "{}/jsonrpc".format(self.bind_web_address) enable_logging = self.env.get('ENABLE_LOGGING', False) debug_mode = self.env.get('DEBUG_MODE', False) if not debug_mode: debug_mode = self.env.get('DEBUG', False) self.skip_cleanup = self.env.get('SKIP_CLEANUP', False) if debug_mode: self.skip_cleanup = True enable_logging = True self.logit("In start up platform enable_logging is {} ".format( enable_logging)) assert self.mode in MODES, 'Invalid platform mode set: ' + str(mode) opts = None # see main.py for how we handle pub sub addresses. ipc = 'ipc://{}{}/run/'.format( '@' if sys.platform.startswith('linux') else '', self.volttron_home) self.local_vip_address = ipc + 'vip.socket' self.set_auth_dict(auth_dict) self.opts = { 'verify_agents': False, 'volttron_home': self.volttron_home, 'vip_address': vip_address, 'vip_local_address': ipc + 'vip.socket', 'publish_address': ipc + 'publish', 'subscribe_address': ipc + 'subscribe', 'bind_web_address': bind_web_address, 'volttron_central_address': volttron_central_address, 'volttron_central_serverkey': volttron_central_serverkey, 'platform_name': None, 'developer_mode': not encrypt, 'log': os.path.join(self.volttron_home, 'volttron.log'), 'log_config': None, 'monitor': True, 'autostart': True, 'log_level': logging.DEBUG, 'verboseness': logging.DEBUG } pconfig = os.path.join(self.volttron_home, 'config') config = {} # Set up the configuration file based upon the passed parameters. parser = configparser.ConfigParser() parser.add_section('volttron') parser.set('volttron', 'vip-address', vip_address) if bind_web_address: parser.set('volttron', 'bind-web-address', bind_web_address) if volttron_central_address: parser.set('volttron', 'volttron-central-address', volttron_central_address) if self.mode == UNRESTRICTED: # TODO Restricted code should set with volttron as contianer # if RESTRICTED_AVAILABLE: # config['mobility'] = False # config['resource-monitor'] = False # config['verify'] = False with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_UNRESTRICTED.format(**config)) parser.write(cfg) elif self.mode == RESTRICTED: if not RESTRICTED_AVAILABLE: raise ValueError("restricted is not available.") certsdir = os.path.join(self.volttron_home, 'certificates') print("certsdir", certsdir) self.certsobj = certs.Certs(certsdir) with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_RESTRICTED.format(**config)) # opts = type('Options', (), {'resource-monitor':False, # 'verify_agents': True, # 'volttron_home': self.volttron_home})() else: raise PlatformWrapperError( "Invalid platform mode specified: {}".format(mode)) log = os.path.join(self.volttron_home, 'volttron.log') if enable_logging: cmd = ['volttron', '-vv', '-l{}'.format(log)] else: cmd = ['volttron', '-l{}'.format(log)] if not encrypt: cmd.append('--developer-mode') print('process environment: {}'.format(self.env)) print('popen params: {}'.format(cmd)) self.p_process = Popen(cmd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert self.p_process is not None # A None value means that the process is still running. # A negative means that the process exited with an error. assert self.p_process.poll() is None self.serverkey = self.keystore.public() assert self.serverkey agent = self.build_agent() has_control = False times = 0 while not has_control and times < 10: times += 1 try: has_control = agent.vip.peerlist().get(timeout=.2) except gevent.Timeout: pass if not has_control: self.shutdown_platform() raise "Couldn't connect to core platform!" if bind_web_address: times = 0 has_discovery = False while times < 10: times += 1 try: resp = requests.get(self.discovery_address) if resp.ok: has_discovery = True break except Exception as e: gevent.sleep(0.1) self.logit("Connection error found {}".format(e)) if not has_discovery: raise "Couldn't connect to discovery platform." self.use_twistd = use_twistd # TODO: Revise this to start twistd with platform. if self.use_twistd: tconfig = os.path.join(self.volttron_home, TMP_SMAP_CONFIG_FILENAME) with closing(open(tconfig, 'w')) as cfg: cfg.write(TWISTED_CONFIG.format(**config)) tparams = [TWISTED_START, "-n", "smap", tconfig] self.t_process = subprocess.Popen(tparams, env=self.env) time.sleep(5) #self.t_process = subprocess.Popen(["twistd", "-n", "smap", "test-smap.ini"]) def is_running(self): self.logit("PROCESS IS RUNNING: {}".format(self.p_process)) return self.p_process is not None and self.p_process.poll() is None def twistd_is_running(self): return self.t_process is not None # def publish(self, topic, data): # '''Publish data to a zmq context. # # The publisher is goint to use the platform that is contained within # this wrapper to write data to. # ''' # if not self.zmq_context: # self.zmq_context = zmq.Context() # self.logit("binding publisher to: ", self.env['AGENT_PUB_ADDR']) # pub = zmq.Socket(self.zmq_context, zmq.PUB) # pub.bind(self.env['AGENT_PUB_ADDR']) # pub.send_multipart([topic, data]) # def fillout_file(self, filename, template, config_file): # # try: # config = json.loads(open(config_file, 'r').read()) # except Exception as e: # sys.stderr.write (str(e)) # raise PlatformWrapperError("Could not load configuration file for tests") # # config['tmpdir'] = self.tmpdir # # outfile = os.path.join(self.tmpdir, filename) # with closing(open(outfile, 'w')) as cfg: # cfg.write(template.format(**config)) # # return outfile def direct_sign_agentpackage_creator(self, package): assert (RESTRICTED), "Auth not available" print("wrapper.certsobj", self.certsobj.cert_dir) assert (auth.sign_as_creator( package, 'creator', certsobj=self.certsobj)), "Signing as {} failed.".format('creator') def direct_sign_agentpackage_admin(self, package): assert (RESTRICTED), "Auth not available" assert (auth.sign_as_admin( package, 'admin', certsobj=self.certsobj)), "Signing as {} failed.".format('admin') def direct_sign_agentpackage_initiator(self, package, config_file, contract): assert (RESTRICTED), "Auth not available" files = {"config_file": config_file, "contract": contract} assert (auth.sign_as_initiator(package, 'initiator', files=files, certsobj=self.certsobj) ), "Signing as {} failed.".format('initiator') def _aip(self): opts = type('Options', (), self.opts) aip = AIPplatform(opts) aip.setup() return aip # TODO Remove when verified that the other method works properly. # def _install_agent(self, wheel_file, start, vip_identity): # aip = self._aip() # auuid = aip.install_agent(wheel_file, vip_identity=vip_identity) # assert auuid is not None # if start: # self.logit('STARTING: {}'.format(wheel_file)) # status = self.start_agent(auuid) # # aip.start_agent(auuid) # # status = aip.agent_status(auuid) # self.logit('STATUS NOW: {}'.format(status)) # assert status > 0 # # return auuid def _install_agent(self, wheel_file, start, vip_identity): agent = self.build_agent() self.logit('Creating channel for sending the agent.') channel_name = str(uuid.uuid4()) channel = agent.vip.channel('control', channel_name) gevent.sleep(0.3) self.logit('calling control install agent.') result = agent.vip.rpc.call('control', 'install_agent', wheel_file, channel_name, vip_identity) self.logit('waiting for ready') response = channel.recv() if response != b'ready': raise ValueError( 'Invalid channel protocol returned {}'.format(response)) with open(wheel_file, 'rb') as fin: _log.debug('sending wheel to control.') while True: data = fin.read(8125) if not data: _log.debug('Finished sending data') break channel.send(data) _log.debug('sending done message.') channel.send('done') try: # must do this before channel closes or process will hang. auuid = result.get(timeout=10) _log.debug('closing channel') except gevent.Timeout: _log.error('Timeout in channel') finally: channel.close(linger=0) del channel if start: self.start_agent(auuid) return auuid def install_multiple_agents(self, agent_configs): """ Installs mutltiple agents on the platform. :param agent_configs:list A list of 3-tuple that allows the configuration of a platform in a single go. The tuple order is 1. path to the agent directory. 2. configuration data (either file or json data) 3. Whether the agent should be started or not. :return:list: A list of uuid's associated with the agents that were installed. :Note: In order for this method to be called the platform must be currently running. """ if not self.is_running(): raise PlatformWrapperError("Instance isn't running!") results = [] for path, config, start in agent_configs: results = self.install_agent(agent_dir=path, config_file=config, start=start) return results def install_agent(self, agent_wheel=None, agent_dir=None, config_file=None, start=True, vip_identity=None): """ Install and optionally start an agent on the instance. This function allows installation from an agent wheel or an agent directory (NOT BOTH). If an agent_wheel is specified then it is assumed to be ready for installation (has a config file). If an agent_dir is specified then a config_file file must be specified or if it is not specified then it is assumed that the file agent_dir/config is to be used as the configuration file. If none of these exist then an assertion error will be thrown. This function will return with a uuid of the installed agent. :param agent_wheel: :param agent_dir: :param config_file: :param start: :param vip_identity: :return: """ assert self.is_running(), "Instance must be running to install agent." assert agent_wheel or agent_dir, "Invalid agent_wheel or agent_dir." if agent_wheel: assert not agent_dir assert not config_file assert os.path.exists(agent_wheel) wheel_file = agent_wheel if agent_dir: assert not agent_wheel if isinstance(config_file, dict): from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps(config_file)) config_file = temp_config elif not config_file: assert os.path.exists(os.path.join(agent_dir, "config")) config_file = os.path.join(agent_dir, "config") elif os.path.exists(config_file): pass # config_file already set! else: raise ValueError("Can't determine correct config file.") self.logit('Building agent package') wheel_file = self.build_agentpackage(agent_dir, config_file) assert wheel_file agent_uuid = self._install_agent(wheel_file, start, vip_identity) assert agent_uuid is not None if start: assert self.is_agent_running(agent_uuid) return agent_uuid def start_agent(self, agent_uuid): self.logit('Starting agent {}'.format(agent_uuid)) self.logit("VOLTTRON_HOME SETTING: {}".format( self.env['VOLTTRON_HOME'])) cmd = ['volttron-ctl', 'start', agent_uuid] p = Popen(cmd, env=self.env, stdout=sys.stdout, stderr=sys.stderr) p.wait() # Confirm agent running cmd = ['volttron-ctl', 'status', agent_uuid] res = subprocess.check_output(cmd, env=self.env) self.logit("Subprocess res is {}".format(res)) assert 'running' in res pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos:pidend]) self.started_agent_pids.append(pid) return int(pid) def stop_agent(self, agent_uuid): # Confirm agent running _log.debug("STOPPING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl', 'stop', agent_uuid] res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_status(agent_uuid) def list_agents(self): agent = self.build_agent() print('PEER LIST: {}'.format(agent.vip.peerlist().get(timeout=10))) agent_list = agent.vip.rpc('control', 'list_agents').get(timeout=10) agent.core.stop(timeout=3) return agent_list def remove_agent(self, agent_uuid): """Remove the agent specified by agent_uuid""" _log.debug("REMOVING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl', 'remove', agent_uuid] res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_status(agent_uuid) def is_agent_running(self, agent_uuid): return self.agent_status(agent_uuid) is not None def agent_status(self, agent_uuid): _log.debug("AGENT_STATUS: {}".format(agent_uuid)) # Confirm agent running cmd = ['volttron-ctl', 'status', agent_uuid] pid = None try: res = subprocess.check_output(cmd, env=self.env) try: pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos:pidend]) except: pid = None except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return pid def build_agentpackage(self, agent_dir, config_file): assert os.path.exists(agent_dir) assert os.path.exists(config_file) wheel_path = packaging.create_package(agent_dir, self.packaged_dir) packaging.add_files_to_package( wheel_path, {'config_file': os.path.join('./', config_file)}) return wheel_path # def direct_build_agentpackage(self, agent_dir): # self.logit("Building agent_directory ", agent_dir) # wheel_path = packaging.create_package(os.path.join('./', agent_dir), # self.packaged_dir) # # return wheel_path # # def direct_send_agent(self, package, target): # pparams = [VCTRL, SEND_AGENT, target, package] # print (pparams, "CWD", os.getcwd()) # send_process = subprocess.call(pparams, env=self.env) # print ("Done sending to", target) # # def direct_configure_agentpackage(self, agent_wheel, config_file): # packaging.add_files_to_package(agent_wheel, { # 'config_file':os.path.join('./', config_file) # }) # # # def direct_build_install_agent(self, agent_dir, config_file): # agent_wheel = self.build_agentpackage(agent_dir=agent_dir, # config_file=config_file) # self.direct_configure_agentpackage(agent_wheel, config_file) # assert(agent_wheel is not None,"Agent wheel was not built") # # uuid = self.test_aip.install_agent(agent_wheel) # #aip volttron_home, verify_agents # return uuid # # conn.call.start_agent() # def direct_build_install_run_agent(self, agent_dir, config_file): # agent_uuid = self.direct_build_install_agent(agent_dir, config_file) # self.direct_start_agent(agent_uuid) # return agent_uuid # # def direct_build_send_agent(self, agent_dir, config_file, target): # agent_uuid = self.direct_buid_install_agent(agent_dir, config_file) # self.direct_start_agent(agent_uuid) # return agent_uuid def confirm_agent_running(self, agent_name, max_retries=5, timeout_seconds=2): running = False retries = 0 while not running and retries < max_retries: status = self.test_aip.status_agents() print("Status", status) if len(status) > 0: status_name = status[0][1] assert status_name == agent_name assert len( status[0][2]) == 2, 'Unexpected agent status message' status_agent_status = status[0][2][1] running = not isinstance(status_agent_status, int) retries += 1 time.sleep(timeout_seconds) return running # def direct_stop_agent(self, agent_uuid): # result = self.conn.call.stop_agent(agent_uuid) # print result def shutdown_platform(self): '''Stop platform here This function will shutdown the platform and attempt to kill any process that the platformwrapper has started. ''' import signal self.logit('shutting down platform: PIDS: {}'.format( self.started_agent_pids)) while self.started_agent_pids: pid = self.started_agent_pids.pop() self.logit('ending pid: {}'.format(pid)) try: os.kill(pid, signal.SIGTERM) except: self.logit('could not kill: {} '.format(pid)) if self.p_process is not None: try: gevent.sleep(0.2) self.p_process.terminate() gevent.sleep(0.2) except OSError: self.logit('Platform process was terminated.') else: self.logit("platform process was null") if self.use_twistd and self.t_process != None: self.t_process.kill() self.t_process.wait() elif self.use_twistd: self.logit("twistd process was null") if not self.skip_cleanup: self.logit('Removing {}'.format(self.volttron_home)) shutil.rmtree(self.volttron_home, ignore_errors=True)
def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, agent_class=Agent, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :param agent_class: Agent class to build :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if serverkey is None: serverkey = self.serverkey if publickey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret if address is None: self.logit('Using vip-address ' + self.vip_address) address = self.vip_address if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = agent_class(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event) # .join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) agent.publickey = publickey return agent
class PlatformWrapper: def __init__(self): """ Initializes a new VOLTTRON instance Creates a temporary VOLTTRON_HOME directory with a packaged directory for agents that are built. """ # This is hopefully going to keep us from attempting to shutdown # multiple times. For example if a fixture calls shutdown and a # lower level fixture calls shutdown, this won't hang. self._instance_shutdown = False self.volttron_home = tempfile.mkdtemp() self.packaged_dir = os.path.join(self.volttron_home, "packaged") os.makedirs(self.packaged_dir) # in the context of this platform it is very important not to # use the main os.environ for anything. self.env = { 'VOLTTRON_HOME': self.volttron_home, 'PACKAGED_DIR': self.packaged_dir, 'DEBUG_MODE': os.environ.get('DEBUG_MODE', ''), 'DEBUG': os.environ.get('DEBUG', ''), 'PATH': VOLTTRON_ROOT + ':' + os.environ['PATH'] } self.volttron_root = VOLTTRON_ROOT volttron_exe = subprocess.check_output(['which', 'volttron']).strip() assert os.path.exists(volttron_exe) self.python = os.path.join(os.path.dirname(volttron_exe), 'python') assert os.path.exists(self.python) # By default no web server should be started. self.bind_web_address = None self.discovery_address = None self.jsonrpc_endpoint = None self.volttron_central_address = None self.instance_name = None self.serverkey = None self.p_process = None self.t_process = None self.started_agent_pids = [] self.local_vip_address = None self.vip_address = None self.logit('Creating platform wrapper') # This was used when we are testing the SMAP historian. self.use_twistd = False # Added restricted code properties self.certsobj = None # Control whether the instance directory is cleaned up when shutdown. # if the environment variable DEBUG is set to a True value then the # instance is not cleaned up. self.skip_cleanup = False # This is used as command line entry replacement. Especially working # with older 2.0 agents. self.opts = None keystorefile = os.path.join(self.volttron_home, 'keystore') self.keystore = KeyStore(keystorefile) self.keystore.generate() def logit(self, message): print('{}: {}'.format(self.volttron_home, message)) def allow_all_connections(self): """ Add a /.*/ entry to the auth.json file. """ entry = AuthEntry(credentials="/.*/") authfile = AuthFile(self.volttron_home + "/auth.json") try: authfile.add(entry) except AuthFileEntryAlreadyExists: pass def build_connection(self, peer=None, address=None, identity=None, publickey=None, secretkey=None, serverkey=None, capabilities=[], **kwargs): self.logit('Building connection to {}'.format(peer)) self.allow_all_connections() if address is None: self.logit( 'Default address was None so setting to current instances') address = self.vip_address serverkey = self.serverkey if serverkey is None: self.logit("serverkey wasn't set but the address was.") raise Exception("Invalid state.") if publickey is None or secretkey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret entry = AuthEntry(capabilities=capabilities, comments="Added by test", credentials=keys.public) file = AuthFile(self.volttron_home + "/auth.json") file.add(entry) conn = Connection(address=address, peer=peer, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home) return conn def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, agent_class=Agent, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :param agent_class: Agent class to build :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if serverkey is None: serverkey = self.serverkey if publickey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret if address is None: self.logit('Using vip-address ' + self.vip_address) address = self.vip_address if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = agent_class(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event) # .join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) agent.publickey = publickey return agent def _read_auth_file(self): auth_path = os.path.join(self.volttron_home, 'auth.json') try: with open(auth_path, 'r') as fd: data = strip_comments(FileObject(fd, close=False).read()) if data: auth = jsonapi.loads(data) else: auth = {} except IOError: auth = {} if 'allow' not in auth: auth['allow'] = [] return auth, auth_path def _append_allow_curve_key(self, publickey): entry = AuthEntry(credentials=publickey) authfile = AuthFile(self.volttron_home + "/auth.json") try: authfile.add(entry) except AuthFileEntryAlreadyExists: pass def add_vc(self): return add_vc_to_instance(self) def add_capabilities(self, publickey, capabilities): if isinstance(capabilities, basestring): capabilities = [capabilities] auth, auth_path = self._read_auth_file() cred = publickey allow = auth['allow'] entry = next((item for item in allow if item['credentials'] == cred), {}) caps = entry.get('capabilities', []) entry['capabilities'] = list(set(caps + capabilities)) with open(auth_path, 'w+') as fd: json.dump(auth, fd) def set_auth_dict(self, auth_dict): if auth_dict: with open(os.path.join(self.volttron_home, 'auth.json'), 'w') as fd: fd.write(json.dumps(auth_dict)) def startup_platform(self, vip_address, auth_dict=None, use_twistd=False, mode=UNRESTRICTED, bind_web_address=None, volttron_central_address=None, volttron_central_serverkey=None): # if not isinstance(vip_address, list): # self.vip_address = [vip_address] # else: # self.vip_address = vip_address self.vip_address = vip_address self.mode = mode self.bind_web_address = bind_web_address if self.bind_web_address: self.discovery_address = "{}/discovery/".format( self.bind_web_address) # Only available if vc is installed! self.jsonrpc_endpoint = "{}/jsonrpc".format( self.bind_web_address) enable_logging = self.env.get('ENABLE_LOGGING', False) debug_mode = self.env.get('DEBUG_MODE', False) if not debug_mode: debug_mode = self.env.get('DEBUG', False) self.skip_cleanup = self.env.get('SKIP_CLEANUP', False) if debug_mode: self.skip_cleanup = True enable_logging = True self.logit( "In start up platform enable_logging is {} ".format(enable_logging)) assert self.mode in MODES, 'Invalid platform mode set: ' + str(mode) opts = None # see main.py for how we handle pub sub addresses. ipc = 'ipc://{}{}/run/'.format( '@' if sys.platform.startswith('linux') else '', self.volttron_home) self.local_vip_address = ipc + 'vip.socket' self.set_auth_dict(auth_dict) self.opts = {'verify_agents': False, 'volttron_home': self.volttron_home, 'vip_address': vip_address, 'vip_local_address': ipc + 'vip.socket', 'publish_address': ipc + 'publish', 'subscribe_address': ipc + 'subscribe', 'bind_web_address': bind_web_address, 'volttron_central_address': volttron_central_address, 'volttron_central_serverkey': volttron_central_serverkey, 'platform_name': None, 'log': os.path.join(self.volttron_home, 'volttron.log'), 'log_config': None, 'monitor': True, 'autostart': True, 'log_level': logging.DEBUG, 'verboseness': logging.DEBUG} pconfig = os.path.join(self.volttron_home, 'config') config = {} # Add platform's public key to known hosts file publickey = self.keystore.public known_hosts_file = os.path.join(self.volttron_home, 'known_hosts') known_hosts = KnownHostsStore(known_hosts_file) known_hosts.add(self.opts['vip_local_address'], publickey) known_hosts.add(self.opts['vip_address'], publickey) # Set up the configuration file based upon the passed parameters. parser = configparser.ConfigParser() parser.add_section('volttron') parser.set('volttron', 'vip-address', vip_address) if bind_web_address: parser.set('volttron', 'bind-web-address', bind_web_address) if volttron_central_address: parser.set('volttron', 'volttron-central-address', volttron_central_address) if volttron_central_serverkey: parser.set('volttron', 'volttron-central-serverkey', volttron_central_serverkey) if self.mode == UNRESTRICTED: # TODO Restricted code should set with volttron as contianer # if RESTRICTED_AVAILABLE: # config['mobility'] = False # config['resource-monitor'] = False # config['verify'] = False with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_UNRESTRICTED.format(**config)) parser.write(cfg) elif self.mode == RESTRICTED: if not RESTRICTED_AVAILABLE: raise ValueError("restricted is not available.") certsdir = os.path.join(self.volttron_home, 'certificates') print ("certsdir", certsdir) self.certsobj = certs.Certs(certsdir) with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_RESTRICTED.format(**config)) else: raise PlatformWrapperError( "Invalid platform mode specified: {}".format(mode)) log = os.path.join(self.volttron_home, 'volttron.log') if enable_logging: cmd = ['volttron', '-vv', '-l{}'.format(log)] else: cmd = ['volttron', '-l{}'.format(log)] print('process environment: {}'.format(self.env)) print('popen params: {}'.format(cmd)) self.p_process = Popen(cmd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert self.p_process is not None # A None value means that the process is still running. # A negative means that the process exited with an error. assert self.p_process.poll() is None self.serverkey = self.keystore.public assert self.serverkey agent = self.build_agent() has_control = False times = 0 while not has_control and times < 10: times += 1 try: has_control = agent.vip.peerlist().get(timeout=.2) except gevent.Timeout: pass if not has_control: self.shutdown_platform() raise "Couldn't connect to core platform!" if bind_web_address: times = 0 has_discovery = False while times < 10: times += 1 try: resp = requests.get(self.discovery_address) if resp.ok: has_discovery = True break except Exception as e: gevent.sleep(0.1) self.logit("Connection error found {}".format(e)) if not has_discovery: raise "Couldn't connect to discovery platform." self.use_twistd = use_twistd # TODO: Revise this to start twistd with platform. if self.use_twistd: tconfig = os.path.join(self.volttron_home, TMP_SMAP_CONFIG_FILENAME) with closing(open(tconfig, 'w')) as cfg: cfg.write(TWISTED_CONFIG.format(**config)) tparams = [TWISTED_START, "-n", "smap", tconfig] self.t_process = subprocess.Popen(tparams, env=self.env) time.sleep(5) def is_running(self): self.logit("PROCESS IS RUNNING: {}".format(self.p_process)) return self.p_process is not None and self.p_process.poll() is None def twistd_is_running(self): return self.t_process is not None def direct_sign_agentpackage_creator(self, package): assert (RESTRICTED), "Auth not available" print ("wrapper.certsobj", self.certsobj.cert_dir) assert ( auth.sign_as_creator(package, 'creator', certsobj=self.certsobj)), "Signing as {} failed.".format( 'creator') def direct_sign_agentpackage_admin(self, package): assert (RESTRICTED), "Auth not available" assert (auth.sign_as_admin(package, 'admin', certsobj=self.certsobj)), "Signing as {} failed.".format( 'admin') def direct_sign_agentpackage_initiator(self, package, config_file, contract): assert (RESTRICTED), "Auth not available" files = {"config_file": config_file, "contract": contract} assert (auth.sign_as_initiator(package, 'initiator', files=files, certsobj=self.certsobj)), "Signing as {} failed.".format( 'initiator') def _aip(self): opts = type('Options', (), self.opts) aip = AIPplatform(opts) aip.setup() return aip def _install_agent(self, wheel_file, start, vip_identity): self.logit('Creating channel for sending the agent.') gevent.sleep(0.3) self.logit('calling control install agent.') self.logit("VOLTTRON_HOME SETTING: {}".format( self.env['VOLTTRON_HOME'])) env = self.env.copy() cmd = ['volttron-ctl', '-vv', 'install', wheel_file] if vip_identity: cmd.extend(['--vip-identity', vip_identity]) self.logit("cmd: {}".format(cmd)) res = subprocess.check_output(cmd, env=env) assert res, "failed to install wheel:{}".format(wheel_file) agent_uuid = res.split(' ')[-2] self.logit(agent_uuid) if start: self.start_agent(agent_uuid) return agent_uuid def install_multiple_agents(self, agent_configs): """ Installs mutltiple agents on the platform. :param agent_configs:list A list of 3-tuple that allows the configuration of a platform in a single go. The tuple order is 1. path to the agent directory. 2. configuration data (either file or json data) 3. Whether the agent should be started or not. :return:list: A list of uuid's associated with the agents that were installed. :Note: In order for this method to be called the platform must be currently running. """ if not self.is_running(): raise PlatformWrapperError("Instance isn't running!") results = [] for path, config, start in agent_configs: results = self.install_agent(agent_dir=path, config_file=config, start=start) return results def install_agent(self, agent_wheel=None, agent_dir=None, config_file=None, start=True, vip_identity=None): """ Install and optionally start an agent on the instance. This function allows installation from an agent wheel or an agent directory (NOT BOTH). If an agent_wheel is specified then it is assumed to be ready for installation (has a config file). If an agent_dir is specified then a config_file file must be specified or if it is not specified then it is assumed that the file agent_dir/config is to be used as the configuration file. If none of these exist then an assertion error will be thrown. This function will return with a uuid of the installed agent. :param agent_wheel: :param agent_dir: :param config_file: :param start: :param vip_identity: :return: """ assert self.is_running(), "Instance must be running to install agent." assert agent_wheel or agent_dir, "Invalid agent_wheel or agent_dir." if agent_wheel: assert not agent_dir assert not config_file assert os.path.exists(agent_wheel) wheel_file = agent_wheel agent_uuid = self._install_agent(wheel_file, start, vip_identity) # Now if the agent_dir is specified. if agent_dir: assert not agent_wheel if isinstance(config_file, dict): from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps(config_file)) config_file = temp_config elif not config_file: if os.path.exists(os.path.join(agent_dir, "config")): config_file = os.path.join(agent_dir, "config") else: from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps({})) config_file = temp_config elif os.path.exists(config_file): pass # config_file already set! else: raise ValueError("Can't determine correct config file.") script = os.path.join(self.volttron_root, "scripts/install-agent.py") cmd = [self.python, script, "--volttron-home", self.volttron_home, "--volttron-root", self.volttron_root, "--agent-source", agent_dir, "--config", config_file, "--json"] if vip_identity: cmd.extend(["--vip-identity", vip_identity]) if start: cmd.extend(["--start"]) results = subprocess.check_output(cmd) # Because we are no longer silencing output from the install, the # the results object is now much more verbose. Our assumption is # the line before the output we care about has WHEEL at the end # of it. new_results = "" found_wheel = False for line in results.split("\n"): if line.endswith("WHEEL"): found_wheel = True elif found_wheel: new_results += line results = new_results # # Response from results is expected as follows depending on # parameters, note this is a json string so parse to get dictionary. # { # "started": true, # "agent_pid": 26241, # "starting": true, # "agent_uuid": "ec1fd94e-922a-491f-9878-c392b24dbe50" # } assert results resultobj = jsonapi.loads(str(results)) if start: assert resultobj['started'] agent_uuid = resultobj['agent_uuid'] assert agent_uuid is not None if start: assert self.is_agent_running(agent_uuid) return agent_uuid def start_agent(self, agent_uuid): self.logit('Starting agent {}'.format(agent_uuid)) self.logit("VOLTTRON_HOME SETTING: {}".format( self.env['VOLTTRON_HOME'])) cmd = ['volttron-ctl'] cmd.extend(['start', agent_uuid]) p = Popen(cmd, env=self.env, stdout=sys.stdout, stderr=sys.stderr) p.wait() # Confirm agent running cmd = ['volttron-ctl'] cmd.extend(['status', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) # 776 TODO: Timing issue where check fails time.sleep(.1) self.logit("Subprocess res is {}".format(res)) assert 'running' in res pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos: pidend]) assert psutil.pid_exists(pid), \ "The pid associated with agent {} does not exist".format(pid) self.started_agent_pids.append(pid) return pid def stop_agent(self, agent_uuid): # Confirm agent running _log.debug("STOPPING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl'] cmd.extend(['stop', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_pid(agent_uuid) def list_agents(self): agent = self.build_agent() print('PEER LIST: {}'.format(agent.vip.peerlist().get(timeout=10))) agent_list = agent.vip.rpc('control', 'list_agents').get(timeout=10) agent.core.stop(timeout=3) return agent_list def remove_agent(self, agent_uuid): """Remove the agent specified by agent_uuid""" _log.debug("REMOVING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl'] cmd.extend(['remove', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_pid(agent_uuid) def is_agent_running(self, agent_uuid): return self.agent_pid(agent_uuid) is not None def agent_pid(self, agent_uuid): """ Returns the pid of a running agent or None :param agent_uuid: :return: """ # Confirm agent running cmd = ['volttron-ctl'] cmd.extend(['status', agent_uuid]) pid = None try: res = subprocess.check_output(cmd, env=self.env) try: pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos: pidend]) except: pid = None except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) # Handle the following exception that seems to happen when getting a # pid of an agent during the platform shutdown phase. # # Logged from file platformwrapper.py, line 797 # AGENT IDENTITY TAG STATUS # Traceback (most recent call last): # File "/usr/lib/python2.7/logging/__init__.py", line 882, in emit # stream.write(fs % msg) # File "/home/volttron/git/volttron/env/local/lib/python2.7/site-packages/_pytest/capture.py", line 244, in write # self.buffer.write(obj) # ValueError: I/O operation on closed file except ValueError: pass # _log.debug("AGENT_PID: {}".format(pid)) return pid def build_agentpackage(self, agent_dir, config_file={}): if isinstance(config_file, dict): cfg_path = os.path.join(agent_dir, "config_temp") with open(cfg_path, "w") as tmp_cfg: tmp_cfg.write(jsonapi.dumps(config_file)) config_file = cfg_path # Handle relative paths from the volttron git directory. if not os.path.isabs(agent_dir): agent_dir = os.path.join(self.volttron_root, agent_dir) assert os.path.exists(config_file) assert os.path.exists(agent_dir) wheel_path = packaging.create_package(agent_dir, self.packaged_dir) packaging.add_files_to_package(wheel_path, { 'config_file': os.path.join('./', config_file) }) return wheel_path def confirm_agent_running(self, agent_name, max_retries=5, timeout_seconds=2): running = False retries = 0 while not running and retries < max_retries: status = self.test_aip.status_agents() print ("Status", status) if len(status) > 0: status_name = status[0][1] assert status_name == agent_name assert len(status[0][2]) == 2, 'Unexpected agent status message' status_agent_status = status[0][2][1] running = not isinstance(status_agent_status, int) retries += 1 time.sleep(timeout_seconds) return running # def direct_stop_agent(self, agent_uuid): # result = self.conn.call.stop_agent(agent_uuid) # print result def shutdown_platform(self): """ Stop platform here. First grab a list of all of the agents that are running on the platform, then shutdown, then if any of the listed agent pids are still running then kill them. """ # Handle cascading calls from multiple levels of fixtures. if self._instance_shutdown: return running_pids = [] for agnt in self.list_agents(): pid = self.agent_pid(agnt['uuid']) if pid is not None and int(pid) > 0: running_pids.append(int(pid)) # First try and nicely shutdown the platform, which should clean all # of the agents up automatically. cmd = ['volttron-ctl'] cmd.extend(['shutdown', '--platform']) try: res = subprocess.check_output(cmd, env=self.env) except CalledProcessError: if self.p_process is not None: try: gevent.sleep(0.2) self.p_process.terminate() gevent.sleep(0.2) except OSError: self.logit('Platform process was terminated.') else: self.logit("platform process was null") for pid in running_pids: if psutil.pid_exists(pid): self.logit("TERMINATING: {}".format(pid)) proc = psutil.Process(pid) proc.terminate() if self.use_twistd and self.t_process is not None: self.t_process.kill() self.t_process.wait() elif self.use_twistd: self.logit("twistd process was null") if os.environ.get('PRINT_LOG'): logpath = os.path.join(self.volttron_home, 'volttron.log') if os.path.exists(logpath): print("************************* Begin {}".format(logpath)) with open(logpath) as f: for l in f.readlines(): print(l) print("************************* End {}".format(logpath)) else: print("######################### No Log Exists: {}".format( logpath )) if not self.skip_cleanup: self.logit('Removing {}'.format(self.volttron_home)) shutil.rmtree(self.volttron_home, ignore_errors=True) self._instance_shutdown = True def __repr__(self): return str(self) def __str__(self): data = [] data.append('volttron_home: {}'.format(self.volttron_home)) return '\n'.join(data)
def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, generatekeys=False, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if address is None: if use_ipc: self.logit('Using IPC vip-address') address = "ipc://@" + self.volttron_home + "/run/vip.socket" else: self.logit('Using vip-address ' + self.vip_address) address = self.vip_address if generatekeys: self.logit('generating new public secret key pair') tf = tempfile.NamedTemporaryFile() ks = KeyStore(tf.name) ks.generate() publickey = ks.public() secretkey = ks.secret() if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = Agent(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event) #.join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) return agent
class PlatformWrapper: def __init__(self): """ Initializes a new VOLTTRON instance Creates a temporary VOLTTRON_HOME directory with a packaged directory for agents that are built. """ # This is hopefully going to keep us from attempting to shutdown # multiple times. For example if a fixture calls shutdown and a # lower level fixture calls shutdown, this won't hang. self._instance_shutdown = False self.volttron_home = tempfile.mkdtemp() self.packaged_dir = os.path.join(self.volttron_home, "packaged") os.makedirs(self.packaged_dir) # in the context of this platform it is very important not to # use the main os.environ for anything. self.env = { 'VOLTTRON_HOME': self.volttron_home, 'PACKAGED_DIR': self.packaged_dir, 'DEBUG_MODE': os.environ.get('DEBUG_MODE', ''), 'DEBUG': os.environ.get('DEBUG', ''), 'PATH': VOLTTRON_ROOT + ':' + os.environ['PATH'] } self.volttron_root = VOLTTRON_ROOT volttron_exe = subprocess.check_output(['which', 'volttron']).strip() assert os.path.exists(volttron_exe) self.python = os.path.join(os.path.dirname(volttron_exe), 'python') assert os.path.exists(self.python) # By default no web server should be started. self.bind_web_address = None self.discovery_address = None self.jsonrpc_endpoint = None self.volttron_central_address = None self.instance_name = None self.serverkey = None self.p_process = None self.t_process = None self.started_agent_pids = [] self.local_vip_address = None self.vip_address = None self.logit('Creating platform wrapper') # This was used when we are testing the SMAP historian. self.use_twistd = False # Added restricted code properties self.certsobj = None # Control whether the instance directory is cleaned up when shutdown. # if the environment variable DEBUG is set to a True value then the # instance is not cleaned up. self.skip_cleanup = False # This is used as command line entry replacement. Especially working # with older 2.0 agents. self.opts = None keystorefile = os.path.join(self.volttron_home, 'keystore') self.keystore = KeyStore(keystorefile) self.keystore.generate() def logit(self, message): print('{}: {}'.format(self.volttron_home, message)) def allow_all_connections(self): """ Add a /.*/ entry to the auth.json file. """ entry = AuthEntry(credentials="/.*/") authfile = AuthFile(self.volttron_home + "/auth.json") try: authfile.add(entry) except AuthFileEntryAlreadyExists: pass def build_connection(self, peer=None, address=None, identity=None, publickey=None, secretkey=None, serverkey=None, capabilities=[], **kwargs): self.logit('Building connection to {}'.format(peer)) self.allow_all_connections() if address is None: self.logit( 'Default address was None so setting to current instances') address = self.vip_address serverkey = self.serverkey if serverkey is None: self.logit("serverkey wasn't set but the address was.") raise Exception("Invalid state.") if publickey is None or secretkey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret entry = AuthEntry(capabilities=capabilities, comments="Added by test", credentials=keys.public) file = AuthFile(self.volttron_home + "/auth.json") file.add(entry) conn = Connection(address=address, peer=peer, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home) return conn def build_agent(self, address=None, should_spawn=True, identity=None, publickey=None, secretkey=None, serverkey=None, agent_class=Agent, **kwargs): """ Build an agent connnected to the passed bus. By default the current instance that this class wraps will be the vip address of the agent. :param address: :param should_spawn: :param identity: :param publickey: :param secretkey: :param serverkey: :param agent_class: Agent class to build :return: """ self.logit("Building generic agent.") use_ipc = kwargs.pop('use_ipc', False) if serverkey is None: serverkey = self.serverkey if publickey is None: self.logit('generating new public secret key pair') keyfile = tempfile.mktemp(".keys", "agent", self.volttron_home) keys = KeyStore(keyfile) keys.generate() publickey = keys.public secretkey = keys.secret if address is None: self.logit('Using vip-address ' + self.vip_address) address = self.vip_address if publickey and not serverkey: self.logit('using instance serverkey: {}'.format(self.publickey)) serverkey = self.publickey agent = agent_class(address=address, identity=identity, publickey=publickey, secretkey=secretkey, serverkey=serverkey, volttron_home=self.volttron_home, **kwargs) self.logit('platformwrapper.build_agent.address: {}'.format(address)) # Automatically add agent's credentials to auth.json file if publickey: self.logit('Adding publickey to auth.json') gevent.spawn(self._append_allow_curve_key, publickey) gevent.sleep(0.1) if should_spawn: self.logit('platformwrapper.build_agent spawning') event = gevent.event.Event() gevent.spawn(agent.core.run, event) # .join(0) event.wait(timeout=2) hello = agent.vip.hello().get(timeout=.3) self.logit('Got hello response {}'.format(hello)) agent.publickey = publickey return agent def _read_auth_file(self): auth_path = os.path.join(self.volttron_home, 'auth.json') try: with open(auth_path, 'r') as fd: data = strip_comments(FileObject(fd, close=False).read()) if data: auth = jsonapi.loads(data) else: auth = {} except IOError: auth = {} if 'allow' not in auth: auth['allow'] = [] return auth, auth_path def _append_allow_curve_key(self, publickey): entry = AuthEntry(credentials=publickey) authfile = AuthFile(self.volttron_home + "/auth.json") try: authfile.add(entry) except AuthFileEntryAlreadyExists: pass def add_vc(self): return add_vc_to_instance(self) def add_capabilities(self, publickey, capabilities): if isinstance(capabilities, basestring): capabilities = [capabilities] auth, auth_path = self._read_auth_file() cred = publickey allow = auth['allow'] entry = next((item for item in allow if item['credentials'] == cred), {}) caps = entry.get('capabilities', []) entry['capabilities'] = list(set(caps + capabilities)) with open(auth_path, 'w+') as fd: json.dump(auth, fd) def set_auth_dict(self, auth_dict): if auth_dict: with open(os.path.join(self.volttron_home, 'auth.json'), 'w') as fd: fd.write(json.dumps(auth_dict)) def startup_platform(self, vip_address, auth_dict=None, use_twistd=False, mode=UNRESTRICTED, bind_web_address=None, volttron_central_address=None, volttron_central_serverkey=None): # if not isinstance(vip_address, list): # self.vip_address = [vip_address] # else: # self.vip_address = vip_address self.vip_address = vip_address self.mode = mode self.bind_web_address = bind_web_address if self.bind_web_address: self.discovery_address = "{}/discovery/".format( self.bind_web_address) # Only available if vc is installed! self.jsonrpc_endpoint = "{}/jsonrpc".format(self.bind_web_address) enable_logging = self.env.get('ENABLE_LOGGING', False) debug_mode = self.env.get('DEBUG_MODE', False) if not debug_mode: debug_mode = self.env.get('DEBUG', False) self.skip_cleanup = self.env.get('SKIP_CLEANUP', False) if debug_mode: self.skip_cleanup = True enable_logging = True self.logit("In start up platform enable_logging is {} ".format( enable_logging)) assert self.mode in MODES, 'Invalid platform mode set: ' + str(mode) opts = None # see main.py for how we handle pub sub addresses. ipc = 'ipc://{}{}/run/'.format( '@' if sys.platform.startswith('linux') else '', self.volttron_home) self.local_vip_address = ipc + 'vip.socket' self.set_auth_dict(auth_dict) self.opts = { 'verify_agents': False, 'volttron_home': self.volttron_home, 'vip_address': vip_address, 'vip_local_address': ipc + 'vip.socket', 'publish_address': ipc + 'publish', 'subscribe_address': ipc + 'subscribe', 'bind_web_address': bind_web_address, 'volttron_central_address': volttron_central_address, 'volttron_central_serverkey': volttron_central_serverkey, 'platform_name': None, 'log': os.path.join(self.volttron_home, 'volttron.log'), 'log_config': None, 'monitor': True, 'autostart': True, 'log_level': logging.DEBUG, 'verboseness': logging.DEBUG } pconfig = os.path.join(self.volttron_home, 'config') config = {} # Add platform's public key to known hosts file publickey = self.keystore.public known_hosts_file = os.path.join(self.volttron_home, 'known_hosts') known_hosts = KnownHostsStore(known_hosts_file) known_hosts.add(self.opts['vip_local_address'], publickey) known_hosts.add(self.opts['vip_address'], publickey) # Set up the configuration file based upon the passed parameters. parser = configparser.ConfigParser() parser.add_section('volttron') parser.set('volttron', 'vip-address', vip_address) if bind_web_address: parser.set('volttron', 'bind-web-address', bind_web_address) if volttron_central_address: parser.set('volttron', 'volttron-central-address', volttron_central_address) if volttron_central_serverkey: parser.set('volttron', 'volttron-central-serverkey', volttron_central_serverkey) if self.mode == UNRESTRICTED: # TODO Restricted code should set with volttron as contianer # if RESTRICTED_AVAILABLE: # config['mobility'] = False # config['resource-monitor'] = False # config['verify'] = False with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_UNRESTRICTED.format(**config)) parser.write(cfg) elif self.mode == RESTRICTED: if not RESTRICTED_AVAILABLE: raise ValueError("restricted is not available.") certsdir = os.path.join(self.volttron_home, 'certificates') print("certsdir", certsdir) self.certsobj = certs.Certs(certsdir) with closing(open(pconfig, 'wb')) as cfg: cfg.write(PLATFORM_CONFIG_RESTRICTED.format(**config)) else: raise PlatformWrapperError( "Invalid platform mode specified: {}".format(mode)) log = os.path.join(self.volttron_home, 'volttron.log') if enable_logging: cmd = ['volttron', '-vv', '-l{}'.format(log)] else: cmd = ['volttron', '-l{}'.format(log)] print('process environment: {}'.format(self.env)) print('popen params: {}'.format(cmd)) self.p_process = Popen(cmd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert self.p_process is not None # A None value means that the process is still running. # A negative means that the process exited with an error. assert self.p_process.poll() is None self.serverkey = self.keystore.public assert self.serverkey agent = self.build_agent() has_control = False times = 0 while not has_control and times < 10: times += 1 try: has_control = agent.vip.peerlist().get(timeout=.2) except gevent.Timeout: pass if not has_control: self.shutdown_platform() raise "Couldn't connect to core platform!" if bind_web_address: times = 0 has_discovery = False while times < 10: times += 1 try: resp = requests.get(self.discovery_address) if resp.ok: has_discovery = True break except Exception as e: gevent.sleep(0.1) self.logit("Connection error found {}".format(e)) if not has_discovery: raise "Couldn't connect to discovery platform." self.use_twistd = use_twistd # TODO: Revise this to start twistd with platform. if self.use_twistd: tconfig = os.path.join(self.volttron_home, TMP_SMAP_CONFIG_FILENAME) with closing(open(tconfig, 'w')) as cfg: cfg.write(TWISTED_CONFIG.format(**config)) tparams = [TWISTED_START, "-n", "smap", tconfig] self.t_process = subprocess.Popen(tparams, env=self.env) time.sleep(5) def is_running(self): self.logit("PROCESS IS RUNNING: {}".format(self.p_process)) return self.p_process is not None and self.p_process.poll() is None def twistd_is_running(self): return self.t_process is not None def direct_sign_agentpackage_creator(self, package): assert (RESTRICTED), "Auth not available" print("wrapper.certsobj", self.certsobj.cert_dir) assert (auth.sign_as_creator( package, 'creator', certsobj=self.certsobj)), "Signing as {} failed.".format('creator') def direct_sign_agentpackage_admin(self, package): assert (RESTRICTED), "Auth not available" assert (auth.sign_as_admin( package, 'admin', certsobj=self.certsobj)), "Signing as {} failed.".format('admin') def direct_sign_agentpackage_initiator(self, package, config_file, contract): assert (RESTRICTED), "Auth not available" files = {"config_file": config_file, "contract": contract} assert (auth.sign_as_initiator(package, 'initiator', files=files, certsobj=self.certsobj) ), "Signing as {} failed.".format('initiator') def _aip(self): opts = type('Options', (), self.opts) aip = AIPplatform(opts) aip.setup() return aip def _install_agent(self, wheel_file, start, vip_identity): self.logit('Creating channel for sending the agent.') gevent.sleep(0.3) self.logit('calling control install agent.') self.logit("VOLTTRON_HOME SETTING: {}".format( self.env['VOLTTRON_HOME'])) env = self.env.copy() cmd = ['volttron-ctl', '-vv', 'install', wheel_file] if vip_identity: cmd.extend(['--vip-identity', vip_identity]) self.logit("cmd: {}".format(cmd)) res = subprocess.check_output(cmd, env=env) assert res, "failed to install wheel:{}".format(wheel_file) agent_uuid = res.split(' ')[-2] self.logit(agent_uuid) if start: self.start_agent(agent_uuid) return agent_uuid def install_multiple_agents(self, agent_configs): """ Installs mutltiple agents on the platform. :param agent_configs:list A list of 3-tuple that allows the configuration of a platform in a single go. The tuple order is 1. path to the agent directory. 2. configuration data (either file or json data) 3. Whether the agent should be started or not. :return:list: A list of uuid's associated with the agents that were installed. :Note: In order for this method to be called the platform must be currently running. """ if not self.is_running(): raise PlatformWrapperError("Instance isn't running!") results = [] for path, config, start in agent_configs: results = self.install_agent(agent_dir=path, config_file=config, start=start) return results def install_agent(self, agent_wheel=None, agent_dir=None, config_file=None, start=True, vip_identity=None): """ Install and optionally start an agent on the instance. This function allows installation from an agent wheel or an agent directory (NOT BOTH). If an agent_wheel is specified then it is assumed to be ready for installation (has a config file). If an agent_dir is specified then a config_file file must be specified or if it is not specified then it is assumed that the file agent_dir/config is to be used as the configuration file. If none of these exist then an assertion error will be thrown. This function will return with a uuid of the installed agent. :param agent_wheel: :param agent_dir: :param config_file: :param start: :param vip_identity: :return: """ assert self.is_running(), "Instance must be running to install agent." assert agent_wheel or agent_dir, "Invalid agent_wheel or agent_dir." if agent_wheel: assert not agent_dir assert not config_file assert os.path.exists(agent_wheel) wheel_file = agent_wheel agent_uuid = self._install_agent(wheel_file, start, vip_identity) # Now if the agent_dir is specified. if agent_dir: assert not agent_wheel if isinstance(config_file, dict): from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps(config_file)) config_file = temp_config elif not config_file: if os.path.exists(os.path.join(agent_dir, "config")): config_file = os.path.join(agent_dir, "config") else: from os.path import join, basename temp_config = join(self.volttron_home, basename(agent_dir) + "_config_file") with open(temp_config, "w") as fp: fp.write(json.dumps({})) config_file = temp_config elif os.path.exists(config_file): pass # config_file already set! else: raise ValueError("Can't determine correct config file.") script = os.path.join(self.volttron_root, "scripts/install-agent.py") cmd = [ self.python, script, "--volttron-home", self.volttron_home, "--volttron-root", self.volttron_root, "--agent-source", agent_dir, "--config", config_file, "--json" ] if vip_identity: cmd.extend(["--vip-identity", vip_identity]) if start: cmd.extend(["--start"]) results = subprocess.check_output(cmd) # Because we are no longer silencing output from the install, the # the results object is now much more verbose. Our assumption is # the line before the output we care about has WHEEL at the end # of it. new_results = "" found_wheel = False for line in results.split("\n"): if line.endswith("WHEEL"): found_wheel = True elif found_wheel: new_results += line results = new_results # # Response from results is expected as follows depending on # parameters, note this is a json string so parse to get dictionary. # { # "started": true, # "agent_pid": 26241, # "starting": true, # "agent_uuid": "ec1fd94e-922a-491f-9878-c392b24dbe50" # } assert results resultobj = jsonapi.loads(str(results)) if start: assert resultobj['started'] agent_uuid = resultobj['agent_uuid'] assert agent_uuid is not None if start: assert self.is_agent_running(agent_uuid) return agent_uuid def start_agent(self, agent_uuid): self.logit('Starting agent {}'.format(agent_uuid)) self.logit("VOLTTRON_HOME SETTING: {}".format( self.env['VOLTTRON_HOME'])) cmd = ['volttron-ctl'] cmd.extend(['start', agent_uuid]) p = Popen(cmd, env=self.env, stdout=sys.stdout, stderr=sys.stderr) p.wait() # Confirm agent running cmd = ['volttron-ctl'] cmd.extend(['status', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) # 776 TODO: Timing issue where check fails time.sleep(.1) self.logit("Subprocess res is {}".format(res)) assert 'running' in res pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos:pidend]) assert psutil.pid_exists(pid), \ "The pid associated with agent {} does not exist".format(pid) self.started_agent_pids.append(pid) return pid def stop_agent(self, agent_uuid): # Confirm agent running _log.debug("STOPPING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl'] cmd.extend(['stop', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_pid(agent_uuid) def list_agents(self): agent = self.build_agent() print('PEER LIST: {}'.format(agent.vip.peerlist().get(timeout=10))) agent_list = agent.vip.rpc('control', 'list_agents').get(timeout=10) agent.core.stop(timeout=3) return agent_list def remove_agent(self, agent_uuid): """Remove the agent specified by agent_uuid""" _log.debug("REMOVING AGENT: {}".format(agent_uuid)) try: cmd = ['volttron-ctl'] cmd.extend(['remove', agent_uuid]) res = subprocess.check_output(cmd, env=self.env) except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) return self.agent_pid(agent_uuid) def is_agent_running(self, agent_uuid): return self.agent_pid(agent_uuid) is not None def agent_pid(self, agent_uuid): """ Returns the pid of a running agent or None :param agent_uuid: :return: """ # Confirm agent running cmd = ['volttron-ctl'] cmd.extend(['status', agent_uuid]) pid = None try: res = subprocess.check_output(cmd, env=self.env) try: pidpos = res.index('[') + 1 pidend = res.index(']') pid = int(res[pidpos:pidend]) except: pid = None except CalledProcessError as ex: _log.error("Exception: {}".format(ex)) # Handle the following exception that seems to happen when getting a # pid of an agent during the platform shutdown phase. # # Logged from file platformwrapper.py, line 797 # AGENT IDENTITY TAG STATUS # Traceback (most recent call last): # File "/usr/lib/python2.7/logging/__init__.py", line 882, in emit # stream.write(fs % msg) # File "/home/volttron/git/volttron/env/local/lib/python2.7/site-packages/_pytest/capture.py", line 244, in write # self.buffer.write(obj) # ValueError: I/O operation on closed file except ValueError: pass # _log.debug("AGENT_PID: {}".format(pid)) return pid def build_agentpackage(self, agent_dir, config_file={}): if isinstance(config_file, dict): cfg_path = os.path.join(agent_dir, "config_temp") with open(cfg_path, "w") as tmp_cfg: tmp_cfg.write(jsonapi.dumps(config_file)) config_file = cfg_path # Handle relative paths from the volttron git directory. if not os.path.isabs(agent_dir): agent_dir = os.path.join(self.volttron_root, agent_dir) assert os.path.exists(config_file) assert os.path.exists(agent_dir) wheel_path = packaging.create_package(agent_dir, self.packaged_dir) packaging.add_files_to_package( wheel_path, {'config_file': os.path.join('./', config_file)}) return wheel_path def confirm_agent_running(self, agent_name, max_retries=5, timeout_seconds=2): running = False retries = 0 while not running and retries < max_retries: status = self.test_aip.status_agents() print("Status", status) if len(status) > 0: status_name = status[0][1] assert status_name == agent_name assert len( status[0][2]) == 2, 'Unexpected agent status message' status_agent_status = status[0][2][1] running = not isinstance(status_agent_status, int) retries += 1 time.sleep(timeout_seconds) return running # def direct_stop_agent(self, agent_uuid): # result = self.conn.call.stop_agent(agent_uuid) # print result def shutdown_platform(self): """ Stop platform here. First grab a list of all of the agents that are running on the platform, then shutdown, then if any of the listed agent pids are still running then kill them. """ # Handle cascading calls from multiple levels of fixtures. if self._instance_shutdown: return running_pids = [] for agnt in self.list_agents(): pid = self.agent_pid(agnt['uuid']) if pid is not None and int(pid) > 0: running_pids.append(int(pid)) # First try and nicely shutdown the platform, which should clean all # of the agents up automatically. cmd = ['volttron-ctl'] cmd.extend(['shutdown', '--platform']) try: res = subprocess.check_output(cmd, env=self.env) except CalledProcessError: if self.p_process is not None: try: gevent.sleep(0.2) self.p_process.terminate() gevent.sleep(0.2) except OSError: self.logit('Platform process was terminated.') else: self.logit("platform process was null") for pid in running_pids: if psutil.pid_exists(pid): self.logit("TERMINATING: {}".format(pid)) proc = psutil.Process(pid) proc.terminate() if self.use_twistd and self.t_process is not None: self.t_process.kill() self.t_process.wait() elif self.use_twistd: self.logit("twistd process was null") if os.environ.get('PRINT_LOG'): logpath = os.path.join(self.volttron_home, 'volttron.log') if os.path.exists(logpath): print("************************* Begin {}".format(logpath)) with open(logpath) as f: for l in f.readlines(): print(l) print("************************* End {}".format(logpath)) else: print("######################### No Log Exists: {}".format( logpath)) if not self.skip_cleanup: self.logit('Removing {}'.format(self.volttron_home)) shutil.rmtree(self.volttron_home, ignore_errors=True) self._instance_shutdown = True def __repr__(self): return str(self) def __str__(self): data = [] data.append('volttron_home: {}'.format(self.volttron_home)) return '\n'.join(data)