def __deregister_flex_vm(self, ip, username, keyfile, parameters, queue_head_ip, force=False): try: if self.check_network_ports(ip, [22]): deregister_command = self.get_remote_command_string( ip=ip, username=username, keyfile=keyfile, command= "sudo ~/stochss/release-tools/flex-cloud/deregister_flex_vm.sh" ) logging.debug( 'deregister_command =\n{}'.format(deregister_command)) os.system(deregister_command) else: logging.debug( 'Flex VM is not accessible via SSH, can not execute deregister command' ) except Exception as e: logging.exception('Failed to deregister Flex VM: {0}'.format(e)) # logging.error(sys.exc_info()) finally: VMStateModel.set_state( params=parameters, ins_ids=[self.get_flex_instance_id(public_ip=ip)], state=VMStateModel.STATE_TERMINATED, description='VM Deregistered.')
def __configure_celery(self, params, public_ips, instance_ids): """ Private method used for uploading the current celery configuration to each instance that is running and ssh connectable. Args parameters A dictionary of parameters public_ips A list of public ips that are going to be configed instance_ids A list of instance_ids that are used for terminating instances and update database if fail on configuration by some reason """ # Update celery config file...it should have the correct IP # of the Queue head node, which should already be running. # Pass it line by line so theres no weird formatting errors from # trying to echo a multi-line file directly on the command line key_file = os.path.join(os.path.dirname(__file__), "..", "{0}.key".format(params["keyname"])) logging.debug("key_file = {0}".format(key_file)) if not os.path.exists(key_file): raise Exception("ssh key_file file not found: {0}".format(key_file)) credentials = params["credentials"] commands = [] commands.append("source /home/ubuntu/.bashrc") commands.append("export AWS_ACCESS_KEY_ID={0}".format(str(credentials["EC2_ACCESS_KEY"]))) commands.append("export AWS_SECRET_ACCESS_KEY={0}".format(str(credentials["EC2_SECRET_KEY"]))) for ip, ins_id in zip(public_ips, instance_ids): # helper.wait_for_ssh_connection(key_file, ip) ins_type = VMStateModel.get_instance_type(params, ins_id) commands.append("export INSTANCE_TYPE={0}".format(ins_type)) success = helper.start_celery_on_vm( instance_type=ins_type, ip=ip, key_file=key_file, agent_type=self.agent_type, worker_name=ip.replace(".", "_"), prepend_commands=commands, ) if success == 0: # update db with successful running vms logging.info("celery started! ") logging.info("host ip: {0}".format(ip)) VMStateModel.set_state(params, [ins_id], VMStateModel.STATE_RUNNING, VMStateModel.DESCRI_SUCCESS) else: self.agent.deregister_some_instances(params, [ins_id]) VMStateModel.set_state( params, [ins_id], VMStateModel.STATE_FAILED, VMStateModel.DESCRI_FAIL_TO_COFIGURE_CELERY ) raise Exception("Failure to start celery on {0}".format(ip)) # get all intstance types and configure the celeryconfig.py locally instance_types = VMStateModel.get_running_instance_types(params) helper.config_celery_queues(agent_type=self.agent_type, instance_types=instance_types)
def __verify_ec2_instances_via_ssh(self, instance_ids, parameters, public_ips): keyfile = os.path.join(os.path.dirname(__file__), '..', '{0}.key'.format(parameters['keyname'])) logging.info('keyfile = {0}'.format(keyfile)) if not os.path.exists(keyfile): raise Exception("ssh keyfile file not found: {0}".format(keyfile)) connected_public_ips = [] connected_instance_ids = [] for (pub_ip, ins_id) in zip(public_ips, instance_ids): logging.info('connecting to {0}...'.format(pub_ip)) success = helper.wait_for_ssh_connection(key_file=keyfile, ip=pub_ip) if success == True: logging.info('{0} is successfully added'.format(pub_ip)) connected_public_ips.append(pub_ip) connected_instance_ids.append(ins_id) # if there are some vms not able to be connected via ssh, # just shut them down explicitly if len(public_ips) != len(connected_public_ips): logging.info( 'Time out on ssh to {0} instances. They will be terminated.'. format(len(public_ips) - len(connected_public_ips))) try: terminate_ins_ids = [] for ins_id in instance_ids: if ins_id not in connected_instance_ids: terminate_ins_ids.append(ins_id) self.agent.deregister_some_instances(parameters, terminate_ins_ids) # update db with failed vms VMStateModel.set_state(parameters, terminate_ins_ids, VMStateModel.STATE_FAILED, VMStateModel.DESCRI_TIMEOUT_ON_SSH) except: raise Exception( "Errors in terminating instances that cannot be connected via ssh." ) public_ips = None instance_ids = None return connected_public_ips, connected_instance_ids
def __verify_ec2_instances_via_ssh(self, instance_ids, parameters, public_ips): keyfile = os.path.join(os.path.dirname(__file__), "..", "{0}.key".format(parameters["keyname"])) logging.info("keyfile = {0}".format(keyfile)) if not os.path.exists(keyfile): raise Exception("ssh keyfile file not found: {0}".format(keyfile)) connected_public_ips = [] connected_instance_ids = [] for (pub_ip, ins_id) in zip(public_ips, instance_ids): logging.info("connecting to {0}...".format(pub_ip)) success = helper.wait_for_ssh_connection(key_file=keyfile, ip=pub_ip) if success == True: logging.info("{0} is successfully added".format(pub_ip)) connected_public_ips.append(pub_ip) connected_instance_ids.append(ins_id) # if there are some vms not able to be connected via ssh, # just shut them down explicitly if len(public_ips) != len(connected_public_ips): logging.info( "Time out on ssh to {0} instances. They will be terminated.".format( len(public_ips) - len(connected_public_ips) ) ) try: terminate_ins_ids = [] for ins_id in instance_ids: if ins_id not in connected_instance_ids: terminate_ins_ids.append(ins_id) self.agent.deregister_some_instances(parameters, terminate_ins_ids) # update db with failed vms VMStateModel.set_state( parameters, terminate_ins_ids, VMStateModel.STATE_FAILED, VMStateModel.DESCRI_TIMEOUT_ON_SSH ) except: raise Exception("Errors in terminating instances that cannot be connected via ssh.") public_ips = None instance_ids = None return connected_public_ips, connected_instance_ids
def __deregister_flex_vm(self, ip, username, keyfile, parameters, queue_head_ip, force=False): try: if self.check_network_ports(ip, [22]): deregister_command = self.get_remote_command_string(ip=ip, username=username, keyfile=keyfile, command="sudo ~/stochss/release-tools/flex-cloud/deregister_flex_vm.sh") logging.debug('deregister_command =\n{}'.format(deregister_command)) os.system(deregister_command) else: logging.debug('Flex VM is not accessible via SSH, can not execute deregister command') except Exception as e: logging.exception('Failed to deregister Flex VM: {0}'.format(e)) # logging.error(sys.exc_info()) finally: VMStateModel.set_state(params=parameters, ins_ids=[self.get_flex_instance_id(public_ip=ip)], state=VMStateModel.STATE_TERMINATED, description='VM Deregistered.')
def prepare_instances(self, parameters, count=None, security_configured=True): """ prepares the specified number of Flex instances using the parameters provided. This method is blocking in that it waits until the requested VMs are properly booted up. However if the requested VMs cannot be procured within 1800 seconds, this method will treat it as an error and return. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters. This must contain 'keyname', 'group', 'image_id' and 'instance_type' parameters. security_configured Uses this boolean value as an heuristic to detect brand new AppScale deployments. Returns: A tuple of the form (instances, public_ips, private_ips) """ logging.debug( 'flex_agent.prepare_instances() parameters={0}'.format(parameters)) try: flex_cloud_machine_info = parameters[ self.PARAM_FLEX_CLOUD_MACHINE_INFO] logging.debug('flex_cloud_machine_info =\n{}'.format( pprint.pformat(flex_cloud_machine_info))) queue_head = parameters[self.PARAM_FLEX_QUEUE_HEAD] logging.debug('queue_head = {}'.format(queue_head)) queue_head_keyfile = queue_head['keyfile'] remote_queue_head_keyfile = os.path.join( FlexConfig.QUEUE_HEAD_KEY_DIR, os.path.basename(queue_head_keyfile)) for machine in flex_cloud_machine_info: ip = machine['ip'] keyfile = machine['keyfile'] os.chmod(keyfile, int('600', 8)) username = machine['username'] is_queue_head = machine[self.PARAM_QUEUE_HEAD] id = self.get_flex_instance_id(public_ip=ip) if not os.path.exists(keyfile): logging.error( 'Keyfile: {0} does not exist!'.format(keyfile)) VMStateModel.set_state( params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_INVALID_KEYFILE) continue logging.debug("[{0}] [{1}] [{2}] [is_queue_head:{3}]".format( ip, keyfile, username, is_queue_head)) scp_command = \ 'scp -o \'UserKnownHostsFile=/dev/null\' -o \'StrictHostKeyChecking no\' -i {keyfile} {source} {target}'.format( keyfile=keyfile, source=queue_head_keyfile, target="{username}@{ip}:{remote_queue_head_keyfile}".format( username=username, ip=ip, remote_queue_head_keyfile=remote_queue_head_keyfile ) ) logging.debug( 'scp command for queue head keyfile =\n{}'.format( scp_command)) res = os.system(scp_command) if res != 0: logging.error( 'scp for queue head keyfile failed!'.format(keyfile)) VMStateModel.set_state( params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue script_lines = [] script_lines.append("#!/bin/bash") script_lines.append( "echo export STOCHKIT_HOME={0} >> ~/.bashrc".format( "~/stochss/StochKit/")) script_lines.append( "echo export STOCHKIT_ODE={0} >> ~/.bashrc".format( "~/stochss/ode/")) script_lines.append( "echo export R_LIBS={0} >> ~/.bashrc".format( "~/stochss/stochoptim/library")) script_lines.append( "echo export C_FORCE_ROOT=1 >> ~/.bashrc".format( "~/stochss/stochoptim/library")) script_lines.append( "chmod 600 {remote_queue_head_keyfile}".format( remote_queue_head_keyfile=remote_queue_head_keyfile)) if is_queue_head: logging.debug( 'Adding extra commands for configuring queue head...') script_lines.append( "sudo rabbitmqctl add_user stochss ucsb") script_lines.append( 'sudo rabbitmqctl set_permissions -p / stochss ".*" ".*" ".*"' ) reset_mysql_script = '~/stochss/release-tools/flex-cloud/reset_mysql_pwd.sh' script_lines.append( "sudo {reset_mysql_script} root {flex_db_password}". format(reset_mysql_script=reset_mysql_script, flex_db_password=parameters[ self.PARAM_FLEX_DB_PASSWORD])) bash_script = '\n'.join(script_lines) logging.debug( "\n\n\nbash_script =\n{0}\n\n\n".format(bash_script)) bash_script_filename = os.path.join(AgentConfig.TMP_DIRNAME, 'stochss_init.sh') with open(bash_script_filename, 'w') as bash_script_file: bash_script_file.write(bash_script) scp_command = 'scp -o \'UserKnownHostsFile=/dev/null\' -o \'StrictHostKeyChecking no\' -i {keyfile} {source} {target}'.format( keyfile=keyfile, source=bash_script_filename, target="{username}@{ip}:~/stochss_init.sh".format( username=username, ip=ip)) logging.debug('scp command =\n{}'.format(scp_command)) res = os.system(scp_command) os.remove(bash_script_filename) if res != 0: logging.error('scp failed!'.format(keyfile)) VMStateModel.set_state( params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue commands = ['chmod +x ~/stochss_init.sh', '~/stochss_init.sh'] command = ';'.join(commands) remote_command_string = self.get_remote_command_string( ip=ip, username=username, keyfile=keyfile, command=command) logging.debug('remote_command_string =\n{}'.format( remote_command_string)) res = os.system(remote_command_string) if res != 0: logging.error('remote command failed!'.format(keyfile)) VMStateModel.set_state( params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue except Exception as e: logging.exception(e) raise
def __configure_celery(self, params, public_ips, instance_ids): ''' Private method used for uploading the current celery configuration to each instance that is running and ssh connectable. Args parameters A dictionary of parameters public_ips A list of public ips that are going to be configed instance_ids A list of instance_ids that are used for terminating instances and update database if fail on configuration by some reason ''' # Update celery config file...it should have the correct IP # of the Queue head node, which should already be running. # Pass it line by line so theres no weird formatting errors from # trying to echo a multi-line file directly on the command line key_file = os.path.join(os.path.dirname(__file__), '..', '{0}.key'.format(params['keyname'])) logging.debug("key_file = {0}".format(key_file)) if not os.path.exists(key_file): raise Exception( "ssh key_file file not found: {0}".format(key_file)) credentials = params['credentials'] commands = [] commands.append('source /home/ubuntu/.bashrc') commands.append('export AWS_ACCESS_KEY_ID={0}'.format( str(credentials['EC2_ACCESS_KEY']))) commands.append('export AWS_SECRET_ACCESS_KEY={0}'.format( str(credentials['EC2_SECRET_KEY']))) for ip, ins_id in zip(public_ips, instance_ids): # helper.wait_for_ssh_connection(key_file, ip) ins_type = VMStateModel.get_instance_type(params, ins_id) commands.append('export INSTANCE_TYPE={0}'.format(ins_type)) success = helper.start_celery_on_vm(instance_type=ins_type, ip=ip, key_file=key_file, agent_type=self.agent_type, worker_name=ip.replace( '.', '_'), prepend_commands=commands) if success == 0: # update db with successful running vms logging.info("celery started! ") logging.info("host ip: {0}".format(ip)) VMStateModel.set_state(params, [ins_id], VMStateModel.STATE_RUNNING, VMStateModel.DESCRI_SUCCESS) else: self.agent.deregister_some_instances(params, [ins_id]) VMStateModel.set_state( params, [ins_id], VMStateModel.STATE_FAILED, VMStateModel.DESCRI_FAIL_TO_COFIGURE_CELERY) raise Exception("Failure to start celery on {0}".format(ip)) # get all intstance types and configure the celeryconfig.py locally instance_types = VMStateModel.get_running_instance_types(params) helper.config_celery_queues(agent_type=self.agent_type, instance_types=instance_types)
def __poll_instances_status(self, num_vms, parameters): ''' Private method that working on polling the state of instances that have already spawned every some time and checking the ssh connectability if they are running. Args num_vms Number of virtual machines that are needed to be polling parameters A dictionary of parameters Return A turple of (public ips, private ips, instance ids). Each of the three is a list ''' logging.info('Start polling task for infrastructure = {0}'.format( parameters['infrastructure'])) ins_ids = self.agent.describe_instances_launched(parameters) logging.info("ins_ids = {0}".format(ins_ids)) # update db with new instance ids and 'pending' VMStateModel.update_ins_ids(parameters, ins_ids, self.reservation_id, from_state=VMStateModel.STATE_CREATING, to_state=VMStateModel.STATE_PENDING) public_ips = None private_ips = None instance_ids = None keyfiles = None for x in xrange(EC2BackendWorker.POLL_COUNT): # get the ips and ids of this keyname public_ips, private_ips, instance_ids, instance_types, keyfiles = self.agent.describe_instances_running( parameters) logging.info("public_ips = {0}".format(public_ips)) logging.debug("private_ips = {0}".format(private_ips)) logging.info("instance_ids = {0}".format(instance_ids)) logging.info("instance_types = {0}".format(instance_types)) logging.info("keyfiles = {0}".format(keyfiles)) # if we get the requested number of vms (the requested number will be 1 if this is queue head), # update reservation information and send a message to the backend server if num_vms == len(public_ips): # update db with new public ips and private ips VMStateModel.update_ips(parameters, instance_ids, public_ips, private_ips, instance_types, keyfiles) break else: if x < EC2BackendWorker.POLL_COUNT - 1: time.sleep(EC2BackendWorker.POLL_WAIT_TIME) logging.info('Polling task: sleep 5 seconds...') else: VMStateModel.update_ips(parameters, instance_ids, public_ips, private_ips, instance_types, keyfiles) logging.info( 'Polling timeout. About to terminate some instances:') terminate_ins_ids = [] for ins_id in ins_ids: if ins_id not in instance_ids: logging.info( 'instance {0} to be terminated'.format(ins_id)) terminate_ins_ids.append(ins_id) # terminate timeout instances self.agent.deregister_some_instances( parameters, terminate_ins_ids) # update db with failure information VMStateModel.set_state(parameters, terminate_ins_ids, VMStateModel.STATE_FAILED, VMStateModel.DESCRI_FAIL_TO_RUN) return public_ips, private_ips, instance_ids
def prepare_instances(self, parameters, count=None, security_configured=True): """ prepares the specified number of Flex instances using the parameters provided. This method is blocking in that it waits until the requested VMs are properly booted up. However if the requested VMs cannot be procured within 1800 seconds, this method will treat it as an error and return. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters. This must contain 'keyname', 'group', 'image_id' and 'instance_type' parameters. security_configured Uses this boolean value as an heuristic to detect brand new AppScale deployments. Returns: A tuple of the form (instances, public_ips, private_ips) """ logging.debug('flex_agent.prepare_instances() parameters={0}'.format(parameters)) try: flex_cloud_machine_info = parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] logging.debug('flex_cloud_machine_info =\n{}'.format(pprint.pformat(flex_cloud_machine_info))) queue_head = parameters[self.PARAM_FLEX_QUEUE_HEAD] logging.debug('queue_head = {}'.format(queue_head)) queue_head_keyfile = queue_head['keyfile'] remote_queue_head_keyfile = os.path.join(FlexConfig.QUEUE_HEAD_KEY_DIR, os.path.basename(queue_head_keyfile)) for machine in flex_cloud_machine_info: ip = machine['ip'] keyfile = machine['keyfile'] os.chmod(keyfile, int('600', 8)) username = machine['username'] is_queue_head = machine[self.PARAM_QUEUE_HEAD] id = self.get_flex_instance_id(public_ip=ip) if not os.path.exists(keyfile): logging.error('Keyfile: {0} does not exist!'.format(keyfile)) VMStateModel.set_state(params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_INVALID_KEYFILE) continue logging.debug("[{0}] [{1}] [{2}] [is_queue_head:{3}]".format(ip, keyfile, username, is_queue_head)) scp_command = \ 'scp -o \'UserKnownHostsFile=/dev/null\' -o \'StrictHostKeyChecking no\' -i {keyfile} {source} {target}'.format( keyfile=keyfile, source=queue_head_keyfile, target="{username}@{ip}:{remote_queue_head_keyfile}".format( username=username, ip=ip, remote_queue_head_keyfile=remote_queue_head_keyfile ) ) logging.debug('scp command for queue head keyfile =\n{}'.format(scp_command)) res = os.system(scp_command) if res != 0: logging.error('scp for queue head keyfile failed!'.format(keyfile)) VMStateModel.set_state(params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue script_lines = [] script_lines.append("#!/bin/bash") script_lines.append("echo export STOCHKIT_HOME={0} >> ~/.bashrc".format("~/stochss/StochKit/")) script_lines.append("echo export STOCHKIT_ODE={0} >> ~/.bashrc".format("~/stochss/ode/")) script_lines.append("echo export R_LIBS={0} >> ~/.bashrc".format("~/stochss/stochoptim/library")) script_lines.append("echo export C_FORCE_ROOT=1 >> ~/.bashrc".format("~/stochss/stochoptim/library")) script_lines.append("chmod 600 {remote_queue_head_keyfile}".format( remote_queue_head_keyfile=remote_queue_head_keyfile)) if is_queue_head: logging.debug('Adding extra commands for configuring queue head...') script_lines.append("sudo rabbitmqctl add_user stochss ucsb") script_lines.append('sudo rabbitmqctl set_permissions -p / stochss ".*" ".*" ".*"') reset_mysql_script = '~/stochss/release-tools/flex-cloud/reset_mysql_pwd.sh' script_lines.append("sudo {reset_mysql_script} root {flex_db_password}".format( reset_mysql_script=reset_mysql_script, flex_db_password=parameters[self.PARAM_FLEX_DB_PASSWORD])) bash_script = '\n'.join(script_lines) logging.debug("\n\n\nbash_script =\n{0}\n\n\n".format(bash_script)) bash_script_filename = os.path.join(AgentConfig.TMP_DIRNAME, 'stochss_init.sh') with open(bash_script_filename, 'w') as bash_script_file: bash_script_file.write(bash_script) scp_command = 'scp -o \'UserKnownHostsFile=/dev/null\' -o \'StrictHostKeyChecking no\' -i {keyfile} {source} {target}'.format( keyfile=keyfile, source=bash_script_filename, target="{username}@{ip}:~/stochss_init.sh".format(username=username, ip=ip)) logging.debug('scp command =\n{}'.format(scp_command)) res = os.system(scp_command) os.remove(bash_script_filename) if res != 0: logging.error('scp failed!'.format(keyfile)) VMStateModel.set_state(params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue commands = ['chmod +x ~/stochss_init.sh', '~/stochss_init.sh'] command = ';'.join(commands) remote_command_string = self.get_remote_command_string(ip=ip, username=username, keyfile=keyfile, command=command) logging.debug('remote_command_string =\n{}'.format(remote_command_string)) res = os.system(remote_command_string) if res != 0: logging.error('remote command failed!'.format(keyfile)) VMStateModel.set_state(params=parameters, ins_ids=[id], state=VMStateModel.STATE_FAILED, description=VMStateModel.DESCRI_FAIL_TO_PREPARE) continue except Exception as e: logging.exception(e) raise
def __poll_instances_status(self, num_vms, parameters): """ Private method that working on polling the state of instances that have already spawned every some time and checking the ssh connectability if they are running. Args num_vms Number of virtual machines that are needed to be polling parameters A dictionary of parameters Return A turple of (public ips, private ips, instance ids). Each of the three is a list """ logging.info("Start polling task for infrastructure = {0}".format(parameters["infrastructure"])) ins_ids = self.agent.describe_instances_launched(parameters) logging.info("ins_ids = {0}".format(ins_ids)) # update db with new instance ids and 'pending' VMStateModel.update_ins_ids( parameters, ins_ids, self.reservation_id, from_state=VMStateModel.STATE_CREATING, to_state=VMStateModel.STATE_PENDING, ) public_ips = None private_ips = None instance_ids = None keyfiles = None for x in xrange(EC2BackendWorker.POLL_COUNT): # get the ips and ids of this keyname public_ips, private_ips, instance_ids, instance_types, keyfiles = self.agent.describe_instances_running( parameters ) logging.info("public_ips = {0}".format(public_ips)) logging.debug("private_ips = {0}".format(private_ips)) logging.info("instance_ids = {0}".format(instance_ids)) logging.info("instance_types = {0}".format(instance_types)) logging.info("keyfiles = {0}".format(keyfiles)) # if we get the requested number of vms (the requested number will be 1 if this is queue head), # update reservation information and send a message to the backend server if num_vms == len(public_ips): # update db with new public ips and private ips VMStateModel.update_ips(parameters, instance_ids, public_ips, private_ips, instance_types, keyfiles) break else: if x < EC2BackendWorker.POLL_COUNT - 1: time.sleep(EC2BackendWorker.POLL_WAIT_TIME) logging.info("Polling task: sleep 5 seconds...") else: VMStateModel.update_ips(parameters, instance_ids, public_ips, private_ips, instance_types, keyfiles) logging.info("Polling timeout. About to terminate some instances:") terminate_ins_ids = [] for ins_id in ins_ids: if ins_id not in instance_ids: logging.info("instance {0} to be terminated".format(ins_id)) terminate_ins_ids.append(ins_id) # terminate timeout instances self.agent.deregister_some_instances(parameters, terminate_ins_ids) # update db with failure information VMStateModel.set_state( parameters, terminate_ins_ids, VMStateModel.STATE_FAILED, VMStateModel.DESCRI_FAIL_TO_RUN ) return public_ips, private_ips, instance_ids