def run_instances(self, prefix, blocking, success=True): i = InfrastructureManager(blocking=blocking) reservation = Reservation() instance = flexmock(name='instance', private_dns_name='private-ip', public_dns_name='public-ip', id='i-id', state='running', key_name='bookeyname', ip_address='public-ip', private_ip_address='private-ip') new_instance = flexmock(name='new-instance', private_dns_name='new-private-ip', public_dns_name='new-public-ip', id='new-i-id', state='running', key_name='bookeyname', ip_address='new-public-ip', private_ip_address='new-private-ip') reservation.instances = [instance] new_reservation = Reservation() new_reservation.instances = [instance, new_instance] self.fake_ec2.should_receive('get_all_instances').and_return([]) \ .and_return([reservation]).and_return([new_reservation]) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': prefix, 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': False, 'region' : 'my-zone-1', 'zone' : 'my-zone-1b', 'autoscale_agent': True } id = '0000000000' # no longer randomly generated full_result = { 'success': True, 'reservation_id': id, 'reason': 'none' } if success: self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info if not blocking: time.sleep(.1) if success: self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state']) vm_info = i.reservations.get(id)['vm_info'] self.assertEquals(['new-public-ip'], vm_info['public_ips']) self.assertEquals(['new-private-ip'], vm_info['private_ips']) self.assertEquals(['new-i-id'], vm_info['instance_ids']) else: if blocking: self.assertRaises(AgentRuntimeException, i.run_instances, full_params, 'secret')
def test_ec2_run_instances(self): i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials': {'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': 'ec2', 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': 'True', 'max_spot_price' : '1.23', 'zone' : 'my-zone-1b' } id = '0000000000' # no longer randomly generated full_result = { 'success': True, 'reservation_id': id, 'reason': 'none' } self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state']) vm_info = i.reservations.get(id)['vm_info'] self.assertEquals(['public-ip'], vm_info['public_ips']) self.assertEquals(['private-ip'], vm_info['private_ips']) self.assertEquals(['i-id'], vm_info['instance_ids'])
def terminate_instances(self, prefix, blocking): i = InfrastructureManager(blocking=blocking) params1 = {'infrastructure': prefix} self.assertRaises(AgentConfigurationException, i.terminate_instances, params1, 'secret') params2 = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://ec2.url.com', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'infrastructure': prefix, 'instance_ids': ['i-12345'], 'region' : 'my-zone-1', 'keyname': 'bookeyname' } reservation = Reservation() instance = flexmock(name='instance', private_dns_name='private-ip', public_dns_name='public-ip', id='i-id', state='terminated', key_name='bookeyname', ip_address='public-ip', private_ip_address='private-ip') reservation.instances = [instance] self.fake_ec2.should_receive('get_all_instances').and_return([reservation]) flexmock(i).should_receive('_InfrastructureManager__kill_vms') result = i.terminate_instances(params2, 'secret') if not blocking: time.sleep(.1) self.assertTrue(result['success'])
def validateCredentials(self, params): ''' This method verifies the validity of ec2 credentials ''' if params['infrastructure'] is None: logging.error("validateCredentials: infrastructure param not set") return False creds = params['credentials'] if creds is None: logging.error("validateCredentials: credentials param not set") return False if creds['EC2_ACCESS_KEY'] is None: logging.error("validateCredentials: credentials EC2_ACCESS_KEY not set") return False if creds['EC2_SECRET_KEY'] is None: logging.error("validateCredentials: credentials EC2_ACCESS_KEY not set") return False logging.debug("validateCredentials: inside method with params : %s", str(params)) try: i = InfrastructureManager() logging.debug("validateCredentials: exiting with result : %s", str(i)) return i.validate_credentials(params) except Exception, e: logging.error("validateCredentials: exiting with error : %s", str(e)) return False
def test_euca_run_instances(self): i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key' }, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': 'euca', 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '2', 'use_spot_instances': False, } id = '0000000000' # no longer randomly generated full_result = {'success': True, 'reservation_id': id, 'reason': 'none'} self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state']) vm_info = i.reservations.get(id)['vm_info'] self.assertEquals(['ABC-public-ip1', 'DEF-public-ip2'], vm_info['public_ips']) self.assertEquals(['DEF-private-ip1', 'ABC-private-ip2'], vm_info['private_ips']) self.assertEquals(['i-id1', 'i-id2'], vm_info['instance_ids'])
def test_euca_run_instances(self): i = InfrastructureManager(blocking=True) reservation = Reservation() instance = flexmock(name='instance', private_dns_name='private-ip', public_dns_name='public-ip', id='i-id', state='running', key_name='bookeyname', ip_address='public-ip', private_ip_address='private-ip') new_instance = flexmock(name='new-instance', private_dns_name='new-private-ip', public_dns_name='new-public-ip', id='new-i-id', state='running', key_name='bookeyname', ip_address='new-public-ip', private_ip_address='new-private-ip') reservation.instances = [instance] new_reservation = Reservation() new_reservation.instances = [instance, new_instance] flexmock(EC2Connection).should_receive('get_all_instances').and_return([]) \ .and_return([reservation]).and_return([reservation]) \ .and_return([new_reservation]).and_return([new_reservation]) # first, validate that the run_instances call goes through successfully # and gives the user an operation id full_params = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key' }, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': 'euca', 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': False, 'zone': 'my-zone-1b', 'autoscale_agent': True, 'IS_VERBOSE': True } id = '0000000000' # no longer randomly generated full_result = {'success': True, 'operation_id': id, 'reason': 'none'} self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its operation info self.assertEquals(InfrastructureManager.STATE_SUCCESS, i.operation_ids.get(id)['state']) vm_info = i.operation_ids.get(id)['vm_info'] self.assertEquals(['new-public-ip'], vm_info['public_ips']) self.assertEquals(['new-private-ip'], vm_info['private_ips']) self.assertEquals(['new-i-id'], vm_info['instance_ids'])
def deregister_flex_cloud(self, parameters, blocking=True): try: i = InfrastructureManager(blocking=blocking) res = i.deregister_instances(parameters=parameters, terminate=False) ret = True except Exception, e: logging.error("deregister_flex_cloud() failed with error : %s", str(e)) ret = False
def validateCredentials(self, params): ''' This method verifies the validity of ec2 credentials ''' logging.info("validateCredentials: inside method with params : %s", str(params)) try: i = InfrastructureManager() logging.info("validateCredentials: exiting with result : %s", str(i)) return i.validate_Credentials(params) except Exception, e: logging.error("validateCredentials: exiting with error : %s", str(e)) return False
def test_attach_persistent_disk(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_credentials = flexmock(name='fake_credentials') fake_credentials.should_receive('invalid').and_return(False) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(fake_credentials) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args( GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) fake_instances = flexmock(name='fake_instances') fake_gce = flexmock(name='fake_gce') fake_gce.should_receive('instances').and_return(fake_instances) fake_attach_disk_request = flexmock(name='fake_attach_disk_request') fake_instances.should_receive('get').and_return(fake_attach_disk_request) attach_disk_info = { 'status': 'DONE', 'disks': [] } fake_attach_disk_request.should_receive('execute').with_args( fake_authorized_http).and_return(attach_disk_info) fake_instances.should_receive('attachDisk').with_args(project=self.project, body=dict, instance='my-instance', zone=str).and_return( fake_attach_disk_request) # finally, inject our fake GCE connection flexmock(discovery) discovery.should_receive('build').with_args('compute', GCEAgent.API_VERSION).and_return(fake_gce) iaas = InfrastructureManager(blocking=True) disk_name = 'my-disk-name' instance_id = 'my-instance' expected = '/dev/sdb' actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret') self.assertTrue(actual['success']) self.assertEquals(expected, actual['location'])
def stopMachines(self, params, block=False): """ This method would terminate all the instances associated with the account that have a keyname prefixed with stochss (all instances created by the backend service) params must contain credentials key/value """ try: i = InfrastructureManager(blocking=block) res = i.terminate_instances(params, backendservices.KEYPREFIX) return True except Exception, e: logging.error("Terminate machine failed with error : %s", str(e)) return False
def test_attach_persistent_disk(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_credentials = flexmock(name='fake_credentials', invalid=False) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(fake_credentials) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args( GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) fake_instances = flexmock(name='fake_instances') fake_gce = flexmock(name='fake_gce') fake_gce.should_receive('instances').and_return(fake_instances) fake_attach_disk_request = flexmock(name='fake_attach_disk_request') fake_instances.should_receive('get').and_return(fake_attach_disk_request) attach_disk_info = { 'status': 'DONE', 'disks': [] } fake_attach_disk_request.should_receive('execute').with_args( fake_authorized_http).and_return(attach_disk_info) fake_instances.should_receive('attachDisk').with_args(project=self.project, body=dict, instance='my-instance', zone=str).and_return( fake_attach_disk_request) # finally, inject our fake GCE connection flexmock(discovery) discovery.should_receive('build').with_args('compute', GCEAgent.API_VERSION).and_return(fake_gce) iaas = InfrastructureManager(blocking=True) disk_name = 'my-disk-name' instance_id = 'my-instance' expected = '/dev/sdb' actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret') self.assertTrue(actual['success']) self.assertEquals(expected, actual['location'])
def describe_machines_from_db(self, infrastructure, force=False): parameters = { "infrastructure": infrastructure, "credentials": self.get_credentials(), "key_prefix": self.user_data.user_id, "user_id": self.user_data.user_id, } if infrastructure == AgentTypes.FLEX: parameters['flex_cloud_machine_info'] = self.user_data.get_flex_cloud_machine_info() parameters['reservation_id'] = self.user_data.reservation_id i = InfrastructureManager() i.synchronize_db(parameters, force=force) all_vms = VMStateModel.get_all(parameters) return all_vms
def startMachines(self, params, block=False): ''' This method instantiates ec2 instances ''' logging.info("startMachines : inside method with params : %s", str(params)) try: #make sure that any keynames we use are prefixed with stochss so that #we can do a terminate all based on keyname prefix key_name = params["keyname"] if not key_name.startswith(self.KEYPREFIX): params['keyname'] = self.KEYPREFIX + key_name # NOTE: We are forcing blocking mode within the InfrastructureManager class # for the launching of VMs because of how GAE joins on all threads before # returning a response from a request. i = InfrastructureManager(blocking=block) res = {} # NOTE: We need to make sure that the RabbitMQ server is running if any compute # nodes are running as we are using the AMQP broker option for Celery. compute_check_params = { "credentials": params["credentials"], "key_prefix": params["key_prefix"] } if self.isQueueHeadRunning(compute_check_params): res = i.run_instances(params,[]) else: # Need to start the queue head (RabbitMQ) params["queue_head"] = True vms_requested = int(params["num_vms"]) requested_key_name = params["keyname"] # Only want one queue head, and it must have its own key so # it can be differentiated if necessary params["num_vms"] = 1 params["keyname"] = requested_key_name+'-'+self.QUEUEHEAD_KEY_TAG res = i.run_instances(params,[]) #NOTE: This relies on the InfrastructureManager being run in blocking mode... queue_head_ip = res["vm_info"]["public_ips"][0] self.__update_celery_config_with_queue_head_ip(queue_head_ip) params["keyname"] = requested_key_name params["queue_head"] = False if vms_requested > 1: params["num_vms"] = vms_requested - 1 res = i.run_instances(params,[]) params["num_vms"] = vms_requested logging.info("startMachines : exiting method with result : %s", str(res)) return res except Exception, e: logging.error("startMachines : exiting method with error : {0}".format(str(e))) print "startMachines : exiting method with error :", str(e) return None
def describeMachines(self, params): """ This method gets the status of all the instances of ec2 """ # add calls to the infrastructure manager for getting details of # machines logging.info("describeMachines : inside method with params : %s", str(params)) try: i = InfrastructureManager() res = i.describe_instances(params, [], backendservices.KEYPREFIX) logging.info("describeMachines : exiting method with result : %s", str(res)) return res except Exception, e: logging.error("describeMachines : exiting method with error : %s", str(e)) return None
def describeMachines(self, params): ''' This method gets the status of all the instances of ec2 ''' # add calls to the infrastructure manager for getting details of # machines logging.info("describeMachines : inside method with params : %s", str(params)) try: i = InfrastructureManager() secret =[] res = i.describe_instances(params, secret) logging.info("describeMachines : exiting method with result : %s", str(res)) return res except Exception, e: logging.error("describeMachines : exiting method with error : %s", str(e)) return None
def startMachines(self, params): ''' This method instantiates ec2 instances ''' #this will basically start an instance in ec2 # add call from the infrastructure manager here logging.info("startMachines : inside method with params : %s", str(params)) try: i = InfrastructureManager(blocking=True) res = i.run_instances(params,params) return res logging.info("startMachines : exiting method with result : %s", str(res)) except Exception, e: logging.error("startMachines : exiting method with error : %s", str(e)) return None
def stopMachines(self, params): ''' This method would terminate all the instances associated with the account It expects the following two fields in the parameter argument params ={"infrastructure":"ec2", 'credentials':{"EC2_ACCESS_KEY":"______________", "EC2_SECRET_KEY":"__________"}, } ''' try: i = InfrastructureManager(blocking=True) res = i.terminate_instances(params) print str(res) return True except Exception, e: logging.error("Terminate machine failed with error : %s", str(e)) return False
def stop_ec2_vms(self, params, blocking=False): ''' This method would terminate all the EC2 instances associated with the account that have a keyname prefixed with stochss (all instances created by the backend service) params must contain credentials key/value ''' key_prefix = AgentConfig.get_agent_key_prefix(agent_type=AgentTypes.EC2, key_prefix=params.get('key_prefix', '')) try: logging.debug("Stopping compute nodes with key_prefix: {0}".format(key_prefix)) i = InfrastructureManager(blocking=blocking) res = i.deregister_instances(parameters=params, terminate=True) ret = True except Exception, e: logging.error("Terminate machine failed with error : %s", str(e)) ret = False
def terminate_instances(self, prefix, blocking): i = InfrastructureManager(blocking=blocking) params1 = {'infrastructure': prefix} result1 = i.terminate_instances(params1, 'secret') self.assertFalse(result1['success']) self.assertEquals(result1['reason'], 'no credentials') params2 = { 'credentials': {'a': 'b', 'EC2_URL': 'http://ec2.url.com', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'infrastructure': prefix, 'instance_ids': ['i-12345'] } result2 = i.terminate_instances(params2, 'secret') if not blocking: time.sleep(2) self.assertTrue(result2['success'])
def stopMachines(self, params, block=False): ''' This method would terminate all the instances associated with the account that have a keyname prefixed with stochss (all instances created by the backend service) params must contain credentials key/value ''' key_prefix = self.KEYPREFIX if "key_prefix" in params and not params["key_prefix"].startswith(key_prefix): key_prefix += params["key_prefix"] elif "key_prefix" in params: key_prefix = params["key_prefix"] try: logging.info("Stopping compute nodes with key_prefix: {0}".format(key_prefix)) i = InfrastructureManager(blocking=block) res = i.terminate_instances(params,key_prefix) return True except Exception, e: logging.error("Terminate machine failed with error : %s", str(e)) return False
def test_attach_persistent_disk(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_credentials = flexmock(name="fake_credentials") fake_storage = flexmock(name="fake_storage") fake_storage.should_receive("get").and_return(fake_credentials) flexmock(oauth2client.file) oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) # next, mock out http calls to GCE fake_http = flexmock(name="fake_http") fake_authorized_http = flexmock(name="fake_authorized_http") flexmock(httplib2) httplib2.should_receive("Http").and_return(fake_http) fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http) fake_instances = flexmock(name="fake_instances") fake_gce = flexmock(name="fake_gce") fake_gce.should_receive("instances").and_return(fake_instances) attach_disk_info = {"status": "DONE"} fake_attach_disk_request = flexmock(name="fake_attach_disk_request") fake_attach_disk_request.should_receive("execute").with_args(fake_authorized_http).and_return(attach_disk_info) fake_instances.should_receive("attachDisk").with_args( project=self.project, body=dict, instance="my-instance", zone=str ).and_return(fake_attach_disk_request) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce) iaas = InfrastructureManager(blocking=True) disk_name = "my-disk-name" instance_id = "my-instance" expected = "/dev/sdb" actual = iaas.attach_disk(self.params, disk_name, instance_id, "secret") self.assertTrue(actual["success"]) self.assertEquals(expected, actual["location"])
def run_instances(self, prefix, blocking, success=True): i = InfrastructureManager(blocking=blocking) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': prefix, 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': False, 'region' : 'my-zone-1', 'zone' : 'my-zone-1b' } id = '0000000000' # no longer randomly generated full_result = { 'success': True, 'reservation_id': id, 'reason': 'none' } self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info if not blocking: time.sleep(.1) if success: self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state']) vm_info = i.reservations.get(id)['vm_info'] self.assertEquals(['public-ip'], vm_info['public_ips']) self.assertEquals(['private-ip'], vm_info['private_ips']) self.assertEquals(['i-id'], vm_info['instance_ids']) else: self.assertEquals(InfrastructureManager.STATE_FAILED, i.reservations.get(id)['state'])
def terminate_instances(self, prefix, blocking): i = InfrastructureManager(blocking=blocking) params1 = {'infrastructure': prefix} result1 = i.terminate_instances(params1, 'secret') self.assertFalse(result1['success']) self.assertEquals(result1['reason'], 'no credentials') params2 = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://ec2.url.com', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'infrastructure': prefix, 'instance_ids': ['i-12345'], 'region' : 'my-zone-1' } result2 = i.terminate_instances(params2, 'secret') if not blocking: time.sleep(.1) self.assertTrue(result2['success'])
def prepare_flex_cloud_machines(self, params, blocking=False): logging.debug("prepare_flex_cloud_machines : params : \n%s", pprint.pformat(params)) try: # NOTE: We are forcing blocking mode within the InfrastructureManager class # for the launching of VMs because of how GAE joins on all threads before # returning a response from a request. i = InfrastructureManager(blocking=blocking) res = {} # 4. Prepare Instances res = i.prepare_instances(params) logging.debug("prepare_flex_cloud_machines : exiting method with result : %s", str(res)) return True, '' except Exception, e: traceback.print_exc() logging.error("prepare_flex_cloud_machines : exiting method with error : {0}".format(str(e))) return False, 'Errors occur in preparing machines:' + str(e)
def describeMachines(self, params): ''' This method gets the status of all the instances of ec2 ''' # add calls to the infrastructure manager for getting details of # machines logging.info("describeMachines : inside method with params : %s", str(params)) key_prefix = "" if "key_prefix" in params: key_prefix = params["key_prefix"] if not key_prefix.startswith(self.KEYPREFIX): key_prefix = self.KEYPREFIX + key_prefix else: key_prefix = self.KEYPREFIX try: i = InfrastructureManager() res = i.describe_instances(params, [], key_prefix) logging.info("describeMachines : exiting method with result : %s", str(res)) return res except Exception, e: logging.error("describeMachines : exiting method with error : %s", str(e)) return None
def startMachines(self, params, block=False): """ This method instantiates ec2 instances """ logging.info("startMachines : inside method with params : %s", str(params)) try: keyname = params["keyname"] # make sure that any keynames we use are prefixed with stochss so that # we can do a terminate all based on keyname prefix if not keyname.startswith(backendservices.KEYPREFIX): params["keyname"] = backendservices.KEYPREFIX + keyname # NOTE: We are forcing blocking mode within the InfrastructureManager class # for the launching of VMs because of how GAE joins on all threads before # returning a response from a request. i = InfrastructureManager(blocking=block) res = i.run_instances(params, []) logging.info("startMachines : exiting method with result : %s", str(res)) return res except Exception, e: logging.error("startMachines : exiting method with error : %s", str(e)) return None
def run_instances(self, prefix, blocking, success=True): i = InfrastructureManager(blocking=blocking) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials': {'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'}, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': prefix, 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'spot': 'False', } id = '0000000000' # no longer randomly generated full_result = { 'success': True, 'reservation_id': id, 'reason': 'none' } self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info if not blocking: time.sleep(2) if success: self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state']) vm_info = i.reservations.get(id)['vm_info'] self.assertEquals(['public-ip'], vm_info['public_ips']) self.assertEquals(['private-ip'], vm_info['private_ips']) self.assertEquals(['i-id'], vm_info['instance_ids']) else: self.assertEquals(InfrastructureManager.STATE_FAILED, i.reservations.get(id)['state'])
def test_ec2_run_instances(self): i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user an operation id full_params = { 'credentials': { 'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar', 'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key' }, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': 'ec2', 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': 'True', 'max_spot_price': '1.23', 'region': 'my-zone-1', 'zone': 'my-zone-1b', 'autoscale_agent': True } id = '0000000000' # no longer randomly generated full_result = {'success': True, 'operation_id': id, 'reason': 'none'} self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its operation info self.assertEquals(InfrastructureManager.STATE_SUCCESS, i.operation_ids.get(id)['state']) vm_info = i.operation_ids.get(id)['vm_info'] self.assertEquals(['new-public-ip'], vm_info['public_ips']) self.assertEquals(['new-private-ip'], vm_info['private_ips']) self.assertEquals(['new-i-id'], vm_info['instance_ids'])
def test_gce_run_instances(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_credentials = flexmock(name='fake_credentials') fake_credentials.should_receive('invalid').and_return(False) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(fake_credentials) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args( GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) # add some fake data in where no instances are initially running, then one # is (in response to our insert request) no_instance_info = { } instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d' list_instance_info = { u'items': [{ u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': instance_id, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'tags': {u'fingerprint': u'42WmSpB8rSM='}, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public-ip' }], u'networkIP': u'private-ip', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup', u'name': u'nic0' }] }], u'kind': u'compute#instanceList', u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances' } fake_list_instance_request = flexmock(name='fake_list_instance_request') fake_list_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(no_instance_info).and_return(list_instance_info) fake_instances = flexmock(name='fake_instances') fake_gce = flexmock(name='fake_gce') fake_gce.should_receive('instances').and_return(fake_instances) fake_instances.should_receive('list').with_args(project=self.project, filter="name eq boogroup-.*", zone='my-zone-1b') \ .and_return(fake_list_instance_request) # we only need to create one node, so set up mocks for that add_instance = u'operation-1369248752891-4dd5311848461-afc55a20' add_instance_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': add_instance, u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b', u'startTime': u'2013-05-22T11:52:32.939-07:00', u'insertTime': u'2013-05-22T11:52:32.891-07:00', u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'operationType': u'insert', u'progress': 0, u'id': u'6663616273628949255', u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20', u'user': u'*****@*****.**' } fake_add_instance_request = flexmock(name='fake_add_instance_request') fake_add_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(add_instance_info) fake_instances.should_receive('insert').with_args(project=self.project, body=dict, zone=str).and_return(fake_add_instance_request) created_instance_info = { u'status': u'DONE' } fake_instance_checker = flexmock(name='fake_network_checker') fake_instance_checker.should_receive('execute').and_return( created_instance_info) fake_blocker = flexmock(name='fake_blocker') fake_gce.should_receive('globalOperations').and_return(fake_blocker) fake_blocker.should_receive('get').with_args(project=self.project, operation=add_instance).and_return(fake_instance_checker) # finally, inject our fake GCE connection flexmock(discovery) discovery.should_receive('build').with_args('compute', GCEAgent.API_VERSION).and_return(fake_gce) # next, presume that the persistent disk we want to use exists disk_name = 'my-persistent-disk-1' disk_info = {'status':'DONE'} fake_disk_request = flexmock(name='fake_disk_request') fake_disk_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(disk_info) fake_disks = flexmock(name='fake_disks') fake_disks.should_receive('get').with_args(project=self.project, disk=disk_name, zone=str).and_return(fake_disk_request) fake_disks.should_receive('insert').with_args(project=self.project, sourceImage=str, body=dict, zone=str).and_return(fake_disk_request) fake_gce.should_receive('disks').and_return(fake_disks) public_key = 'ssh-rsa long_key_string' flexmock(utils).should_receive('get_public_key').and_return(public_key) i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_result = { 'success': True, 'reservation_id': self.reservation_id, 'reason': 'none' } self.assertEquals(full_result, i.run_instances(self.params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get( self.reservation_id)['state']) vm_info = i.reservations.get(self.reservation_id)['vm_info'] self.assertEquals(['public-ip'], vm_info['public_ips']) self.assertEquals(['private-ip'], vm_info['private_ips']) self.assertEquals([instance_id], vm_info['instance_ids'])
def test_gce_run_instances(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_credentials = flexmock(name="fake_credentials") fake_storage = flexmock(name="fake_storage") fake_storage.should_receive("get").and_return(fake_credentials) flexmock(oauth2client.file) oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) # next, mock out http calls to GCE fake_http = flexmock(name="fake_http") fake_authorized_http = flexmock(name="fake_authorized_http") flexmock(httplib2) httplib2.should_receive("Http").and_return(fake_http) fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http) # add some fake data in where no instances are initially running, then one # is (in response to our insert request) no_instance_info = {} instance_id = u"appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d" list_instance_info = { u"items": [ { u"status": u"RUNNING", u"kind": u"compute#instance", u"machineType": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1", u"name": instance_id, u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b", u"tags": {u"fingerprint": u"42WmSpB8rSM="}, u"image": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64", u"disks": [ {u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"} ], u"canIpForward": False, u"serviceAccounts": [ {u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"} ], u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="}, u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00", u"id": u"8684033495853907982", u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d", u"networkInterfaces": [ { u"accessConfigs": [ { u"kind": u"compute#accessConfig", u"type": u"ONE_TO_ONE_NAT", u"name": u"External NAT", u"natIP": u"public-ip", } ], u"networkIP": u"private-ip", u"network": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup", u"name": u"nic0", } ], } ], u"kind": u"compute#instanceList", u"id": u"projects/appscale.com:appscale/zones/my-zone-1b/instances", u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances", } fake_list_instance_request = flexmock(name="fake_list_instance_request") fake_list_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return( no_instance_info ).and_return(list_instance_info) fake_instances = flexmock(name="fake_instances") fake_gce = flexmock(name="fake_gce") fake_gce.should_receive("instances").and_return(fake_instances) fake_instances.should_receive("list").with_args( project=self.project, filter="name eq appscale-boogroup-.*", zone="my-zone-1b" ).and_return(fake_list_instance_request) # we only need to create one node, so set up mocks for that add_instance = u"operation-1369248752891-4dd5311848461-afc55a20" add_instance_info = { u"status": u"PENDING", u"kind": u"compute#operation", u"name": add_instance, u"azone": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/my-zone-1b", u"startTime": u"2013-05-22T11:52:32.939-07:00", u"insertTime": u"2013-05-22T11:52:32.891-07:00", u"targetLink": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d", u"operationType": u"insert", u"progress": 0, u"id": u"6663616273628949255", u"selfLink": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20", u"user": u"*****@*****.**", } fake_add_instance_request = flexmock(name="fake_add_instance_request") fake_add_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return( add_instance_info ) fake_instances.should_receive("insert").with_args(project=self.project, body=dict, zone=str).and_return( fake_add_instance_request ) created_instance_info = {u"status": u"DONE"} fake_instance_checker = flexmock(name="fake_network_checker") fake_instance_checker.should_receive("execute").and_return(created_instance_info) fake_blocker = flexmock(name="fake_blocker") fake_gce.should_receive("globalOperations").and_return(fake_blocker) fake_blocker.should_receive("get").with_args(project=self.project, operation=add_instance).and_return( fake_instance_checker ) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce) i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_result = {"success": True, "reservation_id": self.reservation_id, "reason": "none"} self.assertEquals(full_result, i.run_instances(self.params, "secret")) # next, look at run_instances internally to make sure it actually is # updating its reservation info self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(self.reservation_id)["state"]) vm_info = i.reservations.get(self.reservation_id)["vm_info"] self.assertEquals(["public-ip"], vm_info["public_ips"]) self.assertEquals(["private-ip"], vm_info["private_ips"]) self.assertEquals([instance_id], vm_info["instance_ids"])
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, ssl=True): """ Initialize a new instance of the infrastructure manager service. Args: host Hostname to which the service should bind (Optional). Defaults to 0.0.0.0. port Port of the service (Optional). Default to 17444. ssl True if SSL should be engaged or False otherwise (Optional). Defaults to True. When engaged, this implementation expects to find the necessary SSL certificates in the /etc/appscale/certs directory. """ self.host = host self.port = port secret = None while True: try: secret = utils.get_secret(self.APPSCALE_DIR + 'secret.key') break except Exception: logging.info('Waiting for the secret key to become available') utils.sleep(5) logging.info('Found the secret set to: {0}'.format(secret)) SOAPpy.Config.simplify_objects = True if ssl: logging.info('Checking for the certificate and private key') cert = self.APPSCALE_DIR + 'certs/mycert.pem' key = self.APPSCALE_DIR + 'certs/mykey.pem' while True: if os.path.exists(cert) and os.path.exists(key): break else: logging.info('Waiting for certificates') utils.sleep(5) ssl_context = SSL.Context() ssl_context.load_cert(cert, key) self.server = SOAPpy.SOAPServer((host, port), ssl_context=ssl_context) else: self.server = SOAPpy.SOAPServer((host, port)) parent_dir = os.path.dirname(os.path.realpath(sys.argv[0])) config_file = os.path.join(parent_dir, self.CONFIG_FILE) if os.path.exists(config_file): with open(config_file) as file_handle: params = json.load(file_handle) if params.has_key(PersistentStoreFactory.PARAM_STORE_TYPE): logging.info( 'Loading infrastructure manager configuration from ' + config_file) i = InfrastructureManager(params) else: i = InfrastructureManager() else: i = InfrastructureManager() self.server.registerFunction(i.describe_operation) self.server.registerFunction(i.run_instances) self.server.registerFunction(i.terminate_instances) self.server.registerFunction(i.attach_disk) system_manager = SystemManager() self.server.registerFunction(system_manager.get_cpu_usage) self.server.registerFunction(system_manager.get_disk_usage) self.server.registerFunction(system_manager.get_memory_usage) self.server.registerFunction(system_manager.get_service_summary) self.server.registerFunction(system_manager.get_swap_usage) self.server.registerFunction(system_manager.get_loadavg) self.started = False
def test_gce_run_instances(self): # mock out interactions with GCE # first, mock out the oauth library calls fake_flow = flexmock(name='fake_flow') flexmock(oauth2client.client) oauth2client.client.should_receive('flow_from_clientsecrets').with_args( GCEAgent.CLIENT_SECRETS_LOCATION, scope=GCEAgent.GCE_SCOPE).and_return( fake_flow) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(None) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args( GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage) fake_credentials = flexmock(name='fake_credentials') flexmock(oauth2client.tools) oauth2client.tools.should_receive('run').with_args(fake_flow, fake_storage).and_return(fake_credentials) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) # add some fake data in where no instances are initially running, then one # is (in response to our insert request) no_instance_info = { } instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d' list_instance_info = { u'items': [{ u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': instance_id, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a', u'tags': {u'fingerprint': u'42WmSpB8rSM='}, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public-ip' }], u'networkIP': u'private-ip', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup', u'name': u'nic0' }] }], u'kind': u'compute#instanceList', u'id': u'projects/appscale.com:appscale/zones/us-central1-a/instances', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/us-central1-a/instances' } fake_list_instance_request = flexmock(name='fake_list_instance_request') fake_list_instance_request.should_receive('execute').with_args( fake_authorized_http).and_return(no_instance_info).and_return( list_instance_info) fake_instances = flexmock(name='fake_instances') fake_gce = flexmock(name='fake_gce') fake_gce.should_receive('instances').and_return(fake_instances) fake_instances.should_receive('list').with_args(project=self.project, filter="name eq appscale-boogroup-.*", zone=GCEAgent.DEFAULT_ZONE) \ .and_return(fake_list_instance_request) # we only need to create one node, so set up mocks for that add_instance = u'operation-1369248752891-4dd5311848461-afc55a20' add_instance_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': add_instance, u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a', u'startTime': u'2013-05-22T11:52:32.939-07:00', u'insertTime': u'2013-05-22T11:52:32.891-07:00', u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'operationType': u'insert', u'progress': 0, u'id': u'6663616273628949255', u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20', u'user': u'*****@*****.**' } fake_add_instance_request = flexmock(name='fake_add_instance_request') fake_add_instance_request.should_receive('execute').with_args( fake_authorized_http).and_return(add_instance_info) fake_instances.should_receive('insert').with_args(project=self.project, body=dict, zone=str).and_return(fake_add_instance_request) created_instance_info = { u'status': u'DONE' } fake_instance_checker = flexmock(name='fake_network_checker') fake_instance_checker.should_receive('execute').and_return( created_instance_info) fake_blocker = flexmock(name='fake_blocker') fake_gce.should_receive('globalOperations').and_return(fake_blocker) fake_blocker.should_receive('get').with_args(project=self.project, operation=add_instance).and_return(fake_instance_checker) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive('build').with_args('compute', GCEAgent.API_VERSION).and_return(fake_gce) i = InfrastructureManager(blocking=True) # first, validate that the run_instances call goes through successfully # and gives the user a reservation id full_params = { 'credentials' : { 'EC2_URL': None, 'EC2_ACCESS_KEY': None, 'EC2_SECRET_KEY': None, }, 'project': self.project, 'group': 'boogroup', 'image_id': 'booid', 'infrastructure': 'gce', 'instance_type': 'booinstance_type', 'keyname': 'bookeyname', 'num_vms': '1', 'use_spot_instances': False, } full_result = { 'success': True, 'reservation_id': self.reservation_id, 'reason': 'none' } self.assertEquals(full_result, i.run_instances(full_params, 'secret')) # next, look at run_instances internally to make sure it actually is # updating its reservation info self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get( self.reservation_id)['state']) vm_info = i.reservations.get(self.reservation_id)['vm_info'] self.assertEquals(['public-ip'], vm_info['public_ips']) self.assertEquals(['private-ip'], vm_info['private_ips']) self.assertEquals([instance_id], vm_info['instance_ids'])
def start_ec2_vms(self, params, blocking=False): ''' This method instantiates EC2 vm instances ''' logging.debug("start_ec2_vms : inside method with params : \n%s", pprint.pformat(params)) try: # make sure that any keynames we use are prefixed with stochss so that #we can do a terminate all based on keyname prefix key_prefix = AgentConfig.get_agent_key_prefix(agent_type=AgentTypes.EC2, key_prefix=params.get('key_prefix', '')) key_name = params["keyname"] if not key_name.startswith(key_prefix): params['keyname'] = key_prefix + key_name # NOTE: We are forcing blocking mode within the InfrastructureManager class # for the launching of VMs because of how GAE joins on all threads before # returning a response from a request. i = InfrastructureManager(blocking=blocking) res = {} # 1. change the status of 'failed' in the previous launch in db to 'terminated' # NOTE: We need to make sure that the RabbitMQ server is running if any compute # nodes are running as we are using the AMQP broker option for Celery. ins_ids = VMStateModel.terminate_not_active(params) # 2. get user_id, infra, ec2 credentials user_id = self.__get_required_parameter(parameter_key='user_id', params=params) infrastructure = self.__get_required_parameter(parameter_key='infrastructure', params=params) reservation_id = self.__get_required_parameter(parameter_key='reservation_id', params=params) logging.debug('ec2: reservation_id = {0}'.format(reservation_id)) if 'credentials' in params: if 'EC2_ACCESS_KEY' in params['credentials'] and 'EC2_SECRET_KEY' in params['credentials']: ec2_access_key = params['credentials']['EC2_ACCESS_KEY'] ec2_secret_key = params['credentials']['EC2_SECRET_KEY'] else: raise Exception('VMStateModel ERROR: Cannot get access key or secret.') else: raise Exception('VMStateModel ERROR: No credentials are provided.') if ec2_access_key is None or ec2_secret_key is None: raise Exception('VMStateModel ERROR: ec2 credentials are not valid.') # 3. create exact number of entities in db for this launch, and set the status to 'creating' num_vms = 0 if 'vms' in params: for vm in params['vms']: logging.debug('vm: {0}, num: {1}'.format(vm['instance_type'], vm['num_vms'])) num_vms += vm['num_vms'] if 'head_node' in params: num_vms += 1 logging.debug('num = {0}'.format(num_vms)) ids = self.__create_vm_state_model_entries(ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key, infrastructure=infrastructure, num_vms=num_vms, user_id=user_id, reservation_id=reservation_id) # 4. Prepare Instances params[VMStateModel.IDS] = ids res = i.prepare_instances(params) # 5, check and create stochss table exists if it does not exist self.__create_dynamodb_stochss_table(ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key) logging.debug("start_ec2_vms : exiting method with result : %s", str(res)) return True, None except Exception as e: logging.exception("start_ec2_vms : exiting method with error : {0}".format(str(e))) return False, 'Errors occur in starting machines:' + str(e)