def run_instances(self, prefix, blocking, success=True):
i = InfrastructureManager(blocking=blocking)
reservation = Reservation()
instance = flexmock(name='instance', private_dns_name='private-ip',
public_dns_name='public-ip', id='i-id', state='running',
key_name='bookeyname', ip_address='public-ip',
private_ip_address='private-ip')
new_instance = flexmock(name='new-instance', private_dns_name='new-private-ip',
public_dns_name='new-public-ip', id='new-i-id',
state='running', key_name='bookeyname',
ip_address='new-public-ip',
private_ip_address='new-private-ip')
reservation.instances = [instance]
new_reservation = Reservation()
new_reservation.instances = [instance, new_instance]
self.fake_ec2.should_receive('get_all_instances').and_return([]) \
.and_return([reservation]).and_return([new_reservation])
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials': {
'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': prefix,
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': False,
'region' : 'my-zone-1',
'zone' : 'my-zone-1b',
'autoscale_agent': True
}
id = '0000000000' # no longer randomly generated
full_result = {
'success': True,
'reservation_id': id,
'reason': 'none'
}
if success:
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
if not blocking:
time.sleep(.1)
if success:
self.assertEquals(InfrastructureManager.STATE_RUNNING,
i.reservations.get(id)['state'])
vm_info = i.reservations.get(id)['vm_info']
self.assertEquals(['new-public-ip'], vm_info['public_ips'])
self.assertEquals(['new-private-ip'], vm_info['private_ips'])
self.assertEquals(['new-i-id'], vm_info['instance_ids'])
else:
if blocking:
self.assertRaises(AgentRuntimeException, i.run_instances, full_params, 'secret')
def test_ec2_run_instances(self):
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials': {'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'ec2',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': 'True',
'max_spot_price' : '1.23',
'zone' : 'my-zone-1b'
}
id = '0000000000' # no longer randomly generated
full_result = {
'success': True,
'reservation_id': id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state'])
vm_info = i.reservations.get(id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals(['i-id'], vm_info['instance_ids'])
def terminate_instances(self, prefix, blocking):
i = InfrastructureManager(blocking=blocking)
params1 = {'infrastructure': prefix}
self.assertRaises(AgentConfigurationException, i.terminate_instances, params1, 'secret')
params2 = {
'credentials': {
'a': 'b', 'EC2_URL': 'http://ec2.url.com',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'infrastructure': prefix,
'instance_ids': ['i-12345'],
'region' : 'my-zone-1',
'keyname': 'bookeyname'
}
reservation = Reservation()
instance = flexmock(name='instance', private_dns_name='private-ip',
public_dns_name='public-ip', id='i-id', state='terminated',
key_name='bookeyname', ip_address='public-ip',
private_ip_address='private-ip')
reservation.instances = [instance]
self.fake_ec2.should_receive('get_all_instances').and_return([reservation])
flexmock(i).should_receive('_InfrastructureManager__kill_vms')
result = i.terminate_instances(params2, 'secret')
if not blocking:
time.sleep(.1)
self.assertTrue(result['success'])
def validateCredentials(self, params):
'''
This method verifies the validity of ec2 credentials
'''
if params['infrastructure'] is None:
logging.error("validateCredentials: infrastructure param not set")
return False
creds = params['credentials']
if creds is None:
logging.error("validateCredentials: credentials param not set")
return False
if creds['EC2_ACCESS_KEY'] is None:
logging.error("validateCredentials: credentials EC2_ACCESS_KEY not set")
return False
if creds['EC2_SECRET_KEY'] is None:
logging.error("validateCredentials: credentials EC2_ACCESS_KEY not set")
return False
logging.debug("validateCredentials: inside method with params : %s", str(params))
try:
i = InfrastructureManager()
logging.debug("validateCredentials: exiting with result : %s", str(i))
return i.validate_credentials(params)
except Exception, e:
logging.error("validateCredentials: exiting with error : %s", str(e))
return False
def test_euca_run_instances(self):
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials': {
'a': 'b',
'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key',
'EC2_SECRET_KEY': 'secret_key'
},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'euca',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '2',
'use_spot_instances': False,
}
id = '0000000000' # no longer randomly generated
full_result = {'success': True, 'reservation_id': id, 'reason': 'none'}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING,
i.reservations.get(id)['state'])
vm_info = i.reservations.get(id)['vm_info']
self.assertEquals(['ABC-public-ip1', 'DEF-public-ip2'],
vm_info['public_ips'])
self.assertEquals(['DEF-private-ip1', 'ABC-private-ip2'],
vm_info['private_ips'])
self.assertEquals(['i-id1', 'i-id2'], vm_info['instance_ids'])
def test_euca_run_instances(self):
i = InfrastructureManager(blocking=True)
reservation = Reservation()
instance = flexmock(name='instance',
private_dns_name='private-ip',
public_dns_name='public-ip',
id='i-id',
state='running',
key_name='bookeyname',
ip_address='public-ip',
private_ip_address='private-ip')
new_instance = flexmock(name='new-instance',
private_dns_name='new-private-ip',
public_dns_name='new-public-ip',
id='new-i-id',
state='running',
key_name='bookeyname',
ip_address='new-public-ip',
private_ip_address='new-private-ip')
reservation.instances = [instance]
new_reservation = Reservation()
new_reservation.instances = [instance, new_instance]
flexmock(EC2Connection).should_receive('get_all_instances').and_return([]) \
.and_return([reservation]).and_return([reservation]) \
.and_return([new_reservation]).and_return([new_reservation])
# first, validate that the run_instances call goes through successfully
# and gives the user an operation id
full_params = {
'credentials': {
'a': 'b',
'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key',
'EC2_SECRET_KEY': 'secret_key'
},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'euca',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': False,
'zone': 'my-zone-1b',
'autoscale_agent': True,
'IS_VERBOSE': True
}
id = '0000000000' # no longer randomly generated
full_result = {'success': True, 'operation_id': id, 'reason': 'none'}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its operation info
self.assertEquals(InfrastructureManager.STATE_SUCCESS,
i.operation_ids.get(id)['state'])
vm_info = i.operation_ids.get(id)['vm_info']
self.assertEquals(['new-public-ip'], vm_info['public_ips'])
self.assertEquals(['new-private-ip'], vm_info['private_ips'])
self.assertEquals(['new-i-id'], vm_info['instance_ids'])
def deregister_flex_cloud(self, parameters, blocking=True):
try:
i = InfrastructureManager(blocking=blocking)
res = i.deregister_instances(parameters=parameters, terminate=False)
ret = True
except Exception, e:
logging.error("deregister_flex_cloud() failed with error : %s", str(e))
ret = False
def validateCredentials(self, params):
'''
This method verifies the validity of ec2 credentials
'''
logging.info("validateCredentials: inside method with params : %s", str(params))
try:
i = InfrastructureManager()
logging.info("validateCredentials: exiting with result : %s", str(i))
return i.validate_Credentials(params)
except Exception, e:
logging.error("validateCredentials: exiting with error : %s", str(e))
return False
def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials')
fake_credentials.should_receive('invalid').and_return(False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_attach_disk_request = flexmock(name='fake_attach_disk_request')
fake_instances.should_receive('get').and_return(fake_attach_disk_request)
attach_disk_info = {
'status': 'DONE',
'disks': []
}
fake_attach_disk_request.should_receive('execute').with_args(
fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive('attachDisk').with_args(project=self.project,
body=dict, instance='my-instance', zone=str).and_return(
fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = 'my-disk-name'
instance_id = 'my-instance'
expected = '/dev/sdb'
actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret')
self.assertTrue(actual['success'])
self.assertEquals(expected, actual['location'])
def stopMachines(self, params, block=False):
"""
This method would terminate all the instances associated with the account
that have a keyname prefixed with stochss (all instances created by the backend service)
params must contain credentials key/value
"""
try:
i = InfrastructureManager(blocking=block)
res = i.terminate_instances(params, backendservices.KEYPREFIX)
return True
except Exception, e:
logging.error("Terminate machine failed with error : %s", str(e))
return False
def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials', invalid=False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_attach_disk_request = flexmock(name='fake_attach_disk_request')
fake_instances.should_receive('get').and_return(fake_attach_disk_request)
attach_disk_info = {
'status': 'DONE',
'disks': []
}
fake_attach_disk_request.should_receive('execute').with_args(
fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive('attachDisk').with_args(project=self.project,
body=dict, instance='my-instance', zone=str).and_return(
fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = 'my-disk-name'
instance_id = 'my-instance'
expected = '/dev/sdb'
actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret')
self.assertTrue(actual['success'])
self.assertEquals(expected, actual['location'])
def describe_machines_from_db(self, infrastructure, force=False):
parameters = {
"infrastructure": infrastructure,
"credentials": self.get_credentials(),
"key_prefix": self.user_data.user_id,
"user_id": self.user_data.user_id,
}
if infrastructure == AgentTypes.FLEX:
parameters['flex_cloud_machine_info'] = self.user_data.get_flex_cloud_machine_info()
parameters['reservation_id'] = self.user_data.reservation_id
i = InfrastructureManager()
i.synchronize_db(parameters, force=force)
all_vms = VMStateModel.get_all(parameters)
return all_vms
def startMachines(self, params, block=False):
'''
This method instantiates ec2 instances
'''
logging.info("startMachines : inside method with params : %s", str(params))
try:
#make sure that any keynames we use are prefixed with stochss so that
#we can do a terminate all based on keyname prefix
key_name = params["keyname"]
if not key_name.startswith(self.KEYPREFIX):
params['keyname'] = self.KEYPREFIX + key_name
# NOTE: We are forcing blocking mode within the InfrastructureManager class
# for the launching of VMs because of how GAE joins on all threads before
# returning a response from a request.
i = InfrastructureManager(blocking=block)
res = {}
# NOTE: We need to make sure that the RabbitMQ server is running if any compute
# nodes are running as we are using the AMQP broker option for Celery.
compute_check_params = {
"credentials": params["credentials"],
"key_prefix": params["key_prefix"]
}
if self.isQueueHeadRunning(compute_check_params):
res = i.run_instances(params,[])
else:
# Need to start the queue head (RabbitMQ)
params["queue_head"] = True
vms_requested = int(params["num_vms"])
requested_key_name = params["keyname"]
# Only want one queue head, and it must have its own key so
# it can be differentiated if necessary
params["num_vms"] = 1
params["keyname"] = requested_key_name+'-'+self.QUEUEHEAD_KEY_TAG
res = i.run_instances(params,[])
#NOTE: This relies on the InfrastructureManager being run in blocking mode...
queue_head_ip = res["vm_info"]["public_ips"][0]
self.__update_celery_config_with_queue_head_ip(queue_head_ip)
params["keyname"] = requested_key_name
params["queue_head"] = False
if vms_requested > 1:
params["num_vms"] = vms_requested - 1
res = i.run_instances(params,[])
params["num_vms"] = vms_requested
logging.info("startMachines : exiting method with result : %s", str(res))
return res
except Exception, e:
logging.error("startMachines : exiting method with error : {0}".format(str(e)))
print "startMachines : exiting method with error :", str(e)
return None
def describeMachines(self, params):
"""
This method gets the status of all the instances of ec2
"""
# add calls to the infrastructure manager for getting details of
# machines
logging.info("describeMachines : inside method with params : %s", str(params))
try:
i = InfrastructureManager()
res = i.describe_instances(params, [], backendservices.KEYPREFIX)
logging.info("describeMachines : exiting method with result : %s", str(res))
return res
except Exception, e:
logging.error("describeMachines : exiting method with error : %s", str(e))
return None
def describeMachines(self, params):
'''
This method gets the status of all the instances of ec2
'''
# add calls to the infrastructure manager for getting details of
# machines
logging.info("describeMachines : inside method with params : %s", str(params))
try:
i = InfrastructureManager()
secret =[]
res = i.describe_instances(params, secret)
logging.info("describeMachines : exiting method with result : %s", str(res))
return res
except Exception, e:
logging.error("describeMachines : exiting method with error : %s", str(e))
return None
def startMachines(self, params):
'''
This method instantiates ec2 instances
'''
#this will basically start an instance in ec2
# add call from the infrastructure manager here
logging.info("startMachines : inside method with params : %s", str(params))
try:
i = InfrastructureManager(blocking=True)
res = i.run_instances(params,params)
return res
logging.info("startMachines : exiting method with result : %s", str(res))
except Exception, e:
logging.error("startMachines : exiting method with error : %s", str(e))
return None
def stopMachines(self, params):
'''
This method would terminate all the instances associated with the account
It expects the following two fields in the parameter argument
params ={"infrastructure":"ec2",
'credentials':{"EC2_ACCESS_KEY":"______________",
"EC2_SECRET_KEY":"__________"},
}
'''
try:
i = InfrastructureManager(blocking=True)
res = i.terminate_instances(params)
print str(res)
return True
except Exception, e:
logging.error("Terminate machine failed with error : %s", str(e))
return False
def stop_ec2_vms(self, params, blocking=False):
'''
This method would terminate all the EC2 instances associated with the account
that have a keyname prefixed with stochss (all instances created by the backend service)
params must contain credentials key/value
'''
key_prefix = AgentConfig.get_agent_key_prefix(agent_type=AgentTypes.EC2,
key_prefix=params.get('key_prefix', ''))
try:
logging.debug("Stopping compute nodes with key_prefix: {0}".format(key_prefix))
i = InfrastructureManager(blocking=blocking)
res = i.deregister_instances(parameters=params, terminate=True)
ret = True
except Exception, e:
logging.error("Terminate machine failed with error : %s", str(e))
ret = False
def terminate_instances(self, prefix, blocking):
i = InfrastructureManager(blocking=blocking)
params1 = {'infrastructure': prefix}
result1 = i.terminate_instances(params1, 'secret')
self.assertFalse(result1['success'])
self.assertEquals(result1['reason'], 'no credentials')
params2 = {
'credentials': {'a': 'b', 'EC2_URL': 'http://ec2.url.com',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'infrastructure': prefix,
'instance_ids': ['i-12345']
}
result2 = i.terminate_instances(params2, 'secret')
if not blocking:
time.sleep(2)
self.assertTrue(result2['success'])
def stopMachines(self, params, block=False):
'''
This method would terminate all the instances associated with the account
that have a keyname prefixed with stochss (all instances created by the backend service)
params must contain credentials key/value
'''
key_prefix = self.KEYPREFIX
if "key_prefix" in params and not params["key_prefix"].startswith(key_prefix):
key_prefix += params["key_prefix"]
elif "key_prefix" in params:
key_prefix = params["key_prefix"]
try:
logging.info("Stopping compute nodes with key_prefix: {0}".format(key_prefix))
i = InfrastructureManager(blocking=block)
res = i.terminate_instances(params,key_prefix)
return True
except Exception, e:
logging.error("Terminate machine failed with error : %s", str(e))
return False
def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name="fake_credentials")
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
fake_instances = flexmock(name="fake_instances")
fake_gce = flexmock(name="fake_gce")
fake_gce.should_receive("instances").and_return(fake_instances)
attach_disk_info = {"status": "DONE"}
fake_attach_disk_request = flexmock(name="fake_attach_disk_request")
fake_attach_disk_request.should_receive("execute").with_args(fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive("attachDisk").with_args(
project=self.project, body=dict, instance="my-instance", zone=str
).and_return(fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = "my-disk-name"
instance_id = "my-instance"
expected = "/dev/sdb"
actual = iaas.attach_disk(self.params, disk_name, instance_id, "secret")
self.assertTrue(actual["success"])
self.assertEquals(expected, actual["location"])
def run_instances(self, prefix, blocking, success=True):
i = InfrastructureManager(blocking=blocking)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials': {
'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': prefix,
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': False,
'region' : 'my-zone-1',
'zone' : 'my-zone-1b'
}
id = '0000000000' # no longer randomly generated
full_result = {
'success': True,
'reservation_id': id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
if not blocking:
time.sleep(.1)
if success:
self.assertEquals(InfrastructureManager.STATE_RUNNING,
i.reservations.get(id)['state'])
vm_info = i.reservations.get(id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals(['i-id'], vm_info['instance_ids'])
else:
self.assertEquals(InfrastructureManager.STATE_FAILED,
i.reservations.get(id)['state'])
def terminate_instances(self, prefix, blocking):
i = InfrastructureManager(blocking=blocking)
params1 = {'infrastructure': prefix}
result1 = i.terminate_instances(params1, 'secret')
self.assertFalse(result1['success'])
self.assertEquals(result1['reason'], 'no credentials')
params2 = {
'credentials': {
'a': 'b', 'EC2_URL': 'http://ec2.url.com',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'infrastructure': prefix,
'instance_ids': ['i-12345'],
'region' : 'my-zone-1'
}
result2 = i.terminate_instances(params2, 'secret')
if not blocking:
time.sleep(.1)
self.assertTrue(result2['success'])
def prepare_flex_cloud_machines(self, params, blocking=False):
logging.debug("prepare_flex_cloud_machines : params : \n%s", pprint.pformat(params))
try:
# NOTE: We are forcing blocking mode within the InfrastructureManager class
# for the launching of VMs because of how GAE joins on all threads before
# returning a response from a request.
i = InfrastructureManager(blocking=blocking)
res = {}
# 4. Prepare Instances
res = i.prepare_instances(params)
logging.debug("prepare_flex_cloud_machines : exiting method with result : %s", str(res))
return True, ''
except Exception, e:
traceback.print_exc()
logging.error("prepare_flex_cloud_machines : exiting method with error : {0}".format(str(e)))
return False, 'Errors occur in preparing machines:' + str(e)
def describeMachines(self, params):
'''
This method gets the status of all the instances of ec2
'''
# add calls to the infrastructure manager for getting details of
# machines
logging.info("describeMachines : inside method with params : %s", str(params))
key_prefix = ""
if "key_prefix" in params:
key_prefix = params["key_prefix"]
if not key_prefix.startswith(self.KEYPREFIX):
key_prefix = self.KEYPREFIX + key_prefix
else:
key_prefix = self.KEYPREFIX
try:
i = InfrastructureManager()
res = i.describe_instances(params, [], key_prefix)
logging.info("describeMachines : exiting method with result : %s", str(res))
return res
except Exception, e:
logging.error("describeMachines : exiting method with error : %s", str(e))
return None
def startMachines(self, params, block=False):
"""
This method instantiates ec2 instances
"""
logging.info("startMachines : inside method with params : %s", str(params))
try:
keyname = params["keyname"]
# make sure that any keynames we use are prefixed with stochss so that
# we can do a terminate all based on keyname prefix
if not keyname.startswith(backendservices.KEYPREFIX):
params["keyname"] = backendservices.KEYPREFIX + keyname
# NOTE: We are forcing blocking mode within the InfrastructureManager class
# for the launching of VMs because of how GAE joins on all threads before
# returning a response from a request.
i = InfrastructureManager(blocking=block)
res = i.run_instances(params, [])
logging.info("startMachines : exiting method with result : %s", str(res))
return res
except Exception, e:
logging.error("startMachines : exiting method with error : %s", str(e))
return None
def run_instances(self, prefix, blocking, success=True):
i = InfrastructureManager(blocking=blocking)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials': {'a': 'b', 'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key', 'EC2_SECRET_KEY': 'secret_key'},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': prefix,
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'spot': 'False',
}
id = '0000000000' # no longer randomly generated
full_result = {
'success': True,
'reservation_id': id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
if not blocking:
time.sleep(2)
if success:
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(id)['state'])
vm_info = i.reservations.get(id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals(['i-id'], vm_info['instance_ids'])
else:
self.assertEquals(InfrastructureManager.STATE_FAILED, i.reservations.get(id)['state'])
def test_ec2_run_instances(self):
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user an operation id
full_params = {
'credentials': {
'a': 'b',
'EC2_URL': 'http://testing.appscale.com:8773/foo/bar',
'EC2_ACCESS_KEY': 'access_key',
'EC2_SECRET_KEY': 'secret_key'
},
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'ec2',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': 'True',
'max_spot_price': '1.23',
'region': 'my-zone-1',
'zone': 'my-zone-1b',
'autoscale_agent': True
}
id = '0000000000' # no longer randomly generated
full_result = {'success': True, 'operation_id': id, 'reason': 'none'}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its operation info
self.assertEquals(InfrastructureManager.STATE_SUCCESS,
i.operation_ids.get(id)['state'])
vm_info = i.operation_ids.get(id)['vm_info']
self.assertEquals(['new-public-ip'], vm_info['public_ips'])
self.assertEquals(['new-private-ip'], vm_info['private_ips'])
self.assertEquals(['new-i-id'], vm_info['instance_ids'])
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials')
fake_credentials.should_receive('invalid').and_return(False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {
}
instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d'
list_instance_info = {
u'items': [{
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': instance_id,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public-ip'
}],
u'networkIP': u'private-ip',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup',
u'name': u'nic0'
}]
}],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(no_instance_info).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_instances.should_receive('list').with_args(project=self.project,
filter="name eq boogroup-.*", zone='my-zone-1b') \
.and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u'operation-1369248752891-4dd5311848461-afc55a20'
add_instance_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': add_instance,
u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-22T11:52:32.939-07:00',
u'insertTime': u'2013-05-22T11:52:32.891-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'operationType': u'insert',
u'progress': 0,
u'id': u'6663616273628949255',
u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20',
u'user': u'*****@*****.**'
}
fake_add_instance_request = flexmock(name='fake_add_instance_request')
fake_add_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(add_instance_info)
fake_instances.should_receive('insert').with_args(project=self.project,
body=dict, zone=str).and_return(fake_add_instance_request)
created_instance_info = {
u'status': u'DONE'
}
fake_instance_checker = flexmock(name='fake_network_checker')
fake_instance_checker.should_receive('execute').and_return(
created_instance_info)
fake_blocker = flexmock(name='fake_blocker')
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
fake_blocker.should_receive('get').with_args(project=self.project,
operation=add_instance).and_return(fake_instance_checker)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
# next, presume that the persistent disk we want to use exists
disk_name = 'my-persistent-disk-1'
disk_info = {'status':'DONE'}
fake_disk_request = flexmock(name='fake_disk_request')
fake_disk_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(disk_info)
fake_disks = flexmock(name='fake_disks')
fake_disks.should_receive('get').with_args(project=self.project,
disk=disk_name, zone=str).and_return(fake_disk_request)
fake_disks.should_receive('insert').with_args(project=self.project,
sourceImage=str, body=dict, zone=str).and_return(fake_disk_request)
fake_gce.should_receive('disks').and_return(fake_disks)
public_key = 'ssh-rsa long_key_string'
flexmock(utils).should_receive('get_public_key').and_return(public_key)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_result = {
'success': True,
'reservation_id': self.reservation_id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(self.params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(
self.reservation_id)['state'])
vm_info = i.reservations.get(self.reservation_id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals([instance_id], vm_info['instance_ids'])
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name="fake_credentials")
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {}
instance_id = u"appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d"
list_instance_info = {
u"items": [
{
u"status": u"RUNNING",
u"kind": u"compute#instance",
u"machineType": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1",
u"name": instance_id,
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"tags": {u"fingerprint": u"42WmSpB8rSM="},
u"image": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64",
u"disks": [
{u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"}
],
u"canIpForward": False,
u"serviceAccounts": [
{u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"}
],
u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="},
u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00",
u"id": u"8684033495853907982",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"networkInterfaces": [
{
u"accessConfigs": [
{
u"kind": u"compute#accessConfig",
u"type": u"ONE_TO_ONE_NAT",
u"name": u"External NAT",
u"natIP": u"public-ip",
}
],
u"networkIP": u"private-ip",
u"network": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup",
u"name": u"nic0",
}
],
}
],
u"kind": u"compute#instanceList",
u"id": u"projects/appscale.com:appscale/zones/my-zone-1b/instances",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances",
}
fake_list_instance_request = flexmock(name="fake_list_instance_request")
fake_list_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
no_instance_info
).and_return(list_instance_info)
fake_instances = flexmock(name="fake_instances")
fake_gce = flexmock(name="fake_gce")
fake_gce.should_receive("instances").and_return(fake_instances)
fake_instances.should_receive("list").with_args(
project=self.project, filter="name eq appscale-boogroup-.*", zone="my-zone-1b"
).and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u"operation-1369248752891-4dd5311848461-afc55a20"
add_instance_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": add_instance,
u"azone": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/my-zone-1b",
u"startTime": u"2013-05-22T11:52:32.939-07:00",
u"insertTime": u"2013-05-22T11:52:32.891-07:00",
u"targetLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"operationType": u"insert",
u"progress": 0,
u"id": u"6663616273628949255",
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20",
u"user": u"*****@*****.**",
}
fake_add_instance_request = flexmock(name="fake_add_instance_request")
fake_add_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
add_instance_info
)
fake_instances.should_receive("insert").with_args(project=self.project, body=dict, zone=str).and_return(
fake_add_instance_request
)
created_instance_info = {u"status": u"DONE"}
fake_instance_checker = flexmock(name="fake_network_checker")
fake_instance_checker.should_receive("execute").and_return(created_instance_info)
fake_blocker = flexmock(name="fake_blocker")
fake_gce.should_receive("globalOperations").and_return(fake_blocker)
fake_blocker.should_receive("get").with_args(project=self.project, operation=add_instance).and_return(
fake_instance_checker
)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_result = {"success": True, "reservation_id": self.reservation_id, "reason": "none"}
self.assertEquals(full_result, i.run_instances(self.params, "secret"))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(self.reservation_id)["state"])
vm_info = i.reservations.get(self.reservation_id)["vm_info"]
self.assertEquals(["public-ip"], vm_info["public_ips"])
self.assertEquals(["private-ip"], vm_info["private_ips"])
self.assertEquals([instance_id], vm_info["instance_ids"])
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, ssl=True):
"""
Initialize a new instance of the infrastructure manager service.
Args:
host Hostname to which the service should bind (Optional). Defaults
to 0.0.0.0.
port Port of the service (Optional). Default to 17444.
ssl True if SSL should be engaged or False otherwise (Optional).
Defaults to True. When engaged, this implementation expects
to find the necessary SSL certificates in the /etc/appscale/certs
directory.
"""
self.host = host
self.port = port
secret = None
while True:
try:
secret = utils.get_secret(self.APPSCALE_DIR + 'secret.key')
break
except Exception:
logging.info('Waiting for the secret key to become available')
utils.sleep(5)
logging.info('Found the secret set to: {0}'.format(secret))
SOAPpy.Config.simplify_objects = True
if ssl:
logging.info('Checking for the certificate and private key')
cert = self.APPSCALE_DIR + 'certs/mycert.pem'
key = self.APPSCALE_DIR + 'certs/mykey.pem'
while True:
if os.path.exists(cert) and os.path.exists(key):
break
else:
logging.info('Waiting for certificates')
utils.sleep(5)
ssl_context = SSL.Context()
ssl_context.load_cert(cert, key)
self.server = SOAPpy.SOAPServer((host, port),
ssl_context=ssl_context)
else:
self.server = SOAPpy.SOAPServer((host, port))
parent_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
config_file = os.path.join(parent_dir, self.CONFIG_FILE)
if os.path.exists(config_file):
with open(config_file) as file_handle:
params = json.load(file_handle)
if params.has_key(PersistentStoreFactory.PARAM_STORE_TYPE):
logging.info(
'Loading infrastructure manager configuration from ' +
config_file)
i = InfrastructureManager(params)
else:
i = InfrastructureManager()
else:
i = InfrastructureManager()
self.server.registerFunction(i.describe_operation)
self.server.registerFunction(i.run_instances)
self.server.registerFunction(i.terminate_instances)
self.server.registerFunction(i.attach_disk)
system_manager = SystemManager()
self.server.registerFunction(system_manager.get_cpu_usage)
self.server.registerFunction(system_manager.get_disk_usage)
self.server.registerFunction(system_manager.get_memory_usage)
self.server.registerFunction(system_manager.get_service_summary)
self.server.registerFunction(system_manager.get_swap_usage)
self.server.registerFunction(system_manager.get_loadavg)
self.started = False
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive('flow_from_clientsecrets').with_args(
GCEAgent.CLIENT_SECRETS_LOCATION, scope=GCEAgent.GCE_SCOPE).and_return(
fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run').with_args(fake_flow,
fake_storage).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {
}
instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d'
list_instance_info = {
u'items': [{
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': instance_id,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public-ip'
}],
u'networkIP': u'private-ip',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup',
u'name': u'nic0'
}]
}],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/us-central1-a/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/us-central1-a/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
fake_authorized_http).and_return(no_instance_info).and_return(
list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_instances.should_receive('list').with_args(project=self.project,
filter="name eq appscale-boogroup-.*", zone=GCEAgent.DEFAULT_ZONE) \
.and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u'operation-1369248752891-4dd5311848461-afc55a20'
add_instance_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': add_instance,
u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a',
u'startTime': u'2013-05-22T11:52:32.939-07:00',
u'insertTime': u'2013-05-22T11:52:32.891-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'operationType': u'insert',
u'progress': 0,
u'id': u'6663616273628949255',
u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20',
u'user': u'*****@*****.**'
}
fake_add_instance_request = flexmock(name='fake_add_instance_request')
fake_add_instance_request.should_receive('execute').with_args(
fake_authorized_http).and_return(add_instance_info)
fake_instances.should_receive('insert').with_args(project=self.project,
body=dict, zone=str).and_return(fake_add_instance_request)
created_instance_info = {
u'status': u'DONE'
}
fake_instance_checker = flexmock(name='fake_network_checker')
fake_instance_checker.should_receive('execute').and_return(
created_instance_info)
fake_blocker = flexmock(name='fake_blocker')
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
fake_blocker.should_receive('get').with_args(project=self.project,
operation=add_instance).and_return(fake_instance_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials' : {
'EC2_URL': None,
'EC2_ACCESS_KEY': None,
'EC2_SECRET_KEY': None,
},
'project': self.project,
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'gce',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': False,
}
full_result = {
'success': True,
'reservation_id': self.reservation_id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(
self.reservation_id)['state'])
vm_info = i.reservations.get(self.reservation_id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals([instance_id], vm_info['instance_ids'])
def start_ec2_vms(self, params, blocking=False):
'''
This method instantiates EC2 vm instances
'''
logging.debug("start_ec2_vms : inside method with params : \n%s", pprint.pformat(params))
try:
# make sure that any keynames we use are prefixed with stochss so that
#we can do a terminate all based on keyname prefix
key_prefix = AgentConfig.get_agent_key_prefix(agent_type=AgentTypes.EC2,
key_prefix=params.get('key_prefix', ''))
key_name = params["keyname"]
if not key_name.startswith(key_prefix):
params['keyname'] = key_prefix + key_name
# NOTE: We are forcing blocking mode within the InfrastructureManager class
# for the launching of VMs because of how GAE joins on all threads before
# returning a response from a request.
i = InfrastructureManager(blocking=blocking)
res = {}
# 1. change the status of 'failed' in the previous launch in db to 'terminated'
# NOTE: We need to make sure that the RabbitMQ server is running if any compute
# nodes are running as we are using the AMQP broker option for Celery.
ins_ids = VMStateModel.terminate_not_active(params)
# 2. get user_id, infra, ec2 credentials
user_id = self.__get_required_parameter(parameter_key='user_id', params=params)
infrastructure = self.__get_required_parameter(parameter_key='infrastructure', params=params)
reservation_id = self.__get_required_parameter(parameter_key='reservation_id', params=params)
logging.debug('ec2: reservation_id = {0}'.format(reservation_id))
if 'credentials' in params:
if 'EC2_ACCESS_KEY' in params['credentials'] and 'EC2_SECRET_KEY' in params['credentials']:
ec2_access_key = params['credentials']['EC2_ACCESS_KEY']
ec2_secret_key = params['credentials']['EC2_SECRET_KEY']
else:
raise Exception('VMStateModel ERROR: Cannot get access key or secret.')
else:
raise Exception('VMStateModel ERROR: No credentials are provided.')
if ec2_access_key is None or ec2_secret_key is None:
raise Exception('VMStateModel ERROR: ec2 credentials are not valid.')
# 3. create exact number of entities in db for this launch, and set the status to 'creating'
num_vms = 0
if 'vms' in params:
for vm in params['vms']:
logging.debug('vm: {0}, num: {1}'.format(vm['instance_type'], vm['num_vms']))
num_vms += vm['num_vms']
if 'head_node' in params:
num_vms += 1
logging.debug('num = {0}'.format(num_vms))
ids = self.__create_vm_state_model_entries(ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key,
infrastructure=infrastructure, num_vms=num_vms, user_id=user_id,
reservation_id=reservation_id)
# 4. Prepare Instances
params[VMStateModel.IDS] = ids
res = i.prepare_instances(params)
# 5, check and create stochss table exists if it does not exist
self.__create_dynamodb_stochss_table(ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key)
logging.debug("start_ec2_vms : exiting method with result : %s", str(res))
return True, None
except Exception as e:
logging.exception("start_ec2_vms : exiting method with error : {0}".format(str(e)))
return False, 'Errors occur in starting machines:' + str(e)
