def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials')
fake_credentials.should_receive('invalid').and_return(False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_attach_disk_request = flexmock(name='fake_attach_disk_request')
fake_instances.should_receive('get').and_return(fake_attach_disk_request)
attach_disk_info = {
'status': 'DONE',
'disks': []
}
fake_attach_disk_request.should_receive('execute').with_args(
fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive('attachDisk').with_args(project=self.project,
body=dict, instance='my-instance', zone=str).and_return(
fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = 'my-disk-name'
instance_id = 'my-instance'
expected = '/dev/sdb'
actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret')
self.assertTrue(actual['success'])
self.assertEquals(expected, actual['location'])
def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials', invalid=False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_attach_disk_request = flexmock(name='fake_attach_disk_request')
fake_instances.should_receive('get').and_return(fake_attach_disk_request)
attach_disk_info = {
'status': 'DONE',
'disks': []
}
fake_attach_disk_request.should_receive('execute').with_args(
fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive('attachDisk').with_args(project=self.project,
body=dict, instance='my-instance', zone=str).and_return(
fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = 'my-disk-name'
instance_id = 'my-instance'
expected = '/dev/sdb'
actual = iaas.attach_disk(self.params, disk_name, instance_id, 'secret')
self.assertTrue(actual['success'])
self.assertEquals(expected, actual['location'])
def test_attach_persistent_disk(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name="fake_credentials")
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
fake_instances = flexmock(name="fake_instances")
fake_gce = flexmock(name="fake_gce")
fake_gce.should_receive("instances").and_return(fake_instances)
attach_disk_info = {"status": "DONE"}
fake_attach_disk_request = flexmock(name="fake_attach_disk_request")
fake_attach_disk_request.should_receive("execute").with_args(fake_authorized_http).and_return(attach_disk_info)
fake_instances.should_receive("attachDisk").with_args(
project=self.project, body=dict, instance="my-instance", zone=str
).and_return(fake_attach_disk_request)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce)
iaas = InfrastructureManager(blocking=True)
disk_name = "my-disk-name"
instance_id = "my-instance"
expected = "/dev/sdb"
actual = iaas.attach_disk(self.params, disk_name, instance_id, "secret")
self.assertTrue(actual["success"])
self.assertEquals(expected, actual["location"])
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name='fake_credentials')
fake_credentials.should_receive('invalid').and_return(False)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {
}
instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d'
list_instance_info = {
u'items': [{
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': instance_id,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public-ip'
}],
u'networkIP': u'private-ip',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup',
u'name': u'nic0'
}]
}],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(no_instance_info).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_instances.should_receive('list').with_args(project=self.project,
filter="name eq boogroup-.*", zone='my-zone-1b') \
.and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u'operation-1369248752891-4dd5311848461-afc55a20'
add_instance_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': add_instance,
u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-22T11:52:32.939-07:00',
u'insertTime': u'2013-05-22T11:52:32.891-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'operationType': u'insert',
u'progress': 0,
u'id': u'6663616273628949255',
u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20',
u'user': u'*****@*****.**'
}
fake_add_instance_request = flexmock(name='fake_add_instance_request')
fake_add_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(add_instance_info)
fake_instances.should_receive('insert').with_args(project=self.project,
body=dict, zone=str).and_return(fake_add_instance_request)
created_instance_info = {
u'status': u'DONE'
}
fake_instance_checker = flexmock(name='fake_network_checker')
fake_instance_checker.should_receive('execute').and_return(
created_instance_info)
fake_blocker = flexmock(name='fake_blocker')
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
fake_blocker.should_receive('get').with_args(project=self.project,
operation=add_instance).and_return(fake_instance_checker)
# finally, inject our fake GCE connection
flexmock(discovery)
discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
# next, presume that the persistent disk we want to use exists
disk_name = 'my-persistent-disk-1'
disk_info = {'status':'DONE'}
fake_disk_request = flexmock(name='fake_disk_request')
fake_disk_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(disk_info)
fake_disks = flexmock(name='fake_disks')
fake_disks.should_receive('get').with_args(project=self.project,
disk=disk_name, zone=str).and_return(fake_disk_request)
fake_disks.should_receive('insert').with_args(project=self.project,
sourceImage=str, body=dict, zone=str).and_return(fake_disk_request)
fake_gce.should_receive('disks').and_return(fake_disks)
public_key = 'ssh-rsa long_key_string'
flexmock(utils).should_receive('get_public_key').and_return(public_key)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_result = {
'success': True,
'reservation_id': self.reservation_id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(self.params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(
self.reservation_id)['state'])
vm_info = i.reservations.get(self.reservation_id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals([instance_id], vm_info['instance_ids'])
def test_appscale_in_one_node_gce_deployment(self):
# presume that our client_secrets file exists
project_id = "appscale.com:appscale"
client_secrets = "/boo/client_secrets.json"
instance_type = 'n1-standard-8'
zone = 'my-zone1-b'
os.path.should_receive('exists').with_args(client_secrets).and_return(True)
# and that the user does not have an ssh key set up, forcing us to create
# one for them
private_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH,
self.keyname)
public_key = '{0}{1}.pub'.format(LocalState.LOCAL_APPSCALE_PATH,
self.keyname)
os.path.should_receive('exists').with_args(private_key).and_return(False)
os.path.should_receive('exists').with_args(public_key).and_return(False)
self.local_state.should_receive('shell').with_args(re.compile('^ssh-keygen'), False).and_return()
flexmock(os)
original_private_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname
os.should_receive('chmod').with_args(original_private_key, 0600)
os.should_receive('chmod').with_args(public_key, 0600)
flexmock(shutil)
shutil.should_receive('copy').with_args(original_private_key, private_key)
# also, we should be able to copy over our secret.json file fine
shutil.should_receive('copy').with_args(client_secrets,
LocalState.get_client_secrets_location(self.keyname))
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive('flow_from_clientsecrets').with_args(
client_secrets, scope=str).and_return(fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(str).and_return(
fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run').with_args(fake_flow,
fake_storage).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
# presume that there is an ssh key stored, but it isn't ours
metadata_info = {
u'kind': u'compute#project',
u'description': u'',
u'commonInstanceMetadata': {
u'items': [{
u'value': u'cgb:ssh-rsa keyinfo myhost',
u'key': u'sshKeys'}],
u'kind': u'compute#metadata'},
}
fake_metadata_request = flexmock(name='fake_metadata_request')
fake_metadata_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(metadata_info)
fake_projects = flexmock(name='fake_projects')
fake_projects.should_receive('get').with_args(project=project_id) \
.and_return(fake_metadata_request)
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('projects').and_return(fake_projects)
# thus we will need to set the metadata with our ssh key
fake_ssh_pub_key = flexmock(name="fake_ssh_pub_key")
fake_ssh_pub_key.should_receive('read').and_return('ssh-rsa key2info myhost')
self.builtins.should_receive('open').with_args(public_key).and_return(
fake_ssh_pub_key)
new_metadata_body = {
"items": [{
"value" : u'cgb:ssh-rsa key2info myhost\ncgb:ssh-rsa keyinfo myhost',
"key" : "sshKeys"
}],
"kind": "compute#metadata"
}
set_metadata_name = u'operation-222222-4dd41ec7d6c11-8013657f'
set_metadata = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': set_metadata_name,
u'operationType': u'insert',
u'progress': 0,
u'selfLink': unicode(GCEAgent.GCE_URL) + \
u'appscale.com:appscale/global/operations/' + \
u'operation-1369175117235-4dd41ec7d6c11-8013657f',
u'user': u'*****@*****.**'
}
fake_set_metadata_request = flexmock(name='fake_set_metadata_request')
fake_set_metadata_request.should_receive('execute').and_return(set_metadata)
fake_projects.should_receive('setCommonInstanceMetadata').with_args(
project=project_id, body=dict).and_return(fake_set_metadata_request)
updated_metadata_info = {
u'status': u'DONE'
}
fake_metadata_checker = flexmock(name='fake_network_checker')
fake_metadata_checker.should_receive('execute').and_return(
updated_metadata_info)
fake_blocker = flexmock(name='fake_blocker')
fake_blocker.should_receive('get').with_args(project=project_id,
operation=set_metadata_name).and_return(fake_metadata_checker)
# presume that our image does exist in GCE, with some fake data
# acquired by running a not mocked version of this code
image_name = 'appscale-image-name'
image_info = {
u'kind': u'compute#image',
u'description': u'',
u'rawDisk': {u'containerType': u'TAR', u'source': u''},
u'preferredKernel': unicode(GCEAgent.GCE_URL) + \
u'/google/global/kernels/gce-v20130515',
u'sourceType': u'RAW',
u'creationTimestamp': u'2013-05-21T08:05:12.198-07:00',
u'id': u'4235320207849085220',
u'selfLink': unicode(GCEAgent.GCE_URL) + \
u'961228229472/global/images/' + unicode(image_name),
u'name': unicode(image_name)
}
fake_image_request = flexmock(name='fake_image_request')
fake_image_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(image_info)
fake_images = flexmock(name='fake_images')
fake_images.should_receive('get').with_args(project=project_id,
image=image_name).and_return(fake_image_request)
fake_gce.should_receive('images').and_return(fake_images)
# next, presume that the static ip we want to use exists
address_name = 'static-ip'
address_info = {'items':[]}
region_name = 'my-zone1'
fake_address_request = flexmock(name='fake_address_request')
fake_address_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(address_info)
fake_addresses = flexmock(name='fake_addresses')
fake_addresses.should_receive('list').with_args(project=project_id,
filter="address eq static-ip", region=region_name).and_return(
fake_address_request)
fake_gce.should_receive('addresses').and_return(fake_addresses)
# next, presume that the zone we want to use exists
zone_name = 'my-zone1-b'
zone_info = {}
fake_zone_request = flexmock(name='fake_zone_request')
fake_zone_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(zone_info)
fake_zones = flexmock(name='fake_zones')
fake_zones.should_receive('get').with_args(project=project_id,
zone=zone_name).and_return(fake_zone_request)
fake_gce.should_receive('zones').and_return(fake_zones)
# next, presume that the persistent disk we want to use exists
disk_name = 'my-persistent-disk-1'
disk_info = {'status':'DONE'}
fake_disk_request = flexmock(name='fake_disk_request')
fake_disk_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(disk_info)
fake_disks = flexmock(name='fake_disks')
fake_disks.should_receive('get').with_args(project=project_id,
disk=disk_name, zone=zone).and_return(fake_disk_request)
fake_disks.should_receive('insert').with_args(project=project_id,
sourceImage=str, body=dict, zone=zone_name).and_return(fake_disk_request)
fake_gce.should_receive('disks').and_return(fake_disks)
# next, presume that the network doesn't exist yet
fake_network_request = flexmock(name='fake_network_request')
fake_network_request.should_receive('execute').with_args(
http=fake_authorized_http).and_raise(apiclient.errors.HttpError, None,
None)
fake_networks = flexmock(name='fake_networks')
fake_networks.should_receive('get').with_args(project=project_id,
network='bazgroup').and_return(fake_network_request)
fake_gce.should_receive('networks').and_return(fake_networks)
# next, presume that the firewall doesn't exist yet
fake_firewall_request = flexmock(name='fake_firewall_request')
fake_firewall_request.should_receive('execute').with_args(
http=fake_authorized_http).and_raise(apiclient.errors.HttpError, None,
None)
fake_firewalls = flexmock(name='fake_firewalls')
fake_firewalls.should_receive('get').with_args(project=project_id,
firewall='bazgroup').and_return(fake_firewall_request)
fake_gce.should_receive('firewalls').and_return(fake_firewalls)
# presume that we can create the network fine
create_network = u'operation-1369175117235-4dd41ec7d6c11-8013657f'
network_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': create_network,
u'startTime': u'2013-05-21T15:25:17.308-07:00',
u'insertTime': u'2013-05-21T15:25:17.235-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + \
u'appscale.com:appscale/global/networks/bazgroup',
u'operationType': u'insert',
u'progress': 0,
u'id': u'4904874319704759670',
u'selfLink': unicode(GCEAgent.GCE_URL) + \
u'appscale.com:appscale/global/operations/' + \
u'operation-1369175117235-4dd41ec7d6c11-8013657f',
u'user': u'*****@*****.**'
}
fake_network_insert_request = flexmock(name='fake_network_insert_request')
fake_network_insert_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(network_info)
fake_networks.should_receive('insert').with_args(project=project_id,
body=dict).and_return(fake_network_insert_request)
created_network_info = {
u'status': u'DONE'
}
fake_network_checker = flexmock(name='fake_network_checker')
fake_network_checker.should_receive('execute').and_return(
created_network_info)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=create_network).and_return(fake_network_checker)
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
# and presume that we can create the firewall fine
create_firewall = u'operation-1369176378310-4dd4237a84021-68e4dfa6'
firewall_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': create_firewall,
u'startTime': u'2013-05-21T15:46:18.402-07:00',
u'insertTime': u'2013-05-21T15:46:18.310-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + \
u'appscale.com:appscale/global/firewalls/bazgroup',
u'operationType': u'insert',
u'progress': 0,
u'id': u'13248349431060541723',
u'selfLink': unicode(GCEAgent.GCE_URL) + \
u'appscale.com:appscale/global/operations/' + \
u'operation-1369176378310-4dd4237a84021-68e4dfa6',
u'user': u'*****@*****.**'
}
fake_firewall_insert_request = flexmock(name='fake_firewall_insert_request')
fake_firewall_insert_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(firewall_info)
fake_firewalls.should_receive('insert').with_args(project=
u'appscale.com:appscale', body=dict).and_return(fake_firewall_insert_request)
created_firewall_info = {
u'status': u'DONE'
}
fake_firewall_checker = flexmock(name='fake_network_checker')
fake_firewall_checker.should_receive('execute').and_return(
created_firewall_info)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=create_firewall).and_return(fake_firewall_checker)
# we only need to create one node, so set up mocks for that
add_instance = u'operation-1369248752891-4dd5311848461-afc55a20'
instance_id = 'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d'
add_instance_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': add_instance,
u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a',
u'startTime': u'2013-05-22T11:52:32.939-07:00',
u'insertTime': u'2013-05-22T11:52:32.891-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/instances/' + instance_id,
u'operationType': u'insert',
u'progress': 0,
u'id': u'6663616273628949255',
u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20',
u'user': u'*****@*****.**'
}
fake_add_instance_request = flexmock(name='fake_add_instance_request')
fake_add_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(add_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_instances.should_receive('insert').with_args(project=project_id,
body=dict, zone=str).and_return(fake_add_instance_request)
created_instance_info = {
u'status': u'DONE'
}
fake_instance_checker = flexmock(name='fake_network_checker')
fake_instance_checker.should_receive('execute').and_return(
created_instance_info)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=add_instance).and_return(fake_instance_checker)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {
}
list_instance_info = {
u'items': [{
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a/machineTypes/' + instance_type,
u'name': instance_id,
u'zone': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a/instances/' + instance_id,
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP': u'private1',
u'network': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/global/networks/bazgroup',
u'name': u'nic0'
}]
}],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/us-central1-a/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1/projects/961228229472/zones/us-central1-a/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(no_instance_info).and_return(
no_instance_info).and_return(list_instance_info)
fake_instances.should_receive('list').with_args(project=project_id,
filter="name eq appscale-bazgroup-.*", zone=zone) \
.and_return(fake_list_instance_request)
fake_instances.should_receive('list').with_args(project=project_id,
zone=zone).and_return(fake_list_instance_request)
# mock out deleting the public IP from the instance
fake_delete_access_request = flexmock(name='fake_delete_access_request')
fake_delete_access_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return()
fake_instances.should_receive('deleteAccessConfig').with_args(
project=project_id, accessConfig=str, instance=instance_id,
networkInterface=str, zone=zone).and_return(fake_delete_access_request)
# as well as adding in the new, static IP to the instance
fake_add_access_request = flexmock(name='fake_add_access_request')
fake_add_access_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return()
fake_instances.should_receive('addAccessConfig').with_args(
project=project_id, instance=instance_id, networkInterface=str,
zone=zone, body=dict).and_return(fake_add_access_request)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args('compute', str) \
.and_return(fake_gce)
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_raise(ShellException)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('sudo cp')).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
False, 5).and_return()
self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
.format(self.keyname)), False, 5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('controller-17443.cfg'))
self.setup_socket_mocks('static-ip')
self.setup_appcontroller_mocks('static-ip', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "static-ip",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}])))
# copying over the locations yaml and json files should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
self.setup_uaserver_mocks('static-ip')
# Finally, pretend we're using a single persistent disk.
disk_layout = yaml.safe_load("""
node-1: my-persistent-disk-1
""")
argv = [
"--min", "1",
"--max", "1",
"--disks", base64.b64encode(yaml.dump(disk_layout)),
"--group", self.group,
"--infrastructure", "gce",
"--gce_instance_type", instance_type,
"--machine", image_name,
"--keyname", self.keyname,
"--client_secrets", client_secrets,
"--project", project_id,
"--test",
"--zone", "my-zone1-b",
"--static_ip", "static-ip"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_terminate_in_gce_and_succeeds(self):
# let's say that there is a locations.yaml file, which means appscale is
# running, so we should terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_yaml_location(
self.keyname)).and_return(True)
os.path.should_receive('exists').with_args(
LocalState.get_client_secrets_location(
self.keyname)).and_return(True)
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.yaml file, and pretend that we're on
# GCE
project_id = "1234567890"
zone = 'my-zone-1b'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
fake_yaml_file = flexmock(name='fake_file')
fake_yaml_file.should_receive('read').and_return(
yaml.dump({
'infrastructure': 'gce',
'group': self.group,
'project': project_id,
'zone': zone
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_yaml_location(self.keyname), 'r') \
.and_return(fake_yaml_file)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(
json.dumps([{
'public_ip': 'public1',
'jobs': ['shadow']
}, {
'public_ip': 'public2',
'jobs': ['appengine']
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# also add in a fake client-secrets file for GCE
client_secrets = LocalState.get_client_secrets_location(self.keyname)
# mock out talking to GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive(
'flow_from_clientsecrets').with_args(
client_secrets, scope=str).and_return(fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
fake_flags = oauth2client.tools.argparser.parse_args(args=[])
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(str).and_return(
fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run_flow').with_args(
fake_flow, fake_storage, fake_flags).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_gce = flexmock(name='fake_gce')
# let's say that two instances are running
instance_one_info = {
u'status':
u'RUNNING',
u'kind':
u'compute#instance',
u'machineType':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name':
u'appscale-bazboogroup-one',
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {
u'fingerprint': u'42WmSpB8rSM='
},
u'image':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward':
False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email':
u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp':
u'2013-05-22T11:52:33.254-07:00',
u'id':
u'8684033495853907982',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP':
u'private1',
u'network':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name':
u'nic0'
}]
}
instance_two_info = {
u'status':
u'RUNNING',
u'kind':
u'compute#instance',
u'machineType':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name':
u'appscale-bazboogroup-two',
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {
u'fingerprint': u'42WmSpB8rSM='
},
u'image':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward':
False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email':
u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp':
u'2013-05-22T11:52:33.254-07:00',
u'id':
u'8684033495853907982',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP':
u'private1',
u'network':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name':
u'nic0'
}]
}
list_instance_info = {
u'items': [instance_one_info, instance_two_info],
u'kind':
u'compute#instanceList',
u'id':
u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(
name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_instances.should_receive('list').with_args(project=project_id,
filter="name eq appscale-bazboogroup-.*", zone=zone) \
.and_return(fake_list_instance_request)
fake_gce.should_receive('instances').and_return(fake_instances)
# And assume that we can kill both of our instances fine
delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de'
delete_instance_info_one = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
delete_instance_info_two = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
fake_delete_instance_request_one = flexmock(
name='fake_delete_instance_request_one')
fake_delete_instance_request_one.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_one)
fake_instances.should_receive('delete').with_args(
project=project_id, zone=zone,
instance='appscale-bazboogroup-one').and_return(
fake_delete_instance_request_one)
fake_delete_instance_request_two = flexmock(
name='fake_delete_instance_request_two')
fake_delete_instance_request_two.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_two)
fake_instances.should_receive('delete').with_args(
project=project_id, zone=zone,
instance='appscale-bazboogroup-two').and_return(
fake_delete_instance_request_two)
# mock out our waiting for the instances to be deleted
all_done = {u'status': u'DONE'}
fake_instance_checker = flexmock(name='fake_instance_checker')
fake_instance_checker.should_receive('execute').and_return(all_done)
fake_blocker = flexmock(name='fake_blocker')
fake_blocker.should_receive('get').with_args(
project=project_id, operation=delete_instance,
zone=zone).and_return(fake_instance_checker)
fake_gce.should_receive('zoneOperations').and_return(fake_blocker)
# mock out the call to delete the firewall
delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654'
fake_delete_firewall_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_firewall,
u'startTime': u'2013-05-27T11:01:35.482-07:00',
u'insertTime': u'2013-05-27T11:01:35.390-07:00',
u'targetId': u'11748720697396371259',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'15574488986772298961',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654',
u'user': u'*****@*****.**'
}
fake_delete_firewall_request = flexmock(
name='fake_delete_firewall_request')
fake_delete_firewall_request.should_receive('execute').and_return(
fake_delete_firewall_info)
fake_firewalls = flexmock(name='fake_firewalls')
fake_firewalls.should_receive('delete').with_args(
project=project_id,
firewall=self.group).and_return(fake_delete_firewall_request)
fake_gce.should_receive('firewalls').and_return(fake_firewalls)
# mock out the call to make sure the firewall was deleted
fake_firewall_checker = flexmock(name='fake_firewall_checker')
fake_firewall_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(
project=project_id,
operation=delete_firewall).and_return(fake_firewall_checker)
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
# and the call to delete the network
delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca'
fake_delete_network_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_network,
u'startTime': u'2013-05-27T11:02:30.012-07:00',
u'insertTime': u'2013-05-27T11:02:29.954-07:00',
u'targetId': u'17688075350400527692',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'12623697331874594836',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca',
u'user': u'*****@*****.**'
}
fake_delete_network_request = flexmock(
name='fake_delete_network_request')
fake_delete_network_request.should_receive('execute').and_return(
fake_delete_network_info)
fake_networks = flexmock(name='fake_networks')
fake_networks.should_receive('delete').with_args(
project=project_id,
network=self.group).and_return(fake_delete_network_request)
fake_gce.should_receive('networks').and_return(fake_networks)
# mock out the call to make sure the network was deleted
fake_network_checker = flexmock(name='fake_network_checker')
fake_network_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(
project=project_id,
operation=delete_network).and_return(fake_network_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args(
'compute', GCEAgent.API_VERSION).and_return(fake_gce)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_yaml_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
argv = ["--keyname", self.keyname, "--test"]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_terminate_in_gce_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is running, so we should
# terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_client_secrets_location(self.keyname)).and_return(True)
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on GCE
project_id = "1234567890"
zone = 'my-zone-1b'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
# Assume persistent disks are used.
flexmock(LocalState).should_receive('are_disks_used').and_return(True)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(json.dumps({
"infrastructure_info": {
'infrastructure': 'gce',
'group': self.group,
'project': project_id,
'zone': zone
},
"node_info": [
{
'public_ip': 'public1',
'jobs': ['shadow']
},
{
'public_ip': 'public2',
'jobs': ['appengine']
}
]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# also add in a fake client-secrets file for GCE
client_secrets = LocalState.get_client_secrets_location(self.keyname)
# mock out talking to GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive('flow_from_clientsecrets').with_args(
client_secrets, scope=str).and_return(fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
fake_flags = oauth2client.tools.argparser.parse_args(args=[])
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(str).and_return(
fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run_flow').with_args(fake_flow,
fake_storage, fake_flags).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_gce = flexmock(name='fake_gce')
# let's say that two instances are running
instance_one_info = {
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': u'bazboogroup-one',
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP': u'private1',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name': u'nic0'
}]
}
instance_two_info = {
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': u'bazboogroup-two',
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP': u'private1',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name': u'nic0'
}]
}
list_instance_info = {
u'items': [instance_one_info, instance_two_info],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_instances.should_receive('list').with_args(project=project_id,
filter="name eq bazboogroup-.*", zone=zone) \
.and_return(fake_list_instance_request)
fake_gce.should_receive('instances').and_return(fake_instances)
# And assume that we can kill both of our instances fine
delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de'
delete_instance_info_one = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
delete_instance_info_two = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
fake_delete_instance_request_one = flexmock(name='fake_delete_instance_request_one')
fake_delete_instance_request_one.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_one)
fake_instances.should_receive('delete').with_args(project=project_id,
zone=zone, instance='bazboogroup-one').and_return(
fake_delete_instance_request_one)
fake_delete_instance_request_two = flexmock(name='fake_delete_instance_request_two')
fake_delete_instance_request_two.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_two)
fake_instances.should_receive('delete').with_args(project=project_id,
zone=zone, instance='bazboogroup-two').and_return(
fake_delete_instance_request_two)
# mock out our waiting for the instances to be deleted
all_done = {
u'status' : u'DONE'
}
fake_instance_checker = flexmock(name='fake_instance_checker')
fake_instance_checker.should_receive('execute').and_return(all_done)
fake_blocker = flexmock(name='fake_blocker')
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_instance, zone=zone).and_return(
fake_instance_checker)
fake_gce.should_receive('zoneOperations').and_return(fake_blocker)
# mock out the call to delete the firewall
delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654'
fake_delete_firewall_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_firewall,
u'startTime': u'2013-05-27T11:01:35.482-07:00',
u'insertTime': u'2013-05-27T11:01:35.390-07:00',
u'targetId': u'11748720697396371259',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'15574488986772298961',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654',
u'user': u'*****@*****.**'
}
fake_delete_firewall_request = flexmock(name='fake_delete_firewall_request')
fake_delete_firewall_request.should_receive('execute').and_return(fake_delete_firewall_info)
fake_firewalls = flexmock(name='fake_firewalls')
fake_firewalls.should_receive('delete').with_args(project=project_id,
firewall=self.group).and_return(fake_delete_firewall_request)
fake_gce.should_receive('firewalls').and_return(fake_firewalls)
# mock out the call to make sure the firewall was deleted
fake_firewall_checker = flexmock(name='fake_firewall_checker')
fake_firewall_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_firewall).and_return(fake_firewall_checker)
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
# and the call to delete the network
delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca'
fake_delete_network_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_network,
u'startTime': u'2013-05-27T11:02:30.012-07:00',
u'insertTime': u'2013-05-27T11:02:29.954-07:00',
u'targetId': u'17688075350400527692',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'12623697331874594836',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca',
u'user': u'*****@*****.**'
}
fake_delete_network_request = flexmock(name='fake_delete_network_request')
fake_delete_network_request.should_receive('execute').and_return(fake_delete_network_info)
fake_networks = flexmock(name='fake_networks')
fake_networks.should_receive('delete').with_args(project=project_id,
network=self.group).and_return(fake_delete_network_request)
fake_gce.should_receive('networks').and_return(fake_networks)
# mock out the call to make sure the network was deleted
fake_network_checker = flexmock(name='fake_network_checker')
fake_network_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_network).and_return(fake_network_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
flexmock(GCEAgent).should_receive('get_secrets_type')\
.and_return(CredentialTypes.OAUTH)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
flexmock(RemoteHelper).should_receive('terminate_cloud_infrastructure')
argv = [
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive('flow_from_clientsecrets').with_args(
GCEAgent.CLIENT_SECRETS_LOCATION, scope=GCEAgent.GCE_SCOPE).and_return(
fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(
GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run').with_args(fake_flow,
fake_storage).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {
}
instance_id = u'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d'
list_instance_info = {
u'items': [{
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': instance_id,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public-ip'
}],
u'networkIP': u'private-ip',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup',
u'name': u'nic0'
}]
}],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/us-central1-a/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/us-central1-a/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
fake_authorized_http).and_return(no_instance_info).and_return(
list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_gce = flexmock(name='fake_gce')
fake_gce.should_receive('instances').and_return(fake_instances)
fake_instances.should_receive('list').with_args(project=self.project,
filter="name eq appscale-boogroup-.*", zone=GCEAgent.DEFAULT_ZONE) \
.and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u'operation-1369248752891-4dd5311848461-afc55a20'
add_instance_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': add_instance,
u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a',
u'startTime': u'2013-05-22T11:52:32.939-07:00',
u'insertTime': u'2013-05-22T11:52:32.891-07:00',
u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'operationType': u'insert',
u'progress': 0,
u'id': u'6663616273628949255',
u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20',
u'user': u'*****@*****.**'
}
fake_add_instance_request = flexmock(name='fake_add_instance_request')
fake_add_instance_request.should_receive('execute').with_args(
fake_authorized_http).and_return(add_instance_info)
fake_instances.should_receive('insert').with_args(project=self.project,
body=dict, zone=str).and_return(fake_add_instance_request)
created_instance_info = {
u'status': u'DONE'
}
fake_instance_checker = flexmock(name='fake_network_checker')
fake_instance_checker.should_receive('execute').and_return(
created_instance_info)
fake_blocker = flexmock(name='fake_blocker')
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
fake_blocker.should_receive('get').with_args(project=self.project,
operation=add_instance).and_return(fake_instance_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_params = {
'credentials' : {
'EC2_URL': None,
'EC2_ACCESS_KEY': None,
'EC2_SECRET_KEY': None,
},
'project': self.project,
'group': 'boogroup',
'image_id': 'booid',
'infrastructure': 'gce',
'instance_type': 'booinstance_type',
'keyname': 'bookeyname',
'num_vms': '1',
'use_spot_instances': False,
}
full_result = {
'success': True,
'reservation_id': self.reservation_id,
'reason': 'none'
}
self.assertEquals(full_result, i.run_instances(full_params, 'secret'))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(
self.reservation_id)['state'])
vm_info = i.reservations.get(self.reservation_id)['vm_info']
self.assertEquals(['public-ip'], vm_info['public_ips'])
self.assertEquals(['private-ip'], vm_info['private_ips'])
self.assertEquals([instance_id], vm_info['instance_ids'])
def test_terminate_in_gce_and_succeeds(self):
# let's say that there is a locations.yaml file, which means appscale is
# running, so we should terminate the services on each box
flexmock(os.path)
os.path.should_call("exists") # set up the fall-through
os.path.should_receive("exists").with_args(LocalState.get_secret_key_location(self.keyname)).and_return(True)
os.path.should_receive("exists").with_args(LocalState.get_locations_yaml_location(self.keyname)).and_return(
True
)
os.path.should_receive("exists").with_args(LocalState.get_client_secrets_location(self.keyname)).and_return(
True
)
# mock out reading the locations.yaml file, and pretend that we're on
# GCE
project_id = "1234567890"
zone = "my-zone-1b"
builtins = flexmock(sys.modules["__builtin__"])
builtins.should_call("open")
fake_yaml_file = flexmock(name="fake_file")
fake_yaml_file.should_receive("read").and_return(
yaml.dump({"infrastructure": "gce", "group": self.group, "project": project_id, "zone": zone})
)
builtins.should_receive("open").with_args(LocalState.get_locations_yaml_location(self.keyname), "r").and_return(
fake_yaml_file
)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
fake_json_file = flexmock(name="fake_file")
fake_json_file.should_receive("read").and_return(
json.dumps([{"public_ip": "public1", "jobs": ["shadow"]}, {"public_ip": "public2", "jobs": ["appengine"]}])
)
builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return(
fake_json_file
)
# and slip in a fake secret file
fake_secret_file = flexmock(name="fake_file")
fake_secret_file.should_receive("read").and_return("the secret")
builtins.should_receive("open").with_args(LocalState.get_secret_key_location(self.keyname), "r").and_return(
fake_secret_file
)
# also add in a fake client-secrets file for GCE
client_secrets = LocalState.get_client_secrets_location(self.keyname)
# mock out talking to GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name="fake_flow")
flexmock(oauth2client.client)
oauth2client.client.should_receive("flow_from_clientsecrets").with_args(client_secrets, scope=str).and_return(
fake_flow
)
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(None)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(str).and_return(fake_storage)
fake_credentials = flexmock(name="fake_credentials")
flexmock(oauth2client.tools)
oauth2client.tools.should_receive("run").with_args(fake_flow, fake_storage).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
fake_gce = flexmock(name="fake_gce")
# let's say that two instances are running
instance_one_info = {
u"status": u"RUNNING",
u"kind": u"compute#instance",
u"machineType": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1",
u"name": u"appscale-bazboogroup-one",
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"tags": {u"fingerprint": u"42WmSpB8rSM="},
u"image": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64",
u"disks": [{u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"}],
u"canIpForward": False,
u"serviceAccounts": [
{u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"}
],
u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="},
u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00",
u"id": u"8684033495853907982",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"networkInterfaces": [
{
u"accessConfigs": [
{
u"kind": u"compute#accessConfig",
u"type": u"ONE_TO_ONE_NAT",
u"name": u"External NAT",
u"natIP": u"public1",
}
],
u"networkIP": u"private1",
u"network": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup",
u"name": u"nic0",
}
],
}
instance_two_info = {
u"status": u"RUNNING",
u"kind": u"compute#instance",
u"machineType": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1",
u"name": u"appscale-bazboogroup-two",
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"tags": {u"fingerprint": u"42WmSpB8rSM="},
u"image": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64",
u"disks": [{u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"}],
u"canIpForward": False,
u"serviceAccounts": [
{u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"}
],
u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="},
u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00",
u"id": u"8684033495853907982",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"networkInterfaces": [
{
u"accessConfigs": [
{
u"kind": u"compute#accessConfig",
u"type": u"ONE_TO_ONE_NAT",
u"name": u"External NAT",
u"natIP": u"public1",
}
],
u"networkIP": u"private1",
u"network": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup",
u"name": u"nic0",
}
],
}
list_instance_info = {
u"items": [instance_one_info, instance_two_info],
u"kind": u"compute#instanceList",
u"id": u"projects/appscale.com:appscale/zones/my-zone-1b/instances",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances",
}
fake_list_instance_request = flexmock(name="fake_list_instance_request")
fake_list_instance_request.should_receive("execute").with_args(http=fake_authorized_http).and_return(
list_instance_info
)
fake_instances = flexmock(name="fake_instances")
fake_instances.should_receive("list").with_args(
project=project_id, filter="name eq appscale-bazboogroup-.*", zone=zone
).and_return(fake_list_instance_request)
fake_gce.should_receive("instances").and_return(fake_instances)
# And assume that we can kill both of our instances fine
delete_instance = u"operation-1369676691806-4ddb6b4ab6f39-a095d3de"
delete_instance_info_one = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": delete_instance,
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"startTime": u"2013-05-27T10:44:51.849-07:00",
u"insertTime": u"2013-05-27T10:44:51.806-07:00",
u"targetId": u"12912855597472179535",
u"targetLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f",
u"operationType": u"delete",
u"progress": 0,
u"id": u"11114355109942058217",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de",
u"user": u"*****@*****.**",
}
delete_instance_info_two = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": delete_instance,
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"startTime": u"2013-05-27T10:44:51.849-07:00",
u"insertTime": u"2013-05-27T10:44:51.806-07:00",
u"targetId": u"12912855597472179535",
u"targetLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f",
u"operationType": u"delete",
u"progress": 0,
u"id": u"11114355109942058217",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de",
u"user": u"*****@*****.**",
}
fake_delete_instance_request_one = flexmock(name="fake_delete_instance_request_one")
fake_delete_instance_request_one.should_receive("execute").with_args(http=fake_authorized_http).and_return(
delete_instance_info_one
)
fake_instances.should_receive("delete").with_args(
project=project_id, zone=zone, instance="appscale-bazboogroup-one"
).and_return(fake_delete_instance_request_one)
fake_delete_instance_request_two = flexmock(name="fake_delete_instance_request_two")
fake_delete_instance_request_two.should_receive("execute").with_args(http=fake_authorized_http).and_return(
delete_instance_info_two
)
fake_instances.should_receive("delete").with_args(
project=project_id, zone=zone, instance="appscale-bazboogroup-two"
).and_return(fake_delete_instance_request_two)
# mock out our waiting for the instances to be deleted
all_done = {u"status": u"DONE"}
fake_instance_checker = flexmock(name="fake_instance_checker")
fake_instance_checker.should_receive("execute").and_return(all_done)
fake_blocker = flexmock(name="fake_blocker")
fake_blocker.should_receive("get").with_args(
project=project_id, operation=delete_instance, zone=zone
).and_return(fake_instance_checker)
fake_gce.should_receive("zoneOperations").and_return(fake_blocker)
# mock out the call to delete the firewall
delete_firewall = u"operation-1369677695390-4ddb6f07cc611-5a8f1654"
fake_delete_firewall_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": delete_firewall,
u"startTime": u"2013-05-27T11:01:35.482-07:00",
u"insertTime": u"2013-05-27T11:01:35.390-07:00",
u"targetId": u"11748720697396371259",
u"targetLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20",
u"operationType": u"delete",
u"progress": 0,
u"id": u"15574488986772298961",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654",
u"user": u"*****@*****.**",
}
fake_delete_firewall_request = flexmock(name="fake_delete_firewall_request")
fake_delete_firewall_request.should_receive("execute").and_return(fake_delete_firewall_info)
fake_firewalls = flexmock(name="fake_firewalls")
fake_firewalls.should_receive("delete").with_args(project=project_id, firewall=self.group).and_return(
fake_delete_firewall_request
)
fake_gce.should_receive("firewalls").and_return(fake_firewalls)
# mock out the call to make sure the firewall was deleted
fake_firewall_checker = flexmock(name="fake_firewall_checker")
fake_firewall_checker.should_receive("execute").and_return(all_done)
fake_blocker.should_receive("get").with_args(project=project_id, operation=delete_firewall).and_return(
fake_firewall_checker
)
fake_gce.should_receive("globalOperations").and_return(fake_blocker)
# and the call to delete the network
delete_network = u"operation-1369677749954-4ddb6f3bd1849-056cf8ca"
fake_delete_network_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": delete_network,
u"startTime": u"2013-05-27T11:02:30.012-07:00",
u"insertTime": u"2013-05-27T11:02:29.954-07:00",
u"targetId": u"17688075350400527692",
u"targetLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20",
u"operationType": u"delete",
u"progress": 0,
u"id": u"12623697331874594836",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca",
u"user": u"*****@*****.**",
}
fake_delete_network_request = flexmock(name="fake_delete_network_request")
fake_delete_network_request.should_receive("execute").and_return(fake_delete_network_info)
fake_networks = flexmock(name="fake_networks")
fake_networks.should_receive("delete").with_args(project=project_id, network=self.group).and_return(
fake_delete_network_request
)
fake_gce.should_receive("networks").and_return(fake_networks)
# mock out the call to make sure the network was deleted
fake_network_checker = flexmock(name="fake_network_checker")
fake_network_checker.should_receive("execute").and_return(all_done)
fake_blocker.should_receive("get").with_args(project=project_id, operation=delete_network).and_return(
fake_network_checker
)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive("remove").with_args(LocalState.get_locations_yaml_location(self.keyname)).and_return()
os.should_receive("remove").with_args(LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive("remove").with_args(LocalState.get_secret_key_location(self.keyname)).and_return()
argv = ["--keyname", self.keyname, "--test"]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_gce_run_instances(self):
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_credentials = flexmock(name="fake_credentials")
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(fake_credentials)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(GCEAgent.OAUTH2_STORAGE_LOCATION).and_return(fake_storage)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {}
instance_id = u"appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d"
list_instance_info = {
u"items": [
{
u"status": u"RUNNING",
u"kind": u"compute#instance",
u"machineType": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1",
u"name": instance_id,
u"zone": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b",
u"tags": {u"fingerprint": u"42WmSpB8rSM="},
u"image": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64",
u"disks": [
{u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"}
],
u"canIpForward": False,
u"serviceAccounts": [
{u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"}
],
u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="},
u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00",
u"id": u"8684033495853907982",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"networkInterfaces": [
{
u"accessConfigs": [
{
u"kind": u"compute#accessConfig",
u"type": u"ONE_TO_ONE_NAT",
u"name": u"External NAT",
u"natIP": u"public-ip",
}
],
u"networkIP": u"private-ip",
u"network": u"https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazgroup",
u"name": u"nic0",
}
],
}
],
u"kind": u"compute#instanceList",
u"id": u"projects/appscale.com:appscale/zones/my-zone-1b/instances",
u"selfLink": u"https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances",
}
fake_list_instance_request = flexmock(name="fake_list_instance_request")
fake_list_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
no_instance_info
).and_return(list_instance_info)
fake_instances = flexmock(name="fake_instances")
fake_gce = flexmock(name="fake_gce")
fake_gce.should_receive("instances").and_return(fake_instances)
fake_instances.should_receive("list").with_args(
project=self.project, filter="name eq appscale-boogroup-.*", zone="my-zone-1b"
).and_return(fake_list_instance_request)
# we only need to create one node, so set up mocks for that
add_instance = u"operation-1369248752891-4dd5311848461-afc55a20"
add_instance_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": add_instance,
u"azone": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/my-zone-1b",
u"startTime": u"2013-05-22T11:52:32.939-07:00",
u"insertTime": u"2013-05-22T11:52:32.891-07:00",
u"targetLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"operationType": u"insert",
u"progress": 0,
u"id": u"6663616273628949255",
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/my-zone-1b/operations/operation-1369248752891-4dd5311848461-afc55a20",
u"user": u"*****@*****.**",
}
fake_add_instance_request = flexmock(name="fake_add_instance_request")
fake_add_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
add_instance_info
)
fake_instances.should_receive("insert").with_args(project=self.project, body=dict, zone=str).and_return(
fake_add_instance_request
)
created_instance_info = {u"status": u"DONE"}
fake_instance_checker = flexmock(name="fake_network_checker")
fake_instance_checker.should_receive("execute").and_return(created_instance_info)
fake_blocker = flexmock(name="fake_blocker")
fake_gce.should_receive("globalOperations").and_return(fake_blocker)
fake_blocker.should_receive("get").with_args(project=self.project, operation=add_instance).and_return(
fake_instance_checker
)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", GCEAgent.API_VERSION).and_return(fake_gce)
i = InfrastructureManager(blocking=True)
# first, validate that the run_instances call goes through successfully
# and gives the user a reservation id
full_result = {"success": True, "reservation_id": self.reservation_id, "reason": "none"}
self.assertEquals(full_result, i.run_instances(self.params, "secret"))
# next, look at run_instances internally to make sure it actually is
# updating its reservation info
self.assertEquals(InfrastructureManager.STATE_RUNNING, i.reservations.get(self.reservation_id)["state"])
vm_info = i.reservations.get(self.reservation_id)["vm_info"]
self.assertEquals(["public-ip"], vm_info["public_ips"])
self.assertEquals(["private-ip"], vm_info["private_ips"])
self.assertEquals([instance_id], vm_info["instance_ids"])
def test_appscale_in_one_node_gce_deployment(self):
# presume that our client_secrets file exists
project_id = "1234567890"
client_secrets = "/boo/client_secrets.json"
instance_type = "n1-standard-8"
zone = "my-zone-1b"
os.path.should_receive("exists").with_args(client_secrets).and_return(True)
# and that the user does not have an ssh key set up, forcing us to create
# one for them
private_key = "{0}{1}.key".format(LocalState.LOCAL_APPSCALE_PATH, self.keyname)
public_key = "{0}{1}.pub".format(LocalState.LOCAL_APPSCALE_PATH, self.keyname)
os.path.should_receive("exists").with_args(private_key).and_return(False)
os.path.should_receive("exists").with_args(public_key).and_return(False)
self.local_state.should_receive("shell").with_args(re.compile("^ssh-keygen"), False).and_return()
flexmock(os)
original_private_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname
os.should_receive("chmod").with_args(original_private_key, 0600)
os.should_receive("chmod").with_args(public_key, 0600)
flexmock(shutil)
shutil.should_receive("copy").with_args(original_private_key, private_key)
# also, we should be able to copy over our secret.json file fine
shutil.should_receive("copy").with_args(client_secrets, LocalState.get_client_secrets_location(self.keyname))
# let's say that appscale isn't already running
self.local_state.should_receive("ensure_appscale_isnt_running").and_return()
self.local_state.should_receive("make_appscale_directory").and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name="fake_connection")
fake_connection.should_receive("request").with_args("POST", "/upload", str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive("HTTPConnection").with_args("logs.appscale.com").and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive("uuid4").and_return("the secret")
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive("read").and_return("the secret")
fake_secret.should_receive("write").and_return()
self.builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)
self.builtins.should_receive("open").with_args(secret_key_location, "w").and_return(fake_secret)
# mock out interactions with GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name="fake_flow")
flexmock(oauth2client.client)
oauth2client.client.should_receive("flow_from_clientsecrets").with_args(client_secrets, scope=str).and_return(
fake_flow
)
fake_storage = flexmock(name="fake_storage")
fake_storage.should_receive("get").and_return(None)
flexmock(oauth2client.file)
oauth2client.file.should_receive("Storage").with_args(str).and_return(fake_storage)
fake_credentials = flexmock(name="fake_credentials")
flexmock(oauth2client.tools)
oauth2client.tools.should_receive("run").with_args(fake_flow, fake_storage).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name="fake_http")
fake_authorized_http = flexmock(name="fake_authorized_http")
flexmock(httplib2)
httplib2.should_receive("Http").and_return(fake_http)
fake_credentials.should_receive("authorize").with_args(fake_http).and_return(fake_authorized_http)
# presume that there is an ssh key stored, but it isn't ours
metadata_info = {
u"kind": u"compute#project",
u"description": u"",
u"commonInstanceMetadata": {
u"items": [{u"value": u"cgb:ssh-rsa keyinfo myhost", u"key": u"sshKeys"}],
u"kind": u"compute#metadata",
},
}
fake_metadata_request = flexmock(name="fake_metadata_request")
fake_metadata_request.should_receive("execute").with_args(fake_authorized_http).and_return(metadata_info)
fake_projects = flexmock(name="fake_projects")
fake_projects.should_receive("get").with_args(project=project_id).and_return(fake_metadata_request)
fake_gce = flexmock(name="fake_gce")
fake_gce.should_receive("projects").and_return(fake_projects)
# thus we will need to set the metadata with our ssh key
fake_ssh_pub_key = flexmock(name="fake_ssh_pub_key")
fake_ssh_pub_key.should_receive("read").and_return("ssh-rsa key2info myhost")
self.builtins.should_receive("open").with_args(public_key).and_return(fake_ssh_pub_key)
new_metadata_body = {
"items": [{"value": u"cgb:ssh-rsa key2info myhost\ncgb:ssh-rsa keyinfo myhost", "key": "sshKeys"}],
"kind": "compute#metadata",
}
set_metadata_name = u"operation-222222-4dd41ec7d6c11-8013657f"
set_metadata = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": set_metadata_name,
u"operationType": u"insert",
u"progress": 0,
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/global/operations/"
+ u"operation-1369175117235-4dd41ec7d6c11-8013657f",
u"user": u"*****@*****.**",
}
fake_set_metadata_request = flexmock(name="fake_set_metadata_request")
fake_set_metadata_request.should_receive("execute").and_return(set_metadata)
fake_projects.should_receive("setCommonInstanceMetadata").with_args(project=project_id, body=dict).and_return(
fake_set_metadata_request
)
updated_metadata_info = {u"status": u"DONE"}
fake_metadata_checker = flexmock(name="fake_network_checker")
fake_metadata_checker.should_receive("execute").and_return(updated_metadata_info)
fake_blocker = flexmock(name="fake_blocker")
fake_blocker.should_receive("get").with_args(project=project_id, operation=set_metadata_name).and_return(
fake_metadata_checker
)
# presume that our image does exist in GCE, with some fake data
# acquired by running a not mocked version of this code
image_name = "appscale-image-name"
image_info = {
u"kind": u"compute#image",
u"description": u"",
u"rawDisk": {u"containerType": u"TAR", u"source": u""},
u"preferredKernel": unicode(GCEAgent.GCE_URL) + u"/google/global/kernels/gce-v20130515",
u"sourceType": u"RAW",
u"creationTimestamp": u"2013-05-21T08:05:12.198-07:00",
u"id": u"4235320207849085220",
u"selfLink": unicode(GCEAgent.GCE_URL) + u"961228229472/global/images/" + unicode(image_name),
u"name": unicode(image_name),
}
fake_image_request = flexmock(name="fake_image_request")
fake_image_request.should_receive("execute").with_args(fake_authorized_http).and_return(image_info)
fake_images = flexmock(name="fake_images")
fake_images.should_receive("get").with_args(project=project_id, image=image_name).and_return(fake_image_request)
fake_gce.should_receive("images").and_return(fake_images)
# next, presume that the zone we want to use exists
zone_name = "my-zone-1b"
zone_info = {}
fake_zone_request = flexmock(name="fake_zone_request")
fake_zone_request.should_receive("execute").with_args(fake_authorized_http).and_return(zone_info)
fake_zones = flexmock(name="fake_zones")
fake_zones.should_receive("get").with_args(project=project_id, zone=zone_name).and_return(fake_zone_request)
fake_gce.should_receive("zones").and_return(fake_zones)
# next, presume that the persistent disk we want to use exists
disk_name = "my-persistent-disk-1"
disk_info = {}
fake_disk_request = flexmock(name="fake_disk_request")
fake_disk_request.should_receive("execute").with_args(fake_authorized_http).and_return(disk_info)
fake_disks = flexmock(name="fake_disks")
fake_disks.should_receive("get").with_args(project=project_id, disk=disk_name, zone=zone).and_return(
fake_disk_request
)
fake_gce.should_receive("disks").and_return(fake_disks)
# next, presume that the network doesn't exist yet
fake_network_request = flexmock(name="fake_network_request")
fake_network_request.should_receive("execute").with_args(fake_authorized_http).and_raise(
apiclient.errors.HttpError, None, None
)
fake_networks = flexmock(name="fake_networks")
fake_networks.should_receive("get").with_args(project=project_id, network="bazgroup").and_return(
fake_network_request
)
fake_gce.should_receive("networks").and_return(fake_networks)
# next, presume that the firewall doesn't exist yet
fake_firewall_request = flexmock(name="fake_firewall_request")
fake_firewall_request.should_receive("execute").with_args(fake_authorized_http).and_raise(
apiclient.errors.HttpError, None, None
)
fake_firewalls = flexmock(name="fake_firewalls")
fake_firewalls.should_receive("get").with_args(project=project_id, firewall="bazgroup").and_return(
fake_firewall_request
)
fake_gce.should_receive("firewalls").and_return(fake_firewalls)
# presume that we can create the network fine
create_network = u"operation-1369175117235-4dd41ec7d6c11-8013657f"
network_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": create_network,
u"startTime": u"2013-05-21T15:25:17.308-07:00",
u"insertTime": u"2013-05-21T15:25:17.235-07:00",
u"targetLink": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/global/networks/bazgroup",
u"operationType": u"insert",
u"progress": 0,
u"id": u"4904874319704759670",
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/global/operations/"
+ u"operation-1369175117235-4dd41ec7d6c11-8013657f",
u"user": u"*****@*****.**",
}
fake_network_insert_request = flexmock(name="fake_network_insert_request")
fake_network_insert_request.should_receive("execute").with_args(fake_authorized_http).and_return(network_info)
fake_networks.should_receive("insert").with_args(project=project_id, body=dict).and_return(
fake_network_insert_request
)
created_network_info = {u"status": u"DONE"}
fake_network_checker = flexmock(name="fake_network_checker")
fake_network_checker.should_receive("execute").and_return(created_network_info)
fake_blocker.should_receive("get").with_args(project=project_id, operation=create_network).and_return(
fake_network_checker
)
fake_gce.should_receive("globalOperations").and_return(fake_blocker)
# and presume that we can create the firewall fine
create_firewall = u"operation-1369176378310-4dd4237a84021-68e4dfa6"
firewall_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": create_firewall,
u"startTime": u"2013-05-21T15:46:18.402-07:00",
u"insertTime": u"2013-05-21T15:46:18.310-07:00",
u"targetLink": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/global/firewalls/bazgroup",
u"operationType": u"insert",
u"progress": 0,
u"id": u"13248349431060541723",
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/global/operations/"
+ u"operation-1369176378310-4dd4237a84021-68e4dfa6",
u"user": u"*****@*****.**",
}
fake_firewall_insert_request = flexmock(name="fake_firewall_insert_request")
fake_firewall_insert_request.should_receive("execute").with_args(fake_authorized_http).and_return(firewall_info)
fake_firewalls.should_receive("insert").with_args(project=project_id, body=dict).and_return(
fake_firewall_insert_request
)
created_firewall_info = {u"status": u"DONE"}
fake_firewall_checker = flexmock(name="fake_network_checker")
fake_firewall_checker.should_receive("execute").and_return(created_firewall_info)
fake_blocker.should_receive("get").with_args(project=project_id, operation=create_firewall).and_return(
fake_firewall_checker
)
# we only need to create one node, so set up mocks for that
add_instance = u"operation-1369248752891-4dd5311848461-afc55a20"
add_instance_info = {
u"status": u"PENDING",
u"kind": u"compute#operation",
u"name": add_instance,
u"azone": unicode(GCEAgent.GCE_URL) + u"appscale.com:appscale/zones/us-central1-a",
u"startTime": u"2013-05-22T11:52:32.939-07:00",
u"insertTime": u"2013-05-22T11:52:32.891-07:00",
u"targetLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"operationType": u"insert",
u"progress": 0,
u"id": u"6663616273628949255",
u"selfLink": unicode(GCEAgent.GCE_URL)
+ u"appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20",
u"user": u"*****@*****.**",
}
fake_add_instance_request = flexmock(name="fake_add_instance_request")
fake_add_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
add_instance_info
)
fake_instances = flexmock(name="fake_instances")
fake_gce.should_receive("instances").and_return(fake_instances)
fake_instances.should_receive("insert").with_args(project=project_id, body=dict, zone=str).and_return(
fake_add_instance_request
)
created_instance_info = {u"status": u"DONE"}
fake_instance_checker = flexmock(name="fake_network_checker")
fake_instance_checker.should_receive("execute").and_return(created_instance_info)
fake_blocker.should_receive("get").with_args(project=project_id, operation=add_instance).and_return(
fake_instance_checker
)
# add some fake data in where no instances are initially running, then one
# is (in response to our insert request)
no_instance_info = {}
list_instance_info = {
u"items": [
{
u"status": u"RUNNING",
u"kind": u"compute#instance",
u"machineType": u"https://www.googleapis.com/compute/v1beta15/projects/appscale.com:appscale/zones/us-central1-a/machineTypes/"
+ instance_type,
u"name": u"appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"zone": u"https://www.googleapis.com/compute/v1beta15/projects/appscale.com:appscale/zones/us-central1-a",
u"tags": {u"fingerprint": u"42WmSpB8rSM="},
u"image": u"https://www.googleapis.com/compute/v1beta15/projects/appscale.com:appscale/global/images/lucid64",
u"disks": [
{u"index": 0, u"kind": u"compute#attachedDisk", u"type": u"EPHEMERAL", u"mode": u"READ_WRITE"}
],
u"canIpForward": False,
u"serviceAccounts": [
{u"scopes": [GCEAgent.GCE_SCOPE], u"email": u"*****@*****.**"}
],
u"metadata": {u"kind": u"compute#metadata", u"fingerprint": u"42WmSpB8rSM="},
u"creationTimestamp": u"2013-05-22T11:52:33.254-07:00",
u"id": u"8684033495853907982",
u"selfLink": u"https://www.googleapis.com/compute/v1beta15/projects/appscale.com:appscale/zones/us-central1-a/instances/appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d",
u"networkInterfaces": [
{
u"accessConfigs": [
{
u"kind": u"compute#accessConfig",
u"type": u"ONE_TO_ONE_NAT",
u"name": u"External NAT",
u"natIP": u"public1",
}
],
u"networkIP": u"private1",
u"network": u"https://www.googleapis.com/compute/v1beta15/projects/appscale.com:appscale/global/networks/bazgroup",
u"name": u"nic0",
}
],
}
],
u"kind": u"compute#instanceList",
u"id": u"projects/appscale.com:appscale/zones/us-central1-a/instances",
u"selfLink": u"https://www.googleapis.com/compute/v1beta15/projects/961228229472/zones/us-central1-a/instances",
}
fake_list_instance_request = flexmock(name="fake_list_instance_request")
fake_list_instance_request.should_receive("execute").with_args(fake_authorized_http).and_return(
no_instance_info
).and_return(list_instance_info)
fake_instances.should_receive("list").with_args(
project=project_id, filter="name eq appscale-bazgroup-.*", zone=zone
).and_return(fake_list_instance_request)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive("build").with_args("compute", str).and_return(fake_gce)
# assume that root login is not enabled
self.local_state.should_receive("shell").with_args(re.compile("ssh"), False, 5, stdin="ls").and_raise(
ShellException
)
# assume that we can enable root login
self.local_state.should_receive("shell").with_args(
re.compile("ssh"), False, 5, stdin=re.compile("sudo cp")
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive("shell").with_args(re.compile("scp .*[r|d]sa"), False, 5).and_return()
self.local_state.should_receive("shell").with_args(
re.compile("scp .*{0}".format(self.keyname)), False, 5
).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive("shell").with_args(re.compile("openssl"), False, stdin=None)
# assume that we started god fine
self.local_state.should_receive("shell").with_args(re.compile("ssh"), False, 5, stdin=re.compile("nohup god"))
# and that we copied over the AppController's god file
self.local_state.should_receive("shell").with_args(
re.compile("scp"), False, 5, stdin=re.compile("appcontroller.god")
)
# also, that we started the AppController itself
self.local_state.should_receive("shell").with_args(re.compile("ssh"), False, 5, stdin=re.compile("god load"))
self.setup_socket_mocks("public1")
self.setup_appcontroller_mocks("public1", "private1")
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive("get_local_nodes_info").and_return(
json.loads(json.dumps([{"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"]}]))
)
# copying over the locations yaml and json files should be fine
self.local_state.should_receive("shell").with_args(
re.compile("scp"), False, 5, stdin=re.compile("locations-{0}".format(self.keyname))
)
# same for the secret key
self.local_state.should_receive("shell").with_args(
re.compile("scp"), False, 5, stdin=re.compile("{0}.secret".format(self.keyname))
)
self.setup_uaserver_mocks("public1")
# Finally, pretend we're using a single persistent disk.
disk_layout = yaml.safe_load(
"""
node-1: my-persistent-disk-1
"""
)
argv = [
"--min",
"1",
"--max",
"1",
"--disks",
base64.b64encode(yaml.dump(disk_layout)),
"--group",
self.group,
"--infrastructure",
"gce",
"--gce_instance_type",
instance_type,
"--machine",
image_name,
"--keyname",
self.keyname,
"--client_secrets",
client_secrets,
"--project",
project_id,
"--test",
"--zone",
"my-zone-1b",
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
