def create(key):
"""Creates an instance group manager from the given InstanceGroupManager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
Raises:
net.Error: HTTP status code is not 200 (created) or 409 (already created).
"""
instance_group_manager = key.get()
if not instance_group_manager:
logging.warning('InstanceGroupManager does not exist: %s', key)
return
if instance_group_manager.url:
logging.warning(
'Instance group manager for InstanceGroupManager already exists: %s',
key,
)
return
instance_template_revision = key.parent().get()
if not instance_template_revision:
logging.warning('InstanceTemplateRevision does not exist: %s',
key.parent())
return
if not instance_template_revision.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key.parent())
return
if not instance_template_revision.url:
logging.warning('InstanceTemplateRevision URL unspecified: %s',
key.parent())
return
api = gce.Project(instance_template_revision.project)
try:
# Create the instance group manager with 0 instances. The resize cron job
# will adjust this later.
result = api.create_instance_group_manager(
get_name(instance_group_manager),
instance_template_revision.url,
0,
instance_group_manager.key.id(),
base_name=get_base_name(instance_group_manager),
)
except net.Error as e:
if e.status_code == 409:
# If the instance template already exists, just record the URL.
result = api.get_instance_group_manager(
get_name(instance_group_manager),
instance_group_manager.key.id())
update_url(instance_group_manager.key, result['selfLink'])
return
else:
raise
update_url(instance_group_manager.key, result['targetLink'])
def test_yield_instances_with_filter(self):
self.mock_requests([
(
{
'deadline':
120,
'params': {
'filter': 'name eq "inst-filter"',
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/aggregated/instances',
},
{
'items': {
'zone1': {
'instances': [instance('a')]
},
'zone2': {
'instances': [instance('b', status='STOPPED')]
},
},
},
),
])
result = list(gce.Project('123').yield_instances('inst-filter'))
self.assertEqual(
[instance('a'), instance('b', status='STOPPED')], result)
def _delete(instance_template_revision, instance_group_manager, instance):
"""Deletes the given instance.
Args:
instance_template_revision: models.InstanceTemplateRevision.
instance_group_manager: models.InstanceGroupManager.
instance: models.Instance
"""
api = gce.Project(instance_template_revision.project)
try:
result = api.delete_instances(
instance_group_managers.get_name(instance_group_manager),
instance_group_manager.key.id(),
[instance.url],
)
if result['status'] != 'DONE':
logging.warning(
'Instance group manager operation failed: %s\n%s',
parent.key,
json.dumps(result, indent=2),
)
else:
metrics.send_machine_event('DELETION_SCHEDULED', instance.key.id())
except net.Error as e:
if e.status_code == 400:
metrics.send_machine_event('DELETION_SUCCEEDED', instance.key.id())
else:
raise
def test_add_access_config(self):
self.mock_requests([
(
{
'method':
'POST',
'params': {
'networkInterface': 'nic0'
},
'payload': {
'kind': 'compute#accessConfig',
'name': 'External NAT',
'natIP': '1.2.3.4',
'type': 'ONE_TO_ONE_NAT',
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/zone-id/instances/inst_id/addAccessConfig',
},
{
'name': 'operation',
'status': 'DONE',
},
),
])
op = gce.Project('123').add_access_config('zone-id', 'inst_id', 'nic0',
'1.2.3.4')
self.assertTrue(op.done)
def test_set_metadata(self):
self.mock_requests([
(
{
'method':
'POST',
'payload': {
'fingerprint': 'fingerprint',
'items': [{
'key': 'k',
'value': 'v'
}],
'kind': 'compute#metadata'
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/zone-id/instances/inst_id/setMetadata',
},
{
'name': 'operation',
'status': 'DONE',
},
),
])
op = gce.Project('123').set_metadata('zone-id', 'inst_id',
'fingerprint', [{
'key': 'k',
'value': 'v'
}])
self.assertTrue(op.done)
def _delete(instance_template_revision, instance_group_manager, instance):
"""Deletes the given instance.
Args:
instance_template_revision: models.InstanceTemplateRevision.
instance_group_manager: models.InstanceGroupManager.
instance: models.Instance
"""
api = gce.Project(instance_template_revision.project)
try:
result = api.delete_instances(
instance_group_managers.get_name(instance_group_manager),
instance_group_manager.key.id(),
[instance.url],
)
if result['status'] != 'DONE':
logging.warning(
'Instance group manager operation failed: %s\n%s',
parent.key,
json.dumps(result, indent=2),
)
except net.Error as e:
if e.status_code != 400:
# If the instance isn't found, assume it's already deleted.
raise
def resize(key):
"""Resizes the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
"""
entity = key.get()
if not entity:
logging.warning('InstanceGroupManager does not exist: %s', key)
return
if not entity.url:
logging.warning('InstanceGroupManager URL unspecified: %s', key)
parent = key.parent().get()
if not parent:
logging.warning('InstanceTemplateRevision does not exist: %s', key)
return
if not parent.project:
logging.warning('InstanceTemplateRevision project unspecified: %s', key)
return
# For now, just ensure a minimum size.
if entity.current_size >= entity.minimum_size:
return
api = gce.Project(parent.project)
api.resize_managed_instance_group(
get_name(entity), key.id(), entity.minimum_size)
def create(key):
"""Creates an instance group manager from the given InstanceGroupManager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
Raises:
net.Error: HTTP status code is not 200 (created) or 409 (already created).
"""
entity = key.get()
if not entity:
logging.warning('InstanceGroupManager does not exist: %s', key)
return
if entity.url:
logging.warning(
'Instance group manager for InstanceGroupManager already exists: %s',
key,
)
return
parent = key.parent().get()
if not parent:
logging.warning('InstanceTemplateRevision does not exist: %s', key.parent())
return
if not parent.project:
logging.warning(
'InstanceTemplateRevision project unspecified: %s', key.parent())
return
if not parent.url:
logging.warning(
'InstanceTemplateRevision URL unspecified: %s', key.parent())
return
api = gce.Project(parent.project)
try:
result = api.create_instance_group_manager(
get_name(entity),
parent.url,
entity.minimum_size,
entity.key.id(),
base_name=get_base_name(entity),
)
except net.Error as e:
if e.status_code == 409:
# If the instance template already exists, just record the URL.
result = api.get_instance_group_manager(get_name(entity), entity.key.id())
update_url(entity.key, result['selfLink'])
return
else:
raise
update_url(entity.key, result['targetLink'])
def test_get_instance(self):
self.mock_requests([
(
{
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/zone-id/instances/inst_id',
},
instance('inst_id'),
),
])
self.assertEqual(instance('inst_id'),
gce.Project('123').get_instance('zone-id', 'inst_id'))
def update(key):
"""Updates instance metadata.
Args:
key: ndb.Key for a models.instance entity.
"""
entity = key.get()
if not entity:
logging.warning('Instance does not exist: %s', key)
return
if not entity.active_metadata_update:
logging.warning('Instance active metadata update unspecified: %s', key)
return
if entity.active_metadata_update.url:
return
parent = key.parent().get()
if not parent:
logging.warning('InstanceGroupManager does not exist: %s',
key.parent())
return
grandparent = parent.key.parent().get()
if not grandparent:
logging.warning('InstanceTemplateRevision does not exist: %s',
parent.key.parent())
return
if not grandparent.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
grandparent.key)
return
result = net.json_request(entity.url, scopes=gce.AUTH_SCOPES)
api = gce.Project(grandparent.project)
operation = api.set_metadata(
parent.key.id(),
key.id(),
result['metadata']['fingerprint'],
apply_metadata_update(result['metadata']['items'],
entity.active_metadata_update.metadata),
)
metrics.send_machine_event('METADATA_UPDATE_SCHEDULED', key.id())
associate_metadata_operation(
key,
utilities.compute_checksum(entity.active_metadata_update.metadata),
operation.url,
)
def get(self):
# For each template entry in the datastore, create a group manager.
for template in models.InstanceTemplate.query():
logging.info(
'Retrieving instance template %s from project %s',
template.template_name,
template.template_project,
)
api = gce.Project(template.template_project)
try:
instance_template = api.get_instance_template(
template.template_name)
except net.NotFoundError:
logging.error(
'Instance template does not exist: %s',
template.template_name,
)
continue
api = gce.Project(template.instance_group_project)
try:
api.create_instance_group_manager(
template.instance_group_name,
instance_template,
template.initial_size,
template.zone,
)
except net.Error as e:
if e.status_code == 409:
logging.info(
'Instance group manager already exists: %s',
template.template_name,
)
else:
logging.error(
'Could not create instance group manager: %s\n%s',
template.template_name,
e,
)
def fetch(key):
"""Gets instances created by the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
Returns:
A list of instance URLs.
"""
entity = key.get()
if not entity:
logging.warning('InstanceGroupManager does not exist: %s', key)
return []
if not entity.url:
logging.warning('InstanceGroupManager URL unspecified: %s', key)
return []
parent = key.parent().get()
if not parent:
logging.warning('InstanceTemplateRevision does not exist: %s',
key.parent())
return []
if not parent.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key.parent())
return []
api = gce.Project(parent.project)
result = api.get_instances_in_instance_group(
instance_group_managers.get_name(entity),
entity.key.id(),
max_results=500,
)
instance_urls = [
instance['instance'] for instance in result.get('items', [])
]
while result.get('nextPageToken'):
result = api.get_instances_in_instance_group(
instance_group_managers.get_name(entity),
entity.key.id(),
max_results=500,
page_token=result['nextPageToken'],
)
instance_urls.extend(
[instance['instance'] for instance in result['items']])
return instance_urls
def test_yield_instances(self):
self.mock_requests([
(
{
'deadline':
120,
'params': {
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/aggregated/instances',
},
{
'items': {
'zone1': {
'instances': [instance('a')]
},
'zone2': {
'instances': [instance('b')]
},
},
'nextPageToken': 'page-token',
},
),
(
{
'deadline':
120,
'params': {
'maxResults': 250,
'pageToken': 'page-token',
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/aggregated/instances',
},
{
'items': {
'zone1': {
'instances': [instance('c')]
},
},
},
),
])
result = list(gce.Project('123').yield_instances())
self.assertEqual([instance('a'), instance('b'), instance('c')], result)
def test_get_instance_with_fields(self):
self.mock_requests([
(
{
'params': {'fields': 'metadata,name'},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/zone-id/instances/inst_id',
},
instance('inst_id'),
),
])
self.assertEqual(
instance('inst_id'),
gce.Project('123').get_instance(
'zone-id', 'inst_id', ['metadata', 'name']))
def post(self):
"""Updates GCE instance metadata.
Params:
group: Name of the instance group containing the instances to update.
instance_map: JSON-encoded dict of instances mapped to metadata to set.
project: Name of the project the instance group exists in.
zone: Zone the instances exist in. e.g. us-central1-f.
"""
group = self.request.get('group')
instance_map = json.loads(self.request.get('instance_map'))
project = self.request.get('project')
zone = self.request.get('zone')
api = gce.Project(project)
succeeded = {}
failed = []
for instance in instance_map:
new_metadata = instance_map[instance]
try:
existing_metadata = api.get_instance(zone,
instance,
fields=['metadata'])
except net.Error:
existing_metadata = None
if not existing_metadata or not existing_metadata['metadata']:
failed.append(instance)
else:
fingerprint = existing_metadata['metadata']['fingerprint']
items = existing_metadata['metadata']['items']
items.extend({
'key': k,
'value': v
} for k, v in new_metadata.iteritems())
logging.info('New metadata:\n%s', json.dumps(items, indent=2))
try:
operation = api.set_metadata(zone, instance, fingerprint,
items)
succeeded[instance] = operation.name
except net.Error:
failed.append(instance)
set_updating_instance_states(models.InstanceGroup.generate_key(group),
succeeded, failed)
def test_zone_operation_error(self):
op = gce.ZoneOperation(
gce.Project('123'),
'zone-id',
{
'name': 'op',
'status': 'DONE',
'error': {
'errors': [
{'message': 'A', 'code': 'ERROR_CODE'},
{'message': 'B'},
],
},
})
self.assertTrue(op.has_error_code('ERROR_CODE'))
self.assertFalse(op.has_error_code('NOT_ERROR_CODE'))
self.assertEqual('A B', op.error)
def test_list_addresses(self):
self.mock_requests([
(
{
'deadline':
120,
'params': {
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/regions/region-id/addresses',
},
{
'items': [{
'name': 'a'
}, {
'name': 'b'
}],
'nextPageToken': 'page-token',
},
),
(
{
'deadline':
120,
'params': {
'maxResults': 250,
'pageToken': 'page-token',
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/regions/region-id/addresses',
},
{
'items': [{
'name': 'c'
}],
},
),
])
result = list(gce.Project('123').list_addresses('region-id'))
self.assertEqual([{'name': 'a'}, {'name': 'b'}, {'name': 'c'}], result)
def post(self):
"""Deletes GCE instances from an instance group.
Params:
group: Name of the instance group containing the instances to delete.
instance_map: JSON-encoded dict mapping instance names to instance URLs
in the instance group which should be deleted.
project: Name of the project the instance group exists in.
zone: Zone the instances exist in. e.g. us-central1-f.
"""
group = self.request.get('group')
instance_map = json.loads(self.request.get('instance_map'))
project = self.request.get('project')
zone = self.request.get('zone')
instance_group_key = models.InstanceGroup.generate_key(group)
instances = sorted(instance_map.keys())
logging.info(
'Deleting instances from instance group: %s\n%s',
group,
', '.join(instances),
)
api = gce.Project(project)
# Try to delete the instances. If the operation succeeds, update the
# datastore. If it fails, don't update the datastore which will make us
# try again later.
# TODO(smut): Resize the instance group.
# When instances are deleted from an instance group, the instance group's
# size is decreased by the number of deleted instances. We need to resize
# the group back to its original size in order to replace those deleted
# instances.
try:
response = api.delete_instances(group, zone, instance_map.values())
if response.get('status') == 'DONE':
# Either they all succeed or they all fail. If they all succeeded,
# remove them from the datastore and return. In all other cases,
# set them back to PENDING_DELETION to try again later.
delete_instances(instance_group_key, instances)
return
except net.Error as e:
logging.warning('%s', e)
reschedule_instance_deletion(instance_group_key, instances)
def _delete(instance_template_revision, instance_group_manager, instance):
"""Deletes the given instance.
Args:
instance_template_revision: models.InstanceTemplateRevision.
instance_group_manager: models.InstanceGroupManager.
instance: models.Instance
"""
# We don't check if there are any pending deletion calls because we don't
# care. We just want the instance to be deleted, so we make repeated calls
# until the instance is no longer detected. However, we do care how long
# it takes, so we only write instance.deletion_ts once.
api = gce.Project(instance_template_revision.project)
try:
now = utils.utcnow()
result = api.delete_instances(
instance_group_managers.get_name(instance_group_manager),
instance_group_manager.key.id(),
[instance.url],
)
if result['status'] != 'DONE':
# This is not the status of the instance deletion, it's the status of
# scheduling the instance deletions in the managed instance group. If
# it's not DONE, the deletions won't even be attempted. If it is DONE,
# the actual deletions may still fail.
logging.warning(
'Instance group manager operation failed: %s\n%s',
instance_group_manager.key,
json.dumps(result, indent=2),
)
else:
if not instance.deletion_ts:
set_deletion_time(instance.key, now)
metrics.send_machine_event('DELETION_SCHEDULED',
instance.hostname)
except net.Error as e:
if e.status_code == 400:
if not instance.deletion_ts:
set_deletion_time(instance.key, now)
else:
raise
def post(self):
"""Prepares GCE instances for use.
Params:
group: Name of the instance group containing the instances to prepare.
instance_map: JSON-encoded dict of instances to prepare.
project: Name of the project the instance group exists in.
zone: Zone the instances exist in. e.g. us-central1-f.
"""
group = self.request.get('group')
instance_map = json.loads(self.request.get('instance_map'))
project = self.request.get('project')
zone = self.request.get('zone')
api = gce.Project(project)
succeeded = {}
failed = []
# Get the default service account of each instance and set it as the
# instance's Cloud Pub/Sub service account. This service account will
# be sent to the Machine Provider to be authorized to subscribe to the
# machine topic to listen for instructions from Machine Provider.
for instance in instance_map:
try:
service_accounts = api.get_instance(zone,
instance,
fields=['serviceAccounts'])
except net.Error:
service_accounts = None
if not service_accounts or not service_accounts['serviceAccounts']:
failed.append(instance)
else:
# Just assume the first service account is the default.
succeeded[instance] = service_accounts['serviceAccounts'][0][
'email']
# TODO(smut): Any additional preparation.
set_prepared_instance_states(models.InstanceGroup.generate_key(group),
succeeded, failed)
def test_zone_operation_poll(self):
self.mock_requests([
(
{
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/zone-id/operations/op',
},
{
'name': 'op',
'status': 'DONE',
},
),
])
op = gce.ZoneOperation(
gce.Project('123'), 'zone-id', {'name': 'op', 'status': 'PENDING'})
self.assertFalse(op.done)
self.assertFalse(op.error)
self.assertTrue(op.poll())
# Second 'poll' is skipped if the operation is already done.
self.assertTrue(op.poll())
def post(self):
"""Checks that GCE instance metadata has been updated.
Params:
group: Name of the instance group containing the instances to update.
instance_map: JSON-encoded dict of instances mapped to metadata to set.
project: Name of the project the instance group exists in.
zone: Zone the instances exist in. e.g. us-central1-f.
"""
group = self.request.get('group')
instance_map = json.loads(self.request.get('instance_map'))
project = self.request.get('project')
zone = self.request.get('zone')
api = gce.Project(project)
succeeded = []
failed = []
for instance in instance_map:
logging.info('Checking on metadata operation: %s',
instance_map[instance])
try:
result = api.check_zone_operation(zone, instance_map[instance])
# If the operation hasn't completed, consider it neither
# succeeded nor failed. Instead just check again later.
if result['status'] == 'DONE':
if result.get('error', {}).get('errors'):
failed.append(instance)
else:
succeeded.append(instance)
except net.Error:
failed.append(instance)
set_updated_instance_states(models.InstanceGroup.generate_key(group),
succeeded, failed)
def test_project_id(self):
self.assertEqual('123', gce.Project('123').project_id)
def test_yield_instances_bad_filter(self):
with self.assertRaises(ValueError):
list(gce.Project('123').yield_instances('"'))
def resize(key):
"""Resizes the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
"""
# To avoid a massive resize, impose a limit on how much larger we can
# resize the instance group. Repeated calls will eventually allow the
# instance group to reach its target size. Cron timing together with
# this limit controls the rate at which instances are created.
RESIZE_LIMIT = 100
# Ratio of total instances to leased instances.
THRESHOLD = 1.1
instance_group_manager = key.get()
if not instance_group_manager:
logging.warning('InstanceGroupManager does not exist: %s', key)
return
if not instance_group_manager.url:
logging.warning('InstanceGroupManager URL unspecified: %s', key)
return
instance_template_revision = key.parent().get()
if not instance_template_revision:
logging.warning('InstanceTemplateRevision does not exist: %s', key)
return
if not instance_template_revision.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key)
return
# Determine how many total instances exist for all other revisions of this
# InstanceGroupManager. Different revisions will all have the same
# ancestral InstanceTemplate.
instance_template_key = instance_template_revision.key.parent()
other_revision_total_size = 0
for igm in models.InstanceGroupManager.query(
ancestor=instance_template_key):
# Find InstanceGroupManagers in the same zone, except the one being resized.
if igm.key.id() == key.id() and igm.key != key:
logging.info(
'Found another revision of InstanceGroupManager: %s\nSize: %s',
igm.key,
igm.current_size,
)
other_revision_total_size += igm.current_size
# Determine how many total instances for this revision of the
# InstanceGroupManager have been leased out by the Machine Provider
leased = 0
for instance in models.Instance.query(
models.Instance.instance_group_manager == key):
if instance.leased:
leased += 1
api = gce.Project(instance_template_revision.project)
response = api.get_instance_group_manager(get_name(instance_group_manager),
key.id())
# Find out how many instances are idle (i.e. not currently being created
# or deleted). This helps avoid doing too many VM actions simultaneously.
current_size = response.get('currentActions', {}).get('none')
if current_size is None:
logging.error('Unexpected response: %s', json.dumps(response,
indent=2))
return
# Ensure there are at least as many instances as needed, but not more than
# the total allowed at this time.
new_target_size = int(
min(
# Minimum size to aim for. At least THRESHOLD times more than the number
# of instances already leased out, but not less than the minimum
# configured size. If the THRESHOLD suggests we need a fraction of an
# instance, we need to provide at least one additional whole instance.
max(instance_group_manager.minimum_size,
math.ceil(leased * THRESHOLD)),
# Total number of instances for this instance group allowed at this time.
# Ensures that a config change waits for instances of the old revision to
# be deleted before bringing up instances of the new revision.
instance_group_manager.maximum_size - other_revision_total_size,
# Maximum amount the size is allowed to be increased each iteration.
current_size + RESIZE_LIMIT,
))
logging.info(
('Key: %s\nSize: %s\nOld target: %s\nNew target: %s\nMin: %s\nMax: %s'
'\nLeased: %s\nOther revisions: %s'),
key,
current_size,
response['targetSize'],
new_target_size,
instance_group_manager.minimum_size,
instance_group_manager.maximum_size,
leased,
other_revision_total_size,
)
if new_target_size <= min(current_size, response['targetSize']):
return
api.resize_managed_instance_group(response['name'], key.id(),
new_target_size)
def test_yield_instances_in_zones(self):
self.mock_requests([
(
{
'deadline': 120,
'params': {
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/z1/instances',
},
{
'items': [instance('a')],
'nextPageToken': 'page-token',
},
),
(
{
'deadline': 120,
'params': {
'maxResults': 250,
'pageToken': 'page-token',
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/z1/instances',
},
{
'items': [instance('b')],
},
),
(
{
'deadline': 120,
'params': {
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/z2/instances',
},
{
'items': [instance('c')],
},
),
(
{
'deadline': 120,
'params': {
'maxResults': 250,
},
'url':
'https://www.googleapis.com/compute/v1/projects/123'
'/zones/z3/instances',
},
# Missing zone is ignored.
net.Error('Bad request', 400, json.dumps({
'error': {
'errors': [{
'domain': 'global',
'reason': 'invalid',
'message':
'Invalid value for field \'zone\': \'z3\'. Unknown zone.',
}],
'code': 400,
'message':
'Invalid value for field \'zone\': \'z3\'. Unknown zone.',
},
})),
),
])
result = gce.Project('123').yield_instances_in_zones(['z1', 'z2', 'z3'])
self.assertEqual(
[instance('a'), instance('b'), instance('c')], list(result))
def create(key):
"""Creates an instance template from the given InstanceTemplateRevision.
Args:
key: ndb.Key for a models.InstanceTemplateRevision entity.
Raises:
net.Error: HTTP status code is not 200 (created) or 409 (already created).
"""
entity = key.get()
if not entity:
logging.warning('InstanceTemplateRevision does not exist: %s', key)
return
if not entity.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key)
return
if entity.url:
logging.info(
'Instance template for InstanceTemplateRevision already exists: %s',
key,
)
return
if entity.metadata:
metadata = [{
'key': key,
'value': value
} for key, value in entity.metadata.iteritems()]
else:
metadata = []
service_accounts = [{
'email': service_account.name,
'scopes': service_account.scopes
} for service_account in entity.service_accounts]
api = gce.Project(entity.project)
try:
result = api.create_instance_template(
get_name(entity),
entity.disk_size_gb,
gce.get_image_url(api.project_id, entity.image_name),
entity.machine_type,
gce.get_network_url(api.project_id, 'default'),
tags=entity.tags,
metadata=metadata,
service_accounts=service_accounts,
)
except net.Error as e:
if e.status_code == 409:
# If the instance template already exists, just record the URL.
result = api.get_instance_template(get_name(entity))
update_url(entity.key, result['selfLink'])
return
else:
raise
update_url(entity.key, result['targetLink'])
def resize(key):
"""Resizes the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
"""
# To avoid a massive resize, impose a limit on how much larger we can
# resize the instance group. Repeated calls will eventually allow the
# instance group to reach its target size. Cron timing together with
# this limit controls the rate at which instances are created.
RESIZE_LIMIT = 100
entity = key.get()
if not entity:
logging.warning('InstanceGroupManager does not exist: %s', key)
return
if not entity.url:
logging.warning('InstanceGroupManager URL unspecified: %s', key)
parent = key.parent().get()
if not parent:
logging.warning('InstanceTemplateRevision does not exist: %s', key)
return
if not parent.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key)
return
# Determine how many total VMs exist for all other revisions of this
# InstanceGroupManager. Different revisions will all have the same
# grandparent InstanceTemplate.
other_revision_total_size = 0
for igm in models.InstanceGroupManager.query(ancestor=parent.key.parent()):
# Find InstanceGroupManagers in the same zone, except the one being resized.
if igm.key.id() == key.id() and igm.key != key:
logging.info(
'Found another revision of InstanceGroupManager: %s\nSize: %s',
igm.key,
igm.current_size,
)
other_revision_total_size += igm.current_size
api = gce.Project(parent.project)
response = api.get_instance_group_manager(get_name(entity), key.id())
# Find out how many VMs are idle (i.e. not currently being created
# or deleted). This helps avoid doing too many VM actions simultaneously.
current_size = response.get('currentActions', {}).get('none')
if current_size is None:
logging.error('Unexpected response: %s', json.dumps(response,
indent=2))
return
# Try to reach the configured size less the number that exist in other
# revisions of the InstanceGroupManager, but avoid increasing the number of
# instances by more than the resize limit. For now, the target size
# is just the minimum size.
target_size = min(
entity.minimum_size - other_revision_total_size,
current_size + RESIZE_LIMIT,
)
logging.info(
'Key: %s\nSize: %s\nTarget: %s\nMin: %s\nMax: %s\nOther revisions: %s',
key,
current_size,
target_size,
entity.minimum_size,
entity.maximum_size,
other_revision_total_size,
)
if target_size <= current_size:
return
api.resize_managed_instance_group(response['name'], key.id(), target_size)
def get(self):
pubsub_handler = handlers_pubsub.MachineProviderSubscriptionHandler
if not pubsub_handler.is_subscribed():
logging.error(
'Pub/Sub subscription not created:\n%s',
pubsub_handler.get_subscription_name(),
)
return
# For each group manager, tell the Machine Provider about its instances.
for template in models.InstanceTemplate.query():
logging.info(
'Retrieving instance template %s from project %s',
template.template_name,
template.template_project,
)
api = gce.Project(template.template_project)
try:
instance_template = api.get_instance_template(
template.template_name)
except net.NotFoundError:
logging.error(
'Instance template does not exist: %s',
template.template_name,
)
continue
api = gce.Project(template.instance_group_project)
properties = instance_template['properties']
disk_gb = int(
properties['disks'][0]['initializeParams']['diskSizeGb'])
memory_gb = float(
gce.machine_type_to_memory(properties['machineType']))
num_cpus = gce.machine_type_to_num_cpus(properties['machineType'])
os_family = machine_provider.OSFamily.lookup_by_name(
template.os_family)
dimensions = machine_provider.Dimensions(
backend=machine_provider.Backend.GCE,
disk_gb=disk_gb,
memory_gb=memory_gb,
num_cpus=num_cpus,
os_family=os_family,
)
try:
instances = api.get_managed_instances(
template.instance_group_name, template.zone)
except net.NotFoundError:
logging.warning(
'Instance group manager does not exist: %s',
template.instance_group_name,
)
continue
policies = machine_provider.Policies(
backend_attributes=[
machine_provider.KeyValuePair(
key='group', value=template.instance_group_name),
],
backend_project=handlers_pubsub.
MachineProviderSubscriptionHandler.TOPIC_PROJECT,
backend_topic=handlers_pubsub.
MachineProviderSubscriptionHandler.TOPIC,
on_reclamation=machine_provider.MachineReclamationPolicy.
DELETE,
)
process_instance_group(
template.instance_group_name,
dimensions,
policies,
instances,
template.zone,
template.instance_group_project,
)
def create(key):
"""Creates an instance template from the given InstanceTemplateRevision.
Args:
key: ndb.Key for a models.InstanceTemplateRevision entity.
Raises:
net.Error: HTTP status code is not 200 (created) or 409 (already created).
"""
instance_template_revision = key.get()
if not instance_template_revision:
logging.warning('InstanceTemplateRevision does not exist: %s', key)
return
if not instance_template_revision.project:
logging.warning('InstanceTemplateRevision project unspecified: %s',
key)
return
if instance_template_revision.url:
logging.info(
'Instance template for InstanceTemplateRevision already exists: %s',
key,
)
return
if instance_template_revision.metadata:
metadata = [{
'key': k,
'value': v
} for k, v in instance_template_revision.metadata.iteritems()]
else:
metadata = []
service_accounts = [{
'email': service_account.name,
'scopes': service_account.scopes
} for service_account in instance_template_revision.service_accounts]
api = gce.Project(instance_template_revision.project)
try:
image_project = api.project_id
if instance_template_revision.image_project:
image_project = instance_template_revision.image_project
result = api.create_instance_template(
get_name(instance_template_revision),
instance_template_revision.disk_size_gb,
gce.get_image_url(image_project,
instance_template_revision.image_name),
instance_template_revision.machine_type,
auto_assign_external_ip=instance_template_revision.
auto_assign_external_ip,
disk_type=instance_template_revision.disk_type,
metadata=metadata,
min_cpu_platform=instance_template_revision.min_cpu_platform,
network_url=instance_template_revision.network_url,
service_accounts=service_accounts,
tags=instance_template_revision.tags,
)
except net.Error as e:
if e.status_code == 409:
# If the instance template already exists, just record the URL.
result = api.get_instance_template(
get_name(instance_template_revision))
update_url(instance_template_revision.key, result['selfLink'])
return
else:
raise
update_url(instance_template_revision.key, result['targetLink'])
