def create(key): """Creates an instance group manager from the given InstanceGroupManager. Args: key: ndb.Key for a models.InstanceGroupManager entity. Raises: net.Error: HTTP status code is not 200 (created) or 409 (already created). """ instance_group_manager = key.get() if not instance_group_manager: logging.warning('InstanceGroupManager does not exist: %s', key) return if instance_group_manager.url: logging.warning( 'Instance group manager for InstanceGroupManager already exists: %s', key, ) return instance_template_revision = key.parent().get() if not instance_template_revision: logging.warning('InstanceTemplateRevision does not exist: %s', key.parent()) return if not instance_template_revision.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key.parent()) return if not instance_template_revision.url: logging.warning('InstanceTemplateRevision URL unspecified: %s', key.parent()) return api = gce.Project(instance_template_revision.project) try: # Create the instance group manager with 0 instances. The resize cron job # will adjust this later. result = api.create_instance_group_manager( get_name(instance_group_manager), instance_template_revision.url, 0, instance_group_manager.key.id(), base_name=get_base_name(instance_group_manager), ) except net.Error as e: if e.status_code == 409: # If the instance template already exists, just record the URL. result = api.get_instance_group_manager( get_name(instance_group_manager), instance_group_manager.key.id()) update_url(instance_group_manager.key, result['selfLink']) return else: raise update_url(instance_group_manager.key, result['targetLink'])
def test_yield_instances_with_filter(self): self.mock_requests([ ( { 'deadline': 120, 'params': { 'filter': 'name eq "inst-filter"', 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/aggregated/instances', }, { 'items': { 'zone1': { 'instances': [instance('a')] }, 'zone2': { 'instances': [instance('b', status='STOPPED')] }, }, }, ), ]) result = list(gce.Project('123').yield_instances('inst-filter')) self.assertEqual( [instance('a'), instance('b', status='STOPPED')], result)
def _delete(instance_template_revision, instance_group_manager, instance): """Deletes the given instance. Args: instance_template_revision: models.InstanceTemplateRevision. instance_group_manager: models.InstanceGroupManager. instance: models.Instance """ api = gce.Project(instance_template_revision.project) try: result = api.delete_instances( instance_group_managers.get_name(instance_group_manager), instance_group_manager.key.id(), [instance.url], ) if result['status'] != 'DONE': logging.warning( 'Instance group manager operation failed: %s\n%s', parent.key, json.dumps(result, indent=2), ) else: metrics.send_machine_event('DELETION_SCHEDULED', instance.key.id()) except net.Error as e: if e.status_code == 400: metrics.send_machine_event('DELETION_SUCCEEDED', instance.key.id()) else: raise
def test_add_access_config(self): self.mock_requests([ ( { 'method': 'POST', 'params': { 'networkInterface': 'nic0' }, 'payload': { 'kind': 'compute#accessConfig', 'name': 'External NAT', 'natIP': '1.2.3.4', 'type': 'ONE_TO_ONE_NAT', }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/zone-id/instances/inst_id/addAccessConfig', }, { 'name': 'operation', 'status': 'DONE', }, ), ]) op = gce.Project('123').add_access_config('zone-id', 'inst_id', 'nic0', '1.2.3.4') self.assertTrue(op.done)
def test_set_metadata(self): self.mock_requests([ ( { 'method': 'POST', 'payload': { 'fingerprint': 'fingerprint', 'items': [{ 'key': 'k', 'value': 'v' }], 'kind': 'compute#metadata' }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/zone-id/instances/inst_id/setMetadata', }, { 'name': 'operation', 'status': 'DONE', }, ), ]) op = gce.Project('123').set_metadata('zone-id', 'inst_id', 'fingerprint', [{ 'key': 'k', 'value': 'v' }]) self.assertTrue(op.done)
def _delete(instance_template_revision, instance_group_manager, instance): """Deletes the given instance. Args: instance_template_revision: models.InstanceTemplateRevision. instance_group_manager: models.InstanceGroupManager. instance: models.Instance """ api = gce.Project(instance_template_revision.project) try: result = api.delete_instances( instance_group_managers.get_name(instance_group_manager), instance_group_manager.key.id(), [instance.url], ) if result['status'] != 'DONE': logging.warning( 'Instance group manager operation failed: %s\n%s', parent.key, json.dumps(result, indent=2), ) except net.Error as e: if e.status_code != 400: # If the instance isn't found, assume it's already deleted. raise
def resize(key): """Resizes the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. """ entity = key.get() if not entity: logging.warning('InstanceGroupManager does not exist: %s', key) return if not entity.url: logging.warning('InstanceGroupManager URL unspecified: %s', key) parent = key.parent().get() if not parent: logging.warning('InstanceTemplateRevision does not exist: %s', key) return if not parent.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key) return # For now, just ensure a minimum size. if entity.current_size >= entity.minimum_size: return api = gce.Project(parent.project) api.resize_managed_instance_group( get_name(entity), key.id(), entity.minimum_size)
def create(key): """Creates an instance group manager from the given InstanceGroupManager. Args: key: ndb.Key for a models.InstanceGroupManager entity. Raises: net.Error: HTTP status code is not 200 (created) or 409 (already created). """ entity = key.get() if not entity: logging.warning('InstanceGroupManager does not exist: %s', key) return if entity.url: logging.warning( 'Instance group manager for InstanceGroupManager already exists: %s', key, ) return parent = key.parent().get() if not parent: logging.warning('InstanceTemplateRevision does not exist: %s', key.parent()) return if not parent.project: logging.warning( 'InstanceTemplateRevision project unspecified: %s', key.parent()) return if not parent.url: logging.warning( 'InstanceTemplateRevision URL unspecified: %s', key.parent()) return api = gce.Project(parent.project) try: result = api.create_instance_group_manager( get_name(entity), parent.url, entity.minimum_size, entity.key.id(), base_name=get_base_name(entity), ) except net.Error as e: if e.status_code == 409: # If the instance template already exists, just record the URL. result = api.get_instance_group_manager(get_name(entity), entity.key.id()) update_url(entity.key, result['selfLink']) return else: raise update_url(entity.key, result['targetLink'])
def test_get_instance(self): self.mock_requests([ ( { 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/zone-id/instances/inst_id', }, instance('inst_id'), ), ]) self.assertEqual(instance('inst_id'), gce.Project('123').get_instance('zone-id', 'inst_id'))
def update(key): """Updates instance metadata. Args: key: ndb.Key for a models.instance entity. """ entity = key.get() if not entity: logging.warning('Instance does not exist: %s', key) return if not entity.active_metadata_update: logging.warning('Instance active metadata update unspecified: %s', key) return if entity.active_metadata_update.url: return parent = key.parent().get() if not parent: logging.warning('InstanceGroupManager does not exist: %s', key.parent()) return grandparent = parent.key.parent().get() if not grandparent: logging.warning('InstanceTemplateRevision does not exist: %s', parent.key.parent()) return if not grandparent.project: logging.warning('InstanceTemplateRevision project unspecified: %s', grandparent.key) return result = net.json_request(entity.url, scopes=gce.AUTH_SCOPES) api = gce.Project(grandparent.project) operation = api.set_metadata( parent.key.id(), key.id(), result['metadata']['fingerprint'], apply_metadata_update(result['metadata']['items'], entity.active_metadata_update.metadata), ) metrics.send_machine_event('METADATA_UPDATE_SCHEDULED', key.id()) associate_metadata_operation( key, utilities.compute_checksum(entity.active_metadata_update.metadata), operation.url, )
def get(self): # For each template entry in the datastore, create a group manager. for template in models.InstanceTemplate.query(): logging.info( 'Retrieving instance template %s from project %s', template.template_name, template.template_project, ) api = gce.Project(template.template_project) try: instance_template = api.get_instance_template( template.template_name) except net.NotFoundError: logging.error( 'Instance template does not exist: %s', template.template_name, ) continue api = gce.Project(template.instance_group_project) try: api.create_instance_group_manager( template.instance_group_name, instance_template, template.initial_size, template.zone, ) except net.Error as e: if e.status_code == 409: logging.info( 'Instance group manager already exists: %s', template.template_name, ) else: logging.error( 'Could not create instance group manager: %s\n%s', template.template_name, e, )
def fetch(key): """Gets instances created by the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. Returns: A list of instance URLs. """ entity = key.get() if not entity: logging.warning('InstanceGroupManager does not exist: %s', key) return [] if not entity.url: logging.warning('InstanceGroupManager URL unspecified: %s', key) return [] parent = key.parent().get() if not parent: logging.warning('InstanceTemplateRevision does not exist: %s', key.parent()) return [] if not parent.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key.parent()) return [] api = gce.Project(parent.project) result = api.get_instances_in_instance_group( instance_group_managers.get_name(entity), entity.key.id(), max_results=500, ) instance_urls = [ instance['instance'] for instance in result.get('items', []) ] while result.get('nextPageToken'): result = api.get_instances_in_instance_group( instance_group_managers.get_name(entity), entity.key.id(), max_results=500, page_token=result['nextPageToken'], ) instance_urls.extend( [instance['instance'] for instance in result['items']]) return instance_urls
def test_yield_instances(self): self.mock_requests([ ( { 'deadline': 120, 'params': { 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/aggregated/instances', }, { 'items': { 'zone1': { 'instances': [instance('a')] }, 'zone2': { 'instances': [instance('b')] }, }, 'nextPageToken': 'page-token', }, ), ( { 'deadline': 120, 'params': { 'maxResults': 250, 'pageToken': 'page-token', }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/aggregated/instances', }, { 'items': { 'zone1': { 'instances': [instance('c')] }, }, }, ), ]) result = list(gce.Project('123').yield_instances()) self.assertEqual([instance('a'), instance('b'), instance('c')], result)
def test_get_instance_with_fields(self): self.mock_requests([ ( { 'params': {'fields': 'metadata,name'}, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/zone-id/instances/inst_id', }, instance('inst_id'), ), ]) self.assertEqual( instance('inst_id'), gce.Project('123').get_instance( 'zone-id', 'inst_id', ['metadata', 'name']))
def post(self): """Updates GCE instance metadata. Params: group: Name of the instance group containing the instances to update. instance_map: JSON-encoded dict of instances mapped to metadata to set. project: Name of the project the instance group exists in. zone: Zone the instances exist in. e.g. us-central1-f. """ group = self.request.get('group') instance_map = json.loads(self.request.get('instance_map')) project = self.request.get('project') zone = self.request.get('zone') api = gce.Project(project) succeeded = {} failed = [] for instance in instance_map: new_metadata = instance_map[instance] try: existing_metadata = api.get_instance(zone, instance, fields=['metadata']) except net.Error: existing_metadata = None if not existing_metadata or not existing_metadata['metadata']: failed.append(instance) else: fingerprint = existing_metadata['metadata']['fingerprint'] items = existing_metadata['metadata']['items'] items.extend({ 'key': k, 'value': v } for k, v in new_metadata.iteritems()) logging.info('New metadata:\n%s', json.dumps(items, indent=2)) try: operation = api.set_metadata(zone, instance, fingerprint, items) succeeded[instance] = operation.name except net.Error: failed.append(instance) set_updating_instance_states(models.InstanceGroup.generate_key(group), succeeded, failed)
def test_zone_operation_error(self): op = gce.ZoneOperation( gce.Project('123'), 'zone-id', { 'name': 'op', 'status': 'DONE', 'error': { 'errors': [ {'message': 'A', 'code': 'ERROR_CODE'}, {'message': 'B'}, ], }, }) self.assertTrue(op.has_error_code('ERROR_CODE')) self.assertFalse(op.has_error_code('NOT_ERROR_CODE')) self.assertEqual('A B', op.error)
def test_list_addresses(self): self.mock_requests([ ( { 'deadline': 120, 'params': { 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/regions/region-id/addresses', }, { 'items': [{ 'name': 'a' }, { 'name': 'b' }], 'nextPageToken': 'page-token', }, ), ( { 'deadline': 120, 'params': { 'maxResults': 250, 'pageToken': 'page-token', }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/regions/region-id/addresses', }, { 'items': [{ 'name': 'c' }], }, ), ]) result = list(gce.Project('123').list_addresses('region-id')) self.assertEqual([{'name': 'a'}, {'name': 'b'}, {'name': 'c'}], result)
def post(self): """Deletes GCE instances from an instance group. Params: group: Name of the instance group containing the instances to delete. instance_map: JSON-encoded dict mapping instance names to instance URLs in the instance group which should be deleted. project: Name of the project the instance group exists in. zone: Zone the instances exist in. e.g. us-central1-f. """ group = self.request.get('group') instance_map = json.loads(self.request.get('instance_map')) project = self.request.get('project') zone = self.request.get('zone') instance_group_key = models.InstanceGroup.generate_key(group) instances = sorted(instance_map.keys()) logging.info( 'Deleting instances from instance group: %s\n%s', group, ', '.join(instances), ) api = gce.Project(project) # Try to delete the instances. If the operation succeeds, update the # datastore. If it fails, don't update the datastore which will make us # try again later. # TODO(smut): Resize the instance group. # When instances are deleted from an instance group, the instance group's # size is decreased by the number of deleted instances. We need to resize # the group back to its original size in order to replace those deleted # instances. try: response = api.delete_instances(group, zone, instance_map.values()) if response.get('status') == 'DONE': # Either they all succeed or they all fail. If they all succeeded, # remove them from the datastore and return. In all other cases, # set them back to PENDING_DELETION to try again later. delete_instances(instance_group_key, instances) return except net.Error as e: logging.warning('%s', e) reschedule_instance_deletion(instance_group_key, instances)
def _delete(instance_template_revision, instance_group_manager, instance): """Deletes the given instance. Args: instance_template_revision: models.InstanceTemplateRevision. instance_group_manager: models.InstanceGroupManager. instance: models.Instance """ # We don't check if there are any pending deletion calls because we don't # care. We just want the instance to be deleted, so we make repeated calls # until the instance is no longer detected. However, we do care how long # it takes, so we only write instance.deletion_ts once. api = gce.Project(instance_template_revision.project) try: now = utils.utcnow() result = api.delete_instances( instance_group_managers.get_name(instance_group_manager), instance_group_manager.key.id(), [instance.url], ) if result['status'] != 'DONE': # This is not the status of the instance deletion, it's the status of # scheduling the instance deletions in the managed instance group. If # it's not DONE, the deletions won't even be attempted. If it is DONE, # the actual deletions may still fail. logging.warning( 'Instance group manager operation failed: %s\n%s', instance_group_manager.key, json.dumps(result, indent=2), ) else: if not instance.deletion_ts: set_deletion_time(instance.key, now) metrics.send_machine_event('DELETION_SCHEDULED', instance.hostname) except net.Error as e: if e.status_code == 400: if not instance.deletion_ts: set_deletion_time(instance.key, now) else: raise
def post(self): """Prepares GCE instances for use. Params: group: Name of the instance group containing the instances to prepare. instance_map: JSON-encoded dict of instances to prepare. project: Name of the project the instance group exists in. zone: Zone the instances exist in. e.g. us-central1-f. """ group = self.request.get('group') instance_map = json.loads(self.request.get('instance_map')) project = self.request.get('project') zone = self.request.get('zone') api = gce.Project(project) succeeded = {} failed = [] # Get the default service account of each instance and set it as the # instance's Cloud Pub/Sub service account. This service account will # be sent to the Machine Provider to be authorized to subscribe to the # machine topic to listen for instructions from Machine Provider. for instance in instance_map: try: service_accounts = api.get_instance(zone, instance, fields=['serviceAccounts']) except net.Error: service_accounts = None if not service_accounts or not service_accounts['serviceAccounts']: failed.append(instance) else: # Just assume the first service account is the default. succeeded[instance] = service_accounts['serviceAccounts'][0][ 'email'] # TODO(smut): Any additional preparation. set_prepared_instance_states(models.InstanceGroup.generate_key(group), succeeded, failed)
def test_zone_operation_poll(self): self.mock_requests([ ( { 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/zone-id/operations/op', }, { 'name': 'op', 'status': 'DONE', }, ), ]) op = gce.ZoneOperation( gce.Project('123'), 'zone-id', {'name': 'op', 'status': 'PENDING'}) self.assertFalse(op.done) self.assertFalse(op.error) self.assertTrue(op.poll()) # Second 'poll' is skipped if the operation is already done. self.assertTrue(op.poll())
def post(self): """Checks that GCE instance metadata has been updated. Params: group: Name of the instance group containing the instances to update. instance_map: JSON-encoded dict of instances mapped to metadata to set. project: Name of the project the instance group exists in. zone: Zone the instances exist in. e.g. us-central1-f. """ group = self.request.get('group') instance_map = json.loads(self.request.get('instance_map')) project = self.request.get('project') zone = self.request.get('zone') api = gce.Project(project) succeeded = [] failed = [] for instance in instance_map: logging.info('Checking on metadata operation: %s', instance_map[instance]) try: result = api.check_zone_operation(zone, instance_map[instance]) # If the operation hasn't completed, consider it neither # succeeded nor failed. Instead just check again later. if result['status'] == 'DONE': if result.get('error', {}).get('errors'): failed.append(instance) else: succeeded.append(instance) except net.Error: failed.append(instance) set_updated_instance_states(models.InstanceGroup.generate_key(group), succeeded, failed)
def test_project_id(self): self.assertEqual('123', gce.Project('123').project_id)
def test_yield_instances_bad_filter(self): with self.assertRaises(ValueError): list(gce.Project('123').yield_instances('"'))
def resize(key): """Resizes the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. """ # To avoid a massive resize, impose a limit on how much larger we can # resize the instance group. Repeated calls will eventually allow the # instance group to reach its target size. Cron timing together with # this limit controls the rate at which instances are created. RESIZE_LIMIT = 100 # Ratio of total instances to leased instances. THRESHOLD = 1.1 instance_group_manager = key.get() if not instance_group_manager: logging.warning('InstanceGroupManager does not exist: %s', key) return if not instance_group_manager.url: logging.warning('InstanceGroupManager URL unspecified: %s', key) return instance_template_revision = key.parent().get() if not instance_template_revision: logging.warning('InstanceTemplateRevision does not exist: %s', key) return if not instance_template_revision.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key) return # Determine how many total instances exist for all other revisions of this # InstanceGroupManager. Different revisions will all have the same # ancestral InstanceTemplate. instance_template_key = instance_template_revision.key.parent() other_revision_total_size = 0 for igm in models.InstanceGroupManager.query( ancestor=instance_template_key): # Find InstanceGroupManagers in the same zone, except the one being resized. if igm.key.id() == key.id() and igm.key != key: logging.info( 'Found another revision of InstanceGroupManager: %s\nSize: %s', igm.key, igm.current_size, ) other_revision_total_size += igm.current_size # Determine how many total instances for this revision of the # InstanceGroupManager have been leased out by the Machine Provider leased = 0 for instance in models.Instance.query( models.Instance.instance_group_manager == key): if instance.leased: leased += 1 api = gce.Project(instance_template_revision.project) response = api.get_instance_group_manager(get_name(instance_group_manager), key.id()) # Find out how many instances are idle (i.e. not currently being created # or deleted). This helps avoid doing too many VM actions simultaneously. current_size = response.get('currentActions', {}).get('none') if current_size is None: logging.error('Unexpected response: %s', json.dumps(response, indent=2)) return # Ensure there are at least as many instances as needed, but not more than # the total allowed at this time. new_target_size = int( min( # Minimum size to aim for. At least THRESHOLD times more than the number # of instances already leased out, but not less than the minimum # configured size. If the THRESHOLD suggests we need a fraction of an # instance, we need to provide at least one additional whole instance. max(instance_group_manager.minimum_size, math.ceil(leased * THRESHOLD)), # Total number of instances for this instance group allowed at this time. # Ensures that a config change waits for instances of the old revision to # be deleted before bringing up instances of the new revision. instance_group_manager.maximum_size - other_revision_total_size, # Maximum amount the size is allowed to be increased each iteration. current_size + RESIZE_LIMIT, )) logging.info( ('Key: %s\nSize: %s\nOld target: %s\nNew target: %s\nMin: %s\nMax: %s' '\nLeased: %s\nOther revisions: %s'), key, current_size, response['targetSize'], new_target_size, instance_group_manager.minimum_size, instance_group_manager.maximum_size, leased, other_revision_total_size, ) if new_target_size <= min(current_size, response['targetSize']): return api.resize_managed_instance_group(response['name'], key.id(), new_target_size)
def test_yield_instances_in_zones(self): self.mock_requests([ ( { 'deadline': 120, 'params': { 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/z1/instances', }, { 'items': [instance('a')], 'nextPageToken': 'page-token', }, ), ( { 'deadline': 120, 'params': { 'maxResults': 250, 'pageToken': 'page-token', }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/z1/instances', }, { 'items': [instance('b')], }, ), ( { 'deadline': 120, 'params': { 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/z2/instances', }, { 'items': [instance('c')], }, ), ( { 'deadline': 120, 'params': { 'maxResults': 250, }, 'url': 'https://www.googleapis.com/compute/v1/projects/123' '/zones/z3/instances', }, # Missing zone is ignored. net.Error('Bad request', 400, json.dumps({ 'error': { 'errors': [{ 'domain': 'global', 'reason': 'invalid', 'message': 'Invalid value for field \'zone\': \'z3\'. Unknown zone.', }], 'code': 400, 'message': 'Invalid value for field \'zone\': \'z3\'. Unknown zone.', }, })), ), ]) result = gce.Project('123').yield_instances_in_zones(['z1', 'z2', 'z3']) self.assertEqual( [instance('a'), instance('b'), instance('c')], list(result))
def create(key): """Creates an instance template from the given InstanceTemplateRevision. Args: key: ndb.Key for a models.InstanceTemplateRevision entity. Raises: net.Error: HTTP status code is not 200 (created) or 409 (already created). """ entity = key.get() if not entity: logging.warning('InstanceTemplateRevision does not exist: %s', key) return if not entity.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key) return if entity.url: logging.info( 'Instance template for InstanceTemplateRevision already exists: %s', key, ) return if entity.metadata: metadata = [{ 'key': key, 'value': value } for key, value in entity.metadata.iteritems()] else: metadata = [] service_accounts = [{ 'email': service_account.name, 'scopes': service_account.scopes } for service_account in entity.service_accounts] api = gce.Project(entity.project) try: result = api.create_instance_template( get_name(entity), entity.disk_size_gb, gce.get_image_url(api.project_id, entity.image_name), entity.machine_type, gce.get_network_url(api.project_id, 'default'), tags=entity.tags, metadata=metadata, service_accounts=service_accounts, ) except net.Error as e: if e.status_code == 409: # If the instance template already exists, just record the URL. result = api.get_instance_template(get_name(entity)) update_url(entity.key, result['selfLink']) return else: raise update_url(entity.key, result['targetLink'])
def resize(key): """Resizes the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. """ # To avoid a massive resize, impose a limit on how much larger we can # resize the instance group. Repeated calls will eventually allow the # instance group to reach its target size. Cron timing together with # this limit controls the rate at which instances are created. RESIZE_LIMIT = 100 entity = key.get() if not entity: logging.warning('InstanceGroupManager does not exist: %s', key) return if not entity.url: logging.warning('InstanceGroupManager URL unspecified: %s', key) parent = key.parent().get() if not parent: logging.warning('InstanceTemplateRevision does not exist: %s', key) return if not parent.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key) return # Determine how many total VMs exist for all other revisions of this # InstanceGroupManager. Different revisions will all have the same # grandparent InstanceTemplate. other_revision_total_size = 0 for igm in models.InstanceGroupManager.query(ancestor=parent.key.parent()): # Find InstanceGroupManagers in the same zone, except the one being resized. if igm.key.id() == key.id() and igm.key != key: logging.info( 'Found another revision of InstanceGroupManager: %s\nSize: %s', igm.key, igm.current_size, ) other_revision_total_size += igm.current_size api = gce.Project(parent.project) response = api.get_instance_group_manager(get_name(entity), key.id()) # Find out how many VMs are idle (i.e. not currently being created # or deleted). This helps avoid doing too many VM actions simultaneously. current_size = response.get('currentActions', {}).get('none') if current_size is None: logging.error('Unexpected response: %s', json.dumps(response, indent=2)) return # Try to reach the configured size less the number that exist in other # revisions of the InstanceGroupManager, but avoid increasing the number of # instances by more than the resize limit. For now, the target size # is just the minimum size. target_size = min( entity.minimum_size - other_revision_total_size, current_size + RESIZE_LIMIT, ) logging.info( 'Key: %s\nSize: %s\nTarget: %s\nMin: %s\nMax: %s\nOther revisions: %s', key, current_size, target_size, entity.minimum_size, entity.maximum_size, other_revision_total_size, ) if target_size <= current_size: return api.resize_managed_instance_group(response['name'], key.id(), target_size)
def get(self): pubsub_handler = handlers_pubsub.MachineProviderSubscriptionHandler if not pubsub_handler.is_subscribed(): logging.error( 'Pub/Sub subscription not created:\n%s', pubsub_handler.get_subscription_name(), ) return # For each group manager, tell the Machine Provider about its instances. for template in models.InstanceTemplate.query(): logging.info( 'Retrieving instance template %s from project %s', template.template_name, template.template_project, ) api = gce.Project(template.template_project) try: instance_template = api.get_instance_template( template.template_name) except net.NotFoundError: logging.error( 'Instance template does not exist: %s', template.template_name, ) continue api = gce.Project(template.instance_group_project) properties = instance_template['properties'] disk_gb = int( properties['disks'][0]['initializeParams']['diskSizeGb']) memory_gb = float( gce.machine_type_to_memory(properties['machineType'])) num_cpus = gce.machine_type_to_num_cpus(properties['machineType']) os_family = machine_provider.OSFamily.lookup_by_name( template.os_family) dimensions = machine_provider.Dimensions( backend=machine_provider.Backend.GCE, disk_gb=disk_gb, memory_gb=memory_gb, num_cpus=num_cpus, os_family=os_family, ) try: instances = api.get_managed_instances( template.instance_group_name, template.zone) except net.NotFoundError: logging.warning( 'Instance group manager does not exist: %s', template.instance_group_name, ) continue policies = machine_provider.Policies( backend_attributes=[ machine_provider.KeyValuePair( key='group', value=template.instance_group_name), ], backend_project=handlers_pubsub. MachineProviderSubscriptionHandler.TOPIC_PROJECT, backend_topic=handlers_pubsub. MachineProviderSubscriptionHandler.TOPIC, on_reclamation=machine_provider.MachineReclamationPolicy. DELETE, ) process_instance_group( template.instance_group_name, dimensions, policies, instances, template.zone, template.instance_group_project, )
def create(key): """Creates an instance template from the given InstanceTemplateRevision. Args: key: ndb.Key for a models.InstanceTemplateRevision entity. Raises: net.Error: HTTP status code is not 200 (created) or 409 (already created). """ instance_template_revision = key.get() if not instance_template_revision: logging.warning('InstanceTemplateRevision does not exist: %s', key) return if not instance_template_revision.project: logging.warning('InstanceTemplateRevision project unspecified: %s', key) return if instance_template_revision.url: logging.info( 'Instance template for InstanceTemplateRevision already exists: %s', key, ) return if instance_template_revision.metadata: metadata = [{ 'key': k, 'value': v } for k, v in instance_template_revision.metadata.iteritems()] else: metadata = [] service_accounts = [{ 'email': service_account.name, 'scopes': service_account.scopes } for service_account in instance_template_revision.service_accounts] api = gce.Project(instance_template_revision.project) try: image_project = api.project_id if instance_template_revision.image_project: image_project = instance_template_revision.image_project result = api.create_instance_template( get_name(instance_template_revision), instance_template_revision.disk_size_gb, gce.get_image_url(image_project, instance_template_revision.image_name), instance_template_revision.machine_type, auto_assign_external_ip=instance_template_revision. auto_assign_external_ip, disk_type=instance_template_revision.disk_type, metadata=metadata, min_cpu_platform=instance_template_revision.min_cpu_platform, network_url=instance_template_revision.network_url, service_accounts=service_accounts, tags=instance_template_revision.tags, ) except net.Error as e: if e.status_code == 409: # If the instance template already exists, just record the URL. result = api.get_instance_template( get_name(instance_template_revision)) update_url(instance_template_revision.key, result['selfLink']) return else: raise update_url(instance_template_revision.key, result['targetLink'])