def ensure_entities_exist(key, max_concurrent=50):
"""Ensures Instance entities exist for the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
max_concurrent: Maximun number of entities to create concurrently.
"""
urls = fetch(key)
if not urls:
instance_group_managers.set_instances(key, [])
return
base_name = key.parent().parent().id()
revision = key.parent().id()
zone = key.id()
keys = {
url: get_instance_key(base_name, revision, zone,
gce.extract_instance_name(url))
for url in urls
}
utilities.batch_process_async(
urls,
lambda url: ensure_entity_exists(keys[url], url, key),
max_concurrent=max_concurrent,
)
instance_group_managers.set_instances(key, keys.values())
def ensure_entities_exist(key, max_concurrent=50):
"""Ensures Instance entities exist for the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
max_concurrent: Maximun number of entities to create concurrently.
"""
urls = fetch(key)
if not urls:
return
keys = {
url: ndb.Key(models.Instance,
gce.extract_instance_name(url),
parent=key)
for url in urls
}
utilities.batch_process_async(
urls,
lambda url: ensure_entity_exists(keys[url], url),
max_concurrent=max_concurrent,
)
add_instances(key, keys.values())
def parse(template_cfgs, manager_cfgs, max_concurrent=50, max_concurrent_igm=5):
"""Ensures entities exist for the given config.
Ensures the existence of the root InstanceTemplate, the active
InstanceTemplateRevision, and the active InstanceGroupManagers.
Args:
template_cfgs: List of
proto.config_pb2.InstanceTemplateConfig.InstanceTemplate.
manager_cfgs: List of
proto.config_pb2.InstanceGroupManagerConfig.InstanceGroupManagers.
max_concurrent: Maximum number to create concurrently.
max_concurrent_igm: Maximum number of InstanceGroupManagers to create
concurrently for each InstanceTemplate/InstanceTemplateRevision. The
actual maximum number of concurrent entities being created will be
max_concurrent * max_concurrent_igm.
Returns:
ndb.Key for the root models.InstanceTemplate entity.
"""
manager_cfg_map = collections.defaultdict(list)
for manager_cfg in manager_cfgs:
manager_cfg_map[manager_cfg.template_base_name].append(manager_cfg)
def f(template_cfg):
return ensure_entities_exist(
template_cfg,
manager_cfg_map.get(template_cfg.base_name),
max_concurrent=max_concurrent_igm,
)
utilities.batch_process_async(template_cfgs, f, max_concurrent=max_concurrent)
def poll():
"""Polls and processes Pub/Sub messages."""
response = pubsub.pull(pubsub.full_subscription_name(
get_machine_provider_topic_project(),
get_machine_provider_subscription(),
))
utilities.batch_process_async(response.get('receivedMessages', []), process)
def cleanup_instance_templates(max_concurrent=50):
"""Deletes InstanceTemplates.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
models.InstanceTemplate.query().fetch(keys_only=True),
delete_instance_template,
max_concurrent=max_concurrent,
)
def cleanup_instance_template_revisions(max_concurrent=50):
"""Deletes drained InstanceTemplateRevisions.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
instance_templates.get_drained_instance_template_revisions(),
delete_instance_template_revision,
max_concurrent=max_concurrent,
)
def cleanup_instance_group_managers(max_concurrent=50):
"""Deletes drained InstanceGroupManagers.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
instance_group_managers.get_drained_instance_group_managers(),
delete_instance_group_manager,
max_concurrent=max_concurrent,
)
def cleanup_instance_templates(max_concurrent=50):
"""Deletes InstanceTemplates.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
models.InstanceTemplate.query().fetch(keys_only=True),
delete_instance_template,
max_concurrent=max_concurrent,
)
def cleanup_instance_template_revisions(max_concurrent=50):
"""Deletes drained InstanceTemplateRevisions.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
instance_templates.get_drained_instance_template_revisions(),
delete_instance_template_revision,
max_concurrent=max_concurrent,
)
def cleanup_instance_group_managers(max_concurrent=50):
"""Deletes drained InstanceGroupManagers.
Args:
max_concurrent: Maximum number to delete concurrently.
"""
utilities.batch_process_async(
instance_group_managers.get_drained_instance_group_managers(),
delete_instance_group_manager,
max_concurrent=max_concurrent,
)
def parse(template_cfgs,
manager_cfgs,
max_concurrent=50,
max_concurrent_igm=5):
"""Ensures entities exist for and match the given config.
Ensures the existence of the root InstanceTemplate, the active
InstanceTemplateRevision, and the active InstanceGroupManagers.
Args:
template_cfgs: List of
proto.config_pb2.InstanceTemplateConfig.InstanceTemplates.
manager_cfgs: List of
proto.config_pb2.InstanceGroupManagerConfig.InstanceGroupManagers.
max_concurrent: Maximum number to create concurrently.
max_concurrent_igm: Maximum number of InstanceGroupManagers to create
concurrently for each InstanceTemplate/InstanceTemplateRevision. The
actual maximum number of concurrent entities being created will be
max_concurrent * max_concurrent_igm.
"""
manager_cfg_map = collections.defaultdict(list)
for manager_cfg in manager_cfgs:
manager_cfg_map[manager_cfg.template_base_name].append(manager_cfg)
def f(template_cfg):
return ensure_entities_exist(
template_cfg,
manager_cfg_map.get(template_cfg.base_name, []),
max_concurrent=max_concurrent_igm,
)
utilities.batch_process_async(template_cfgs,
f,
max_concurrent=max_concurrent)
# Now go over every InstanceTemplate not mentioned anymore in the config and
# mark its active InstanceTemplateRevision as drained.
template_names = set(template_cfg.base_name
for template_cfg in template_cfgs)
instance_template_keys = []
for instance_template in models.InstanceTemplate.query().fetch():
if instance_template.key.id() not in template_names:
if instance_template.active:
instance_template_keys.append(instance_template.key)
utilities.batch_process_async(
instance_template_keys,
ensure_instance_template_revision_drained,
max_concurrent=max_concurrent,
)
def poll():
"""Polls and processes Pub/Sub messages."""
response = pubsub.pull(
pubsub.full_subscription_name(
get_machine_provider_topic_project(),
get_machine_provider_subscription(),
),
max_messages=400,
)
if response.get('receivedMessages', []):
logging.info('Messages received: %s',
len(response['receivedMessages']))
messages = split_by_entity_group(response['receivedMessages'])
utilities.batch_process_async(messages, process)
def ensure_entities_exist(key, max_concurrent=50):
"""Ensures Instance entities exist for the given instance group manager.
Args:
key: ndb.Key for a models.InstanceGroupManager entity.
max_concurrent: Maximun number of entities to create concurrently.
"""
urls = fetch(key)
if not urls:
return
keys = {
url: ndb.Key(models.Instance, gce.extract_instance_name(url), parent=key)
for url in urls
}
utilities.batch_process_async(
urls,
lambda url: ensure_entity_exists(keys[url], url),
max_concurrent=max_concurrent,
)
add_instances(key, keys.values())
