def ensure_entities_exist(key, max_concurrent=50): """Ensures Instance entities exist for the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. max_concurrent: Maximun number of entities to create concurrently. """ urls = fetch(key) if not urls: instance_group_managers.set_instances(key, []) return base_name = key.parent().parent().id() revision = key.parent().id() zone = key.id() keys = { url: get_instance_key(base_name, revision, zone, gce.extract_instance_name(url)) for url in urls } utilities.batch_process_async( urls, lambda url: ensure_entity_exists(keys[url], url, key), max_concurrent=max_concurrent, ) instance_group_managers.set_instances(key, keys.values())
def ensure_entities_exist(key, max_concurrent=50): """Ensures Instance entities exist for the given instance group manager. Args: key: ndb.Key for a models.InstanceGroupManager entity. max_concurrent: Maximun number of entities to create concurrently. """ urls = fetch(key) if not urls: return keys = { url: ndb.Key(models.Instance, gce.extract_instance_name(url), parent=key) for url in urls } utilities.batch_process_async( urls, lambda url: ensure_entity_exists(keys[url], url), max_concurrent=max_concurrent, ) add_instances(key, keys.values())
def parse(template_cfgs, manager_cfgs, max_concurrent=50, max_concurrent_igm=5): """Ensures entities exist for the given config. Ensures the existence of the root InstanceTemplate, the active InstanceTemplateRevision, and the active InstanceGroupManagers. Args: template_cfgs: List of proto.config_pb2.InstanceTemplateConfig.InstanceTemplate. manager_cfgs: List of proto.config_pb2.InstanceGroupManagerConfig.InstanceGroupManagers. max_concurrent: Maximum number to create concurrently. max_concurrent_igm: Maximum number of InstanceGroupManagers to create concurrently for each InstanceTemplate/InstanceTemplateRevision. The actual maximum number of concurrent entities being created will be max_concurrent * max_concurrent_igm. Returns: ndb.Key for the root models.InstanceTemplate entity. """ manager_cfg_map = collections.defaultdict(list) for manager_cfg in manager_cfgs: manager_cfg_map[manager_cfg.template_base_name].append(manager_cfg) def f(template_cfg): return ensure_entities_exist( template_cfg, manager_cfg_map.get(template_cfg.base_name), max_concurrent=max_concurrent_igm, ) utilities.batch_process_async(template_cfgs, f, max_concurrent=max_concurrent)
def poll(): """Polls and processes Pub/Sub messages.""" response = pubsub.pull(pubsub.full_subscription_name( get_machine_provider_topic_project(), get_machine_provider_subscription(), )) utilities.batch_process_async(response.get('receivedMessages', []), process)
def cleanup_instance_templates(max_concurrent=50): """Deletes InstanceTemplates. Args: max_concurrent: Maximum number to delete concurrently. """ utilities.batch_process_async( models.InstanceTemplate.query().fetch(keys_only=True), delete_instance_template, max_concurrent=max_concurrent, )
def cleanup_instance_template_revisions(max_concurrent=50): """Deletes drained InstanceTemplateRevisions. Args: max_concurrent: Maximum number to delete concurrently. """ utilities.batch_process_async( instance_templates.get_drained_instance_template_revisions(), delete_instance_template_revision, max_concurrent=max_concurrent, )
def cleanup_instance_group_managers(max_concurrent=50): """Deletes drained InstanceGroupManagers. Args: max_concurrent: Maximum number to delete concurrently. """ utilities.batch_process_async( instance_group_managers.get_drained_instance_group_managers(), delete_instance_group_manager, max_concurrent=max_concurrent, )
def parse(template_cfgs, manager_cfgs, max_concurrent=50, max_concurrent_igm=5): """Ensures entities exist for and match the given config. Ensures the existence of the root InstanceTemplate, the active InstanceTemplateRevision, and the active InstanceGroupManagers. Args: template_cfgs: List of proto.config_pb2.InstanceTemplateConfig.InstanceTemplates. manager_cfgs: List of proto.config_pb2.InstanceGroupManagerConfig.InstanceGroupManagers. max_concurrent: Maximum number to create concurrently. max_concurrent_igm: Maximum number of InstanceGroupManagers to create concurrently for each InstanceTemplate/InstanceTemplateRevision. The actual maximum number of concurrent entities being created will be max_concurrent * max_concurrent_igm. """ manager_cfg_map = collections.defaultdict(list) for manager_cfg in manager_cfgs: manager_cfg_map[manager_cfg.template_base_name].append(manager_cfg) def f(template_cfg): return ensure_entities_exist( template_cfg, manager_cfg_map.get(template_cfg.base_name, []), max_concurrent=max_concurrent_igm, ) utilities.batch_process_async(template_cfgs, f, max_concurrent=max_concurrent) # Now go over every InstanceTemplate not mentioned anymore in the config and # mark its active InstanceTemplateRevision as drained. template_names = set(template_cfg.base_name for template_cfg in template_cfgs) instance_template_keys = [] for instance_template in models.InstanceTemplate.query().fetch(): if instance_template.key.id() not in template_names: if instance_template.active: instance_template_keys.append(instance_template.key) utilities.batch_process_async( instance_template_keys, ensure_instance_template_revision_drained, max_concurrent=max_concurrent, )
def poll(): """Polls and processes Pub/Sub messages.""" response = pubsub.pull( pubsub.full_subscription_name( get_machine_provider_topic_project(), get_machine_provider_subscription(), ), max_messages=400, ) if response.get('receivedMessages', []): logging.info('Messages received: %s', len(response['receivedMessages'])) messages = split_by_entity_group(response['receivedMessages']) utilities.batch_process_async(messages, process)