def run(**kwargs): """Calls the functions needed to cleanup templates on RHEVM providers. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ providers = cfme_data['management_systems'] for provider in [prov for prov in providers if providers[prov]['type'] == 'rhevm']: # If a provider was passed, only cleanup on it, otherwise all rhevm providers cli_provider = kwargs.get('provider') if cli_provider and cli_provider != provider: continue print("\n--------Start of {}--------".format(provider)) provider_mgmt = get_mgmt(provider) try: # Raise exceptions here to log the end of provider section if not net.is_pingable(provider_mgmt.kwargs.get('ipaddress')): raise ValueError('Failed to ping provider.') elif not is_ovirt_engine_running(provider_mgmt): print('ovirt-engine service not running..') raise ValueError('ovirt-engine not running on provider') print('connecting to provider, to establish api handler') edomain = kwargs.get('edomain') if not edomain: edomain = provider_mgmt.kwargs['template_upload']['edomain'] # Test API connection to provider, raises RequestError provider_mgmt.api # noqa except Exception as e: print('Failed connecting to provider: {}'.format(e)) logger.exception(e) print("--------FAILURE End of {}--------\n".format(provider)) continue try: cleanup_templates(provider_mgmt.api, edomain, kwargs.get('days_old'), kwargs.get('max_templates')) finally: change_edomain_state(provider_mgmt, 'maintenance', edomain) cleanup_empty_dir_on_edomain(provider_mgmt, edomain) change_edomain_state(provider_mgmt, 'active', edomain) print("--------End of {}--------\n".format(provider)) print("Provider Execution completed")
def run(**kwargs): thread_queue = [] providers = list_provider_keys("virtualcenter") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] else: mgmt_sys = cfme_data.management_systems # Store thread results, no need to use a lock # because threads will not be adding new keys results = {provider: None for provider in providers} for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on %s due to block_upload', provider) continue if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'virtualcenter': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: creds = credentials[mgmt_sys[provider]['credentials']] username = creds['username'] password = creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] client = VMWareSystem(hostname, username, password) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(client, hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'], results)) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() failed_providers = [provider for provider, result in results.items() if result is False] skipped_providers = [provider for provider, result in results.items() if result is None] passed_providers = [provider for provider, result in results.items() if result] logger.info("providers skipped: %s", skipped_providers) logger.info("providers passed: %s", passed_providers) logger.info("providers failed: %s", failed_providers) if not passed_providers: raise Exception("Template upload failed for all providers") else: logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs): try: thread_queue = [] providers = list_provider_keys("openshift") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data'][ 'management_systems'] else: mgmt_sys = cfme_data['management_systems'] for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on {} due to block_upload'.format( provider)) continue if 'podtesting' not in mgmt_sys[provider]['tags']: continue if kwargs['provider_data']: username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']] username = ssh_creds['username'] password = ssh_creds['password'] oc_creds = credentials[mgmt_sys[provider]['credentials']] oc_username = oc_creds['username'] oc_password = oc_creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] upload_parameters = cfme_data['template_upload'][ 'template_upload_openshift'] upload_folder = kwargs.get('upload_folder', upload_parameters['upload_folder']) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'], upload_folder, oc_username, oc_password)) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception: logger.exception('Exception during run method') return False
def run(**kwargs): """Calls the functions needed to cleanup templates on RHEVM providers. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ providers = cfme_data['management_systems'] for provider in [ prov for prov in providers if providers[prov]['type'] == 'rhevm' ]: # If a provider was passed, only cleanup on it, otherwise all rhevm providers cli_provider = kwargs.get('provider') if cli_provider and cli_provider != provider: continue print("\n--------Start of {}--------".format(provider)) provider_mgmt = get_mgmt(provider) try: # Raise exceptions here to log the end of provider section if not net.is_pingable(provider_mgmt.kwargs.get('ipaddress')): raise ValueError('Failed to ping provider.') elif not is_ovirt_engine_running(provider_mgmt): print('ovirt-engine service not running..') raise ValueError('ovirt-engine not running on provider') print('connecting to provider, to establish api handler') edomain = kwargs.get('edomain') if not edomain: edomain = provider_mgmt.kwargs['template_upload']['edomain'] # Test API connection to provider, raises RequestError provider_mgmt.api # noqa except Exception as e: print('Failed connecting to provider: {}'.format(e)) logger.exception(e) print("--------FAILURE End of {}--------\n".format(provider)) continue try: cleanup_templates(provider_mgmt.api, edomain, kwargs.get('days_old'), kwargs.get('max_templates')) finally: change_edomain_state(provider_mgmt, 'maintenance', edomain) cleanup_empty_dir_on_edomain(provider_mgmt, edomain) change_edomain_state(provider_mgmt, 'active', edomain) print("--------End of {}--------\n".format(provider)) print("Provider Execution completed")
def run(**kwargs): thread_queue = [] providers = list_provider_keys("openstack") if kwargs['provider_data']: provider_data = kwargs['provider_data'] mgmt_sys = providers = provider_data['management_systems'] else: mgmt_sys = cfme_data.management_systems for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info( 'Skipping upload on {} due to block_upload'.format(provider)) continue if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'openstack': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] else: mgmt_sys = cfme_data['management_systems'] rhos_credentials = credentials[mgmt_sys[provider]['credentials']] default_host_creds = credentials['host_default'] username = rhos_credentials['username'] password = rhos_credentials['password'] sshname = default_host_creds['username'] sshpass = default_host_creds['password'] rhosip = mgmt_sys[provider]['ipaddress'] auth_url = mgmt_sys[provider]['auth_url'] if not net.is_pingable(rhosip): continue if not net.net_check(ports.SSH, rhosip): logger.error("SSH connection to %r:%r failed, port unavailable", provider, ports.SSH) continue thread = Thread(target=upload_template, args=(rhosip, sshname, sshpass, username, password, auth_url, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): try: thread_queue = [] providers = list_provider_keys("openshift") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] else: mgmt_sys = cfme_data['management_systems'] for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on {} due to block_upload'.format(provider)) continue if 'podtesting' not in mgmt_sys[provider]['tags']: continue if kwargs['provider_data']: username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']] username = ssh_creds['username'] password = ssh_creds['password'] oc_creds = credentials[mgmt_sys[provider]['credentials']] oc_username = oc_creds['username'] oc_password = oc_creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] upload_parameters = cfme_data['template_upload']['template_upload_openshift'] upload_folder = kwargs.get('upload_folder', upload_parameters['upload_folder']) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'], upload_folder, oc_username, oc_password)) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception: logger.exception('Exception during run method') return False
def run(**kwargs): thread_queue = [] providers = list_provider_keys("openstack") if kwargs['provider_data']: provider_data = kwargs['provider_data'] mgmt_sys = providers = provider_data['management_systems'] else: mgmt_sys = cfme_data.management_systems for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on {} due to block_upload'.format(provider)) continue if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'openstack': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] else: mgmt_sys = cfme_data['management_systems'] rhos_credentials = credentials[mgmt_sys[provider]['credentials']] default_host_creds = credentials['host_default'] username = rhos_credentials['username'] password = rhos_credentials['password'] sshname = default_host_creds['username'] sshpass = default_host_creds['password'] rhosip = mgmt_sys[provider]['ipaddress'] auth_url = mgmt_sys[provider]['auth_url'] if not net.is_pingable(rhosip): continue if not net.net_check(ports.SSH, rhosip): logger.error("SSH connection to %r:%r failed, port unavailable", provider, ports.SSH) continue thread = Thread(target=upload_template, args=(rhosip, sshname, sshpass, username, password, auth_url, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): try: thread_queue = [] providers = list_provider_keys("virtualcenter") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data'][ 'management_systems'] else: mgmt_sys = cfme_data.management_systems for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on {} due to block_upload'.format( provider)) continue if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'virtualcenter': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: creds = credentials[mgmt_sys[provider]['credentials']] username = creds['username'] password = creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] client = VMWareSystem(hostname, username, password) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(client, hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception: logger.exception('Exception during run method') return False
def run(**kwargs): try: thread_queue = [] providers = list_provider_keys("virtualcenter") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] else: mgmt_sys = cfme_data.management_systems for provider in providers: # skip provider if block_upload is set if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): logger.info('Skipping upload on {} due to block_upload'.format(provider)) continue if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'virtualcenter': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: creds = credentials[mgmt_sys[provider]['credentials']] username = creds['username'] password = creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] client = VMWareSystem(hostname, username, password) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(client, hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception: logger.exception('Exception during run method') return False
def run(**kwargs): """Calls all the functions needed to upload new template to RHEVM. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ thread_queue = [] valid_providers = [] providers = list_provider_keys("rhevm") if kwargs.get('provider_data'): mgmt_sys = providers = kwargs['provider_data']['management_systems'] for provider in providers: if kwargs.get('provider_data'): if mgmt_sys[provider]['type'] != 'rhevm': continue sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] rhevip = mgmt_sys[provider]['ipaddress'] else: mgmt_sys = cfme_data['management_systems'] ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevip = mgmt_sys[provider]['ipaddress'] if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): # Providers template_upload section indicates upload should not happen on this provider continue logger.info( "RHEVM:%r verifying provider's state before template upload", provider) if not net.is_pingable(rhevip): continue elif not is_ovirt_engine_running(rhevip, sshname, sshpass): logger.info('RHEVM:%r ovirt-engine service not running..', provider) continue valid_providers.append(provider) for provider in valid_providers: if kwargs.get('provider_data'): sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevm_credentials = mgmt_sys[provider]['credentials'] username = credentials[rhevm_credentials]['username'] password = credentials[rhevm_credentials]['password'] rhevip = mgmt_sys[provider]['ipaddress'] thread = Thread(target=upload_template, args=(rhevip, sshname, sshpass, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs.get('provider_data'), kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): """Calls all the functions needed to upload new template to RHEVM. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ thread_queue = [] valid_providers = [] providers = list_provider_keys("rhevm") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] for provider in providers: if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'rhevm': continue sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] rhevip = mgmt_sys[provider]['ipaddress'] else: mgmt_sys = cfme_data['management_systems'] ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevip = mgmt_sys[provider]['ipaddress'] if (mgmt_sys[provider].get('template_upload') and mgmt_sys[provider]['template_upload'].get('block_upload')): # Providers template_upload section indicates upload should not happen on this provider continue logger.info("RHEVM:%r verifying provider's state before template upload", provider) if not net.is_pingable(rhevip): continue elif not is_ovirt_engine_running(rhevip, sshname, sshpass): logger.info('RHEVM:%r ovirt-engine service not running..', provider) continue valid_providers.append(provider) for provider in valid_providers: if kwargs['provider_data']: sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevm_credentials = mgmt_sys[provider]['credentials'] username = credentials[rhevm_credentials]['username'] password = credentials[rhevm_credentials]['password'] rhevip = mgmt_sys[provider]['ipaddress'] thread = Thread(target=upload_template, args=(rhevip, sshname, sshpass, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'], kwargs['glance'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def main(trackerbot_url, mark_usable=None, selected_provider=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = (set(list_provider_keys()) if not selected_provider else set(selected_provider)) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data.management_systems[provider_key].get('ipaddress') if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate( api, api.providertemplate.get())['objects'] ] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) template_info = TemplateName.parse_template(template_name) # Don't want sprout templates if template_info.group_name in ('sprout', 'rhevm-internal'): logger.info('Ignoring %s from group %s', template_name, template_info.group_name) continue seen_templates.add(template_name) group = trackerbot.Group(template_info.group_name, stream=template_info.stream) try: template = trackerbot.Template(template_name, group, template_info.datestamp) except ValueError: logger.exception('Failure parsing provider %s template: %s', provider_key, template_name) continue for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: logger.info('Template %s already tracked for provider %s', template_name, provider_key) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) logger.info( 'Added %s template %s on provider %s (datestamp: %s)', template_info.group_name, template_name, provider_key, template_info.datestamp) except SlumberHttpBaseException: logger.exception('%s: exception marking template %s', provider, template) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: key, template_name = pt['provider']['key'], pt['template']['name'] if key not in template_providers[ template_name] and key not in unresponsive_providers: if key in all_providers: logger.info("Cleaning up template %s on %s", template_name, key) trackerbot.delete_provider_template(api, key, template_name) else: logger.info( "Skipping template cleanup %s on unknown provider %s", template_name, key) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: logger.info("Deleting template %s (no providers)", template['name']) api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_provider_keys()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress') if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template(template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print('Ignoring {} from group {}'.format(template_name, group_name)) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print('Template {} already tracked for provider {}'.format( template_name, provider_key)) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print('Added {} template {} on provider {} (datestamp: {})'.format( group_name, template_name, provider_key, datestamp)) except SlumberHttpBaseException as ex: print("{}\t{}".format(ex.response.status_code, ex.content)) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: provider_key, template_name = pt['provider']['key'], pt['template']['name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print("Cleaning up template {} on {}".format(template_name, provider_key)) trackerbot.delete_provider_template(api, provider_key, template_name) else: print("Skipping template cleanup {} on unknown provider {}".format( template_name, provider_key)) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: print("Deleting template {} (no providers)".format(template['name'])) api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None, selected_provider=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = (set(list_provider_keys()) if not selected_provider else set(selected_provider)) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data.management_systems[provider_key].get('ipaddress') if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) template_info = TemplateName.parse_template(template_name) # it turned out that some providers like ec2 may have templates w/o names. # this is easy protection against such issue. if not template_name.strip(): logger.warn('Ignoring template w/o name on provider %s', provider_key) continue # Don't want sprout templates if template_info.group_name in ('sprout', 'rhevm-internal'): logger.info('Ignoring %s from group %s', template_name, template_info.group_name) continue seen_templates.add(template_name) group = trackerbot.Group(template_info.group_name, stream=template_info.stream) try: template = trackerbot.Template(template_name, group, template_info.datestamp) except ValueError: logger.exception('Failure parsing provider %s template: %s', provider_key, template_name) continue for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: logger.info('Template %s already tracked for provider %s', template_name, provider_key) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) logger.info('Added %s template %s on provider %s (datestamp: %s)', template_info.group_name, template_name, provider_key, template_info.datestamp) except SlumberHttpBaseException: logger.exception('%s: exception marking template %s', provider, template) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: key, template_name = pt['provider']['key'], pt['template']['name'] if key not in template_providers[template_name] and key not in unresponsive_providers: if key in all_providers: logger.info("Cleaning up template %s on %s", template_name, key) trackerbot.delete_provider_template(api, key, template_name) else: logger.info("Skipping template cleanup %s on unknown provider %s", template_name, key) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers'] and template['name'].strip(): logger.info("Deleting template %s (no providers)", template['name']) api.template(template['name']).delete()