def run(**kwargs): try: thread_queue = [] for provider in list_providers("virtualcenter"): mgmt_sys = cfme_data['management_systems'][provider] creds = credentials[mgmt_sys['credentials']] hostname = mgmt_sys['hostname'] username = creds['username'] password = creds['password'] default_name = credentials['host_default']['username'] default_password = credentials['host_default']['password'] host_ip = mgmt_sys['ipaddress'] client = VMWareSystem(hostname, username, password) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(client, hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), default_name, default_password)) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception as e: print(e) return False
def run(**kwargs): thread_queue = [] for provider in list_providers("openstack"): mgmt_sys = cfme_data['management_systems'][provider] rhos_credentials = credentials[mgmt_sys['credentials']] default_host_creds = credentials['host_default'] username = rhos_credentials['username'] password = rhos_credentials['password'] auth_url = mgmt_sys['auth_url'] rhosip = mgmt_sys['ipaddress'] sshname = default_host_creds['username'] sshpass = default_host_creds['password'] if not net.is_pingable(rhosip): continue if not net.net_check(ports.SSH, rhosip): print("SSH connection to {}:{} failed, port unavailable".format( provider, ports.SSH)) continue thread = Thread(target=upload_template, args=(rhosip, sshname, sshpass, username, password, auth_url, provider, kwargs.get('image_url'), kwargs.get('template_name'))) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): """Calls all the functions needed to upload new template to RHEVM. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ thread_queue = [] valid_providers = [] providers = list_provider_keys("rhevm") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] for provider in providers: if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'rhevm': continue sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] rhevip = mgmt_sys[provider]['ipaddress'] else: mgmt_sys = cfme_data['management_systems'] ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevip = mgmt_sys[provider]['ipaddress'] print("RHEVM:{} verifying provider's state before template upload".format(provider)) if not net.is_pingable(rhevip): continue elif not is_ovirt_engine_running(rhevip, sshname, sshpass): print('RHEVM:{} ovirt-engine service not running..'.format(provider)) continue valid_providers.append(provider) for provider in valid_providers: if kwargs['provider_data']: sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevm_credentials = mgmt_sys[provider]['credentials'] username = credentials[rhevm_credentials]['username'] password = credentials[rhevm_credentials]['password'] rhevip = mgmt_sys[provider]['ipaddress'] thread = Thread(target=upload_template, args=(rhevip, sshname, sshpass, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): """Calls all the functions needed to upload new template to RHEVM. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ thread_queue = [] valid_providers = [] providers = list_providers("rhevm") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data']['management_systems'] for provider in providers: if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'rhevm': continue sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] rhevip = mgmt_sys[provider]['ipaddress'] else: mgmt_sys = cfme_data['management_systems'] ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevip = mgmt_sys[provider]['ipaddress'] print("RHEVM:{} verifying provider's state before template upload".format(provider)) if not net.is_pingable(rhevip): continue elif not is_ovirt_engine_running(rhevip, sshname, sshpass): print('RHEVM:{} ovirt-engine service not running..'.format(provider)) continue valid_providers.append(provider) for provider in valid_providers: if kwargs['provider_data']: sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] rhevm_credentials = mgmt_sys[provider]['credentials'] username = credentials[rhevm_credentials]['username'] password = credentials[rhevm_credentials]['password'] rhevip = mgmt_sys[provider]['ipaddress'] thread = Thread(target=upload_template, args=(rhevip, sshname, sshpass, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): """Calls the functions needed to cleanup templates on RHEVM providers. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ providers = cfme_data['management_systems'] for provider in providers: if cfme_data['management_systems'][provider]['type'] != 'rhevm': continue if args.provider: if args.provider != provider: continue mgmt_sys = cfme_data['management_systems'][provider] ssh_rhevm_creds = mgmt_sys['hosts'][0]['credentials'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] if not net.is_pingable( cfme_data['management_systems'][provider]['ipaddress']): continue elif not is_ovirt_engine_running( cfme_data['management_systems'][provider]['ipaddress'], sshname, sshpass): print('ovirt-engine service not running..') continue try: print('connecting to provider, to establish api handler') api = get_mgmt(provider).api edomain = get_edomain(api) if args.edomain: edomain = args.edomain path, edomain_ip = get_edomain_path(api, edomain) except Exception as e: logger.exception(e) continue try: print("\n--------Start of {}--------".format(provider)) cleanup_templates(api, edomain, args.days_old, args.max_templates) finally: change_edomain_state(api, 'maintenance', edomain) cleanup_empty_dir_on_edomain(path, edomain_ip, sshname, sshpass) change_edomain_state(api, 'active', edomain) print("--------End of {}--------\n".format(provider)) print("Provider Execution completed")
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True): providers = providers or list_providers() providers_data = cfme_data.get("management_systems", {}) delta = datetime.timedelta(hours=int(max_hours)) vms_to_delete = defaultdict(set) thread_queue = [] # precompile regexes matchers = [re.compile(text, re.IGNORECASE) for text in texts] for provider_key in providers: ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress', None) if ipaddress and not net.is_pingable(ipaddress): continue provider_type = providers_data[provider_key].get('type', None) thread = Thread(target=process_provider_vms, args=(provider_key, provider_type, matchers, delta, vms_to_delete)) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() # Join the queued calls for thread in thread_queue: thread.join() for provider_key, vm_set in vms_to_delete.items(): print('{}:'.format(provider_key)) for vm_name, vm_delta in vm_set: days, hours = vm_delta.days, vm_delta.seconds / 3600 print(' {} is {} days, {} hours old'.format(vm_name, days, hours)) if vms_to_delete and prompt: yesno = raw_input('Delete these VMs? [y/N]: ') if str(yesno).lower() != 'y': print('Exiting.') return 0 if not vms_to_delete: print('No VMs to delete.') thread_queue = [] for provider_key, vm_set in vms_to_delete.items(): thread = Thread(target=delete_provider_vms, args=(provider_key, [name for name, t_delta in vm_set])) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() print("Deleting finished")
def run(**kwargs): """Calls the functions needed to cleanup templates on RHEVM providers. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ providers = cfme_data['management_systems'] for provider in providers: if cfme_data['management_systems'][provider]['type'] != 'rhevm': continue if args.provider: if args.provider != provider: continue mgmt_sys = cfme_data['management_systems'][provider] ssh_rhevm_creds = mgmt_sys['hosts'][0]['credentials'] sshname = credentials[ssh_rhevm_creds]['username'] sshpass = credentials[ssh_rhevm_creds]['password'] if not net.is_pingable(cfme_data['management_systems'][provider]['ipaddress']): continue elif not is_ovirt_engine_running(cfme_data['management_systems'][provider]['ipaddress'], sshname, sshpass): print('ovirt-engine service not running..') continue try: print('connecting to provider, to establish api handler') api = get_mgmt(provider).api edomain = get_edomain(api) if args.edomain: edomain = args.edomain path, edomain_ip = get_edomain_path(api, edomain) except Exception as e: logger.exception(e) continue try: print("\n--------Start of {}--------".format(provider)) cleanup_templates(api, edomain, args.days_old, args.max_templates) finally: change_edomain_state(api, 'maintenance', edomain) cleanup_empty_dir_on_edomain(path, edomain_ip, sshname, sshpass) change_edomain_state(api, 'active', edomain) print("--------End of {}--------\n".format(provider)) print("Provider Execution completed")
def run(**kwargs): """Calls the functions needed to cleanup templates on RHEVM providers. This is called either by template_upload_all script, or by main function. Args: **kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm']. """ providers = cfme_data['management_systems'] for provider in [prov for prov in providers if providers[prov]['type'] == 'rhevm']: # If a provider was passed, only cleanup on it, otherwise all rhevm providers cli_provider = kwargs.get('provider', None) if cli_provider and cli_provider != provider: continue provider_mgmt = get_mgmt(provider) if not net.is_pingable(provider_mgmt.kwargs.get('ipaddress', None)): continue elif not is_ovirt_engine_running(provider_mgmt): print('ovirt-engine service not running..') continue try: print('connecting to provider, to establish api handler') edomain = kwargs.get('edomain', None) if not edomain: edomain = provider_mgmt.kwargs['template_upload']['edomain'] except Exception as e: logger.exception(e) continue try: print("\n--------Start of {}--------".format(provider)) cleanup_templates(provider_mgmt.api, edomain, kwargs.get('days_old'), kwargs.get('max_templates')) finally: change_edomain_state(provider_mgmt, 'maintenance', edomain) cleanup_empty_dir_on_edomain(provider_mgmt, edomain) change_edomain_state(provider_mgmt, 'active', edomain) print("--------End of {}--------\n".format(provider)) print("Provider Execution completed")
def run(**kwargs): thread_queue = [] providers = list_provider_keys("openstack") if kwargs['provider_data']: provider_data = kwargs['provider_data'] mgmt_sys = providers = provider_data['management_systems'] for provider in providers: if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'openstack': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] sshname = mgmt_sys[provider]['sshname'] sshpass = mgmt_sys[provider]['sshpass'] else: mgmt_sys = cfme_data['management_systems'] rhos_credentials = credentials[mgmt_sys[provider]['credentials']] default_host_creds = credentials['host_default'] username = rhos_credentials['username'] password = rhos_credentials['password'] sshname = default_host_creds['username'] sshpass = default_host_creds['password'] rhosip = mgmt_sys[provider]['ipaddress'] auth_url = mgmt_sys[provider]['auth_url'] if not net.is_pingable(rhosip): continue if not net.net_check(ports.SSH, rhosip): print("SSH connection to {}:{} failed, port unavailable".format( provider, ports.SSH)) continue thread = Thread(target=upload_template, args=(rhosip, sshname, sshpass, username, password, auth_url, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def run(**kwargs): try: thread_queue = [] providers = list_provider_keys("virtualcenter") if kwargs['provider_data']: mgmt_sys = providers = kwargs['provider_data'][ 'management_systems'] for provider in providers: if kwargs['provider_data']: if mgmt_sys[provider]['type'] != 'virtualcenter': continue username = mgmt_sys[provider]['username'] password = mgmt_sys[provider]['password'] else: mgmt_sys = cfme_data['management_systems'] creds = credentials[mgmt_sys[provider]['credentials']] username = creds['username'] password = creds['password'] host_ip = mgmt_sys[provider]['ipaddress'] hostname = mgmt_sys[provider]['hostname'] client = VMWareSystem(hostname, username, password) if not net.is_pingable(host_ip): continue thread = Thread(target=upload_template, args=(client, hostname, username, password, provider, kwargs.get('image_url'), kwargs.get('template_name'), kwargs['provider_data'], kwargs['stream'])) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() except Exception: logger.exception('Exception during run method') return False
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_provider_keys()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data['management_systems'][provider_key].get( 'ipaddress', None) if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate( api, api.providertemplate.get())['objects'] ] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template( template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print('Ignoring {} from group {}'.format(template_name, group_name)) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print('Template {} already tracked for provider {}'.format( template_name, provider_key)) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print('Added {} template {} on provider {} (datestamp: {})'. format(group_name, template_name, provider_key, datestamp)) except SlumberHttpBaseException as ex: print("{}\t{}".format(ex.response.status_code, ex.content)) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: provider_key, template_name = pt['provider']['key'], pt['template'][ 'name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print("Cleaning up template {} on {}".format( template_name, provider_key)) trackerbot.delete_provider_template(api, provider_key, template_name) else: print("Skipping template cleanup {} on unknown provider {}". format(template_name, provider_key)) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: print("Deleting template {} (no providers)".format( template['name'])) api.template(template['name']).delete()
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True): """ Main method for the cleanup process Generates regex match objects Checks providers for cleanup boolean in yaml Checks provider connectivity (using ping) Threads process_provider_vms to build list of vms to delete Prompts user to continue with delete Threads deleting of the vms :param texts: list of strings to match against :param max_hours: integer maximum number of hours that the VM can exist for :param providers: list of provider keys :param prompt: boolean, whether or not to prompt the user for each delete :return: 0 if user declines delete when prompt is True """ providers = providers or list_provider_keys() delta = datetime.timedelta(hours=int(max_hours)) vms_to_delete = defaultdict(set) thread_queue = [] # precompile regexes matchers = [re.compile(text, re.IGNORECASE) for text in texts] print( 'Matching VM names against the following case-insensitive strings: {}'. format(texts)) for provider_key in providers: # check for cleanup boolean if not cfme_data['management_systems'][provider_key].get( 'cleanup', False): print('Skipping {}, cleanup map set to false or missing in yaml'. format(provider_key)) continue ipaddress = cfme_data['management_systems'][provider_key].get( 'ipaddress') if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=process_provider_vms, args=(provider_key, matchers, delta, vms_to_delete)) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() # Join the queued calls for thread in thread_queue: thread.join() if vms_to_delete and prompt: yesno = raw_input('Delete these VMs? [y/N]: ') if str(yesno).lower() != 'y': print('Exiting.') return 0 if not vms_to_delete: print('No VMs to delete.') thread_queue = [] for provider_key, vm_set in vms_to_delete.items(): provider_mgmt = get_mgmt(provider_key) names_ages = [] for vm_name, vm_delta in vm_set: days, hours = vm_delta.days, vm_delta.seconds / 3600 age = '{} days, {} hours old'.format(days, hours) names_ages.append(NameAge(vm_name, age)) thread = Thread(target=delete_provider_vms, args=(provider_key, provider_mgmt, names_ages)) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() with open(args.outfile, 'a') as report: report.write('## VM/Instances deleted via:\n' '## text matches: {}\n' '## age matches: {}\n'.format(texts, max_hours)) message = tabulate( deleted_vms_list, headers=['Provider', 'Name', 'Age', 'Status', 'Delete RC'], tablefmt='orgtbl') report.write(message + '\n') print(message) print("Deleting finished") return 0
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True): providers = providers or list_provider_keys() providers_data = cfme_data.get("management_systems", {}) delta = datetime.timedelta(hours=int(max_hours)) vms_to_delete = defaultdict(set) thread_queue = [] # precompile regexes matchers = [re.compile(text, re.IGNORECASE) for text in texts] for provider_key in providers: # check for cleanup boolean if not cfme_data['management_systems'][provider_key].get( 'cleanup', False): print('Skipping {}, cleanup set to false in yaml'.format( provider_key)) continue ipaddress = cfme_data['management_systems'][provider_key].get( 'ipaddress', None) if ipaddress and not net.is_pingable(ipaddress): continue provider_type = providers_data[provider_key].get('type', None) list_vms = args.list_vms and provider_type in args.provider_type thread = Thread(target=process_provider_vms, args=(provider_key, provider_type, matchers, delta, vms_to_delete, list_vms)) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() # Join the queued calls for thread in thread_queue: thread.join() if providers_vm_list: with open(args.outfile, 'a+') as report: message = tabulate(providers_vm_list, headers=[ 'ProviderType', 'ProviderKey', 'InstanceName', 'CreatedSince', 'InstanceType', 'InstanceStatus' ], tablefmt='orgtbl') report.write(message) print(message) for provider_key, vm_set in vms_to_delete.items(): print('{}:'.format(provider_key)) for vm_name, vm_delta in vm_set: days, hours = vm_delta.days, vm_delta.seconds / 3600 print(' {} is {} days, {} hours old'.format(vm_name, days, hours)) if vms_to_delete and prompt: yesno = raw_input('Delete these VMs? [y/N]: ') if str(yesno).lower() != 'y': print('Exiting.') return 0 if not vms_to_delete: print('No VMs to delete.') thread_queue = [] for provider_key, vm_set in vms_to_delete.items(): thread = Thread(target=delete_provider_vms, args=(provider_key, [name for name, t_delta in vm_set])) # Mark as daemon thread for easy-mode KeyboardInterrupt handling thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join() print("Deleting finished")
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_provider_keys()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress', None) if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template(template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print('Ignoring {} from group {}'.format(template_name, group_name)) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print('Template {} already tracked for provider {}'.format( template_name, provider_key)) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print('Added {} template {} on provider {} (datestamp: {})'.format( group_name, template_name, provider_key, datestamp)) except SlumberHttpBaseException as ex: print("{}\t{}".format(ex.response.status_code, ex.content)) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: provider_key, template_name = pt['provider']['key'], pt['template']['name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print("Cleaning up template {} on {}".format(template_name, provider_key)) trackerbot.delete_provider_template(api, provider_key, template_name) else: print("Skipping template cleanup {} on unknown provider {}".format( template_name, provider_key)) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: print("Deleting template {} (no providers)".format(template['name'])) api.template(template['name']).delete()